From 6e6073ff4c4e01ab3171d3724dd6fe5f44df463e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 17 Oct 2011 14:19:17 -0700 Subject: [PATCH 001/149] it compiles (but doesn't do anything useful) --- src/CMakeLists.txt | 4 ++ src/Func.cc | 3 ++ src/InputMgr.cc | 90 +++++++++++++++++++++++++++++++++++++++++ src/InputMgr.h | 24 +++++++++++ src/InputReader.cc | 12 ++++++ src/InputReader.h | 29 +++++++++++++ src/InputReaderAscii.cc | 12 ++++++ src/InputReaderAscii.h | 20 +++++++++ src/NetVar.cc | 2 + src/input.bif | 16 ++++++++ src/main.cc | 4 ++ src/types.bif | 7 ++++ 12 files changed, 223 insertions(+) create mode 100644 src/InputMgr.cc create mode 100644 src/InputMgr.h create mode 100644 src/InputReader.cc create mode 100644 src/InputReader.h create mode 100644 src/InputReaderAscii.cc create mode 100644 src/InputReaderAscii.h create mode 100644 src/input.bif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b4779e1557..1693bad4eb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -141,6 +141,7 @@ endmacro(GET_BIF_OUTPUT_FILES) set(BIF_SRCS bro.bif logging.bif + input.bif event.bif const.bif types.bif @@ -332,6 +333,9 @@ set(bro_SRCS IRC.cc List.cc Reporter.cc + InputMgr.cc + InputReader.cc + InputReaderAscii.cc LogMgr.cc LogWriter.cc LogWriterAscii.cc diff --git a/src/Func.cc b/src/Func.cc index 65cb22b09d..829bc89238 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -523,11 +523,13 @@ void builtin_error(const char* msg, BroObj* arg) #include "bro.bif.func_h" #include "logging.bif.func_h" +#include "input.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" #include "bro.bif.func_def" #include "logging.bif.func_def" +#include "input.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" @@ -542,6 +544,7 @@ void init_builtin_funcs() #include "bro.bif.func_init" #include "logging.bif.func_init" +#include "input.bif.func_init" #include "reporter.bif.func_init" #include "strings.bif.func_init" diff --git a/src/InputMgr.cc b/src/InputMgr.cc new file mode 100644 index 0000000000..cacc512bcf --- /dev/null +++ b/src/InputMgr.cc @@ -0,0 +1,90 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include "InputMgr.h" +#include "Event.h" +#include "EventHandler.h" +#include "NetVar.h" +#include "Net.h" + + +#include "InputReader.h" + +#include "InputReaderAscii.h" + + +struct InputReaderDefinition { + bro_int_t type; // the type + const char *name; // descriptive name for error messages + bool (*init)(); // optional one-time inifializing function + InputReader* (*factory)(); // factory function for creating instances +}; + +InputReaderDefinition input_readers[] = { + { BifEnum::Input::READER_ASCII, "Ascii", 0, InputReaderAscii::Instantiate }, + + // End marker + { BifEnum::Input::READER_DEFAULT, "None", 0, (InputReader* (*)())0 } +}; + +InputMgr::InputMgr() +{ + DBG_LOG(DBG_LOGGING, "this has to happen"); +} + + +// create a new input reader object to be used at whomevers leisure lateron. +InputReader* InputMgr::CreateReader(EnumVal* reader, string source) +{ + InputReaderDefinition* ir = input_readers; + exit(12); + + while ( true ) { + if ( ir->type == BifEnum::Input::READER_DEFAULT ) + { + DBG_LOG(DBG_LOGGING, "unknown reader when creating reader"); + reporter->Error("unknown reader when creating reader"); + return 0; + } + + if ( ir->type != reader->AsEnum() ) { + // no, didn't find the right one... + ++ir; + continue; + } + + + // call init function of writer if presnt + if ( ir->init ) + { + if ( (*ir->init)() ) + { + //clear it to be not called again + ir->init = 0; + } else { + // ohok. init failed, kill factory for all eternity + ir->factory = 0; + DBG_LOG(DBG_LOGGING, "failed to init input class %s", ir->name); + return 0; + } + + } + + if ( !ir->factory ) + // no factory? + return 0; + + // all done. break. + break; + } + + assert(ir->factory); + InputReader* reader_obj = (*ir->factory)(); + assert(reader_obj); + + return reader_obj; + +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h new file mode 100644 index 0000000000..978481afba --- /dev/null +++ b/src/InputMgr.h @@ -0,0 +1,24 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUTMGR_H +#define INPUTMGR_H + +#include "InputReader.h" +#include "BroString.h" + +#include "Val.h" +#include "EventHandler.h" +#include "RemoteSerializer.h" + + +class InputMgr { +public: + InputMgr(); + + InputReader* CreateReader(EnumVal* reader, string source); +}; + +extern InputMgr* input_mgr; + + +#endif /* INPUTMGR_H */ diff --git a/src/InputReader.cc b/src/InputReader.cc new file mode 100644 index 0000000000..f2b3b05801 --- /dev/null +++ b/src/InputReader.cc @@ -0,0 +1,12 @@ + +#include "InputReader.h" + +InputReader::InputReader() +{ + +} + +InputReader::~InputReader() +{ + +} \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h new file mode 100644 index 0000000000..ce8303383a --- /dev/null +++ b/src/InputReader.h @@ -0,0 +1,29 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Same notes about thread safety as in LogWriter.h apply. + + +#ifndef INPUTREADER_H +#define INPUTREADER_H + +class InputReader { +public: + InputReader(); + virtual ~InputReader(); + +protected: + // Methods that have to be overwritten by the individual readers + +private: + friend class InputMgr; + + // When an error occurs, this method is called to set a flag marking the + // writer as disabled. + + bool disabled; + + bool Disabled() { return disabled; } +}; + + +#endif /* INPUTREADER_H */ diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc new file mode 100644 index 0000000000..97933c2a6e --- /dev/null +++ b/src/InputReaderAscii.cc @@ -0,0 +1,12 @@ + +#include "InputReaderAscii.h" +#include "DebugLogger.h" + +InputReaderAscii::InputReaderAscii() +{ + DBG_LOG(DBG_LOGGING, "input reader initialized"); +} + +InputReaderAscii::~InputReaderAscii() +{ +} \ No newline at end of file diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h new file mode 100644 index 0000000000..c471639fd4 --- /dev/null +++ b/src/InputReaderAscii.h @@ -0,0 +1,20 @@ + +#ifndef INPUTREADERASCII_H +#define INPUTREADERASCII_H + +#include "InputReader.h" + +class InputReaderAscii : public InputReader { +public: + InputReaderAscii(); + ~InputReaderAscii(); + + static InputReader* Instantiate() { return new InputReaderAscii; } + +protected: + +private: +}; + + +#endif /* INPUTREADERASCII_H */ diff --git a/src/NetVar.cc b/src/NetVar.cc index 25e4f7a0bc..5a6ac96fba 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -255,6 +255,7 @@ StringVal* cmd_line_bpf_filter; #include "types.bif.netvar_def" #include "event.bif.netvar_def" #include "logging.bif.netvar_def" +#include "input.bif.netvar_def" #include "reporter.bif.netvar_def" void init_event_handlers() @@ -315,6 +316,7 @@ void init_net_var() #include "const.bif.netvar_init" #include "types.bif.netvar_init" #include "logging.bif.netvar_init" +#include "input.bif.netvar_init" #include "reporter.bif.netvar_init" conn_id = internal_type("conn_id")->AsRecordType(); diff --git a/src/input.bif b/src/input.bif new file mode 100644 index 0000000000..edb6b4e9bb --- /dev/null +++ b/src/input.bif @@ -0,0 +1,16 @@ +# functions and types for the input framework + +module Input; + +%%{ +#include "InputMgr.h" +#include "NetVar.h" +%%} + +function Input::__create_reader%(reader: Input::Reader, source: string%) : bool + %{ + exit(5); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); + return new Val( the_reader != 0, TYPE_BOOL ); + %} + diff --git a/src/main.cc b/src/main.cc index dfa46c3050..b3f2512b40 100644 --- a/src/main.cc +++ b/src/main.cc @@ -30,6 +30,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "File.h" #include "Reporter.h" #include "LogMgr.h" +#include "InputMgr.h" #include "Net.h" #include "NetVar.h" #include "Var.h" @@ -72,6 +73,7 @@ name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; LogMgr* log_mgr; +InputMgr* input_mgr; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -724,6 +726,8 @@ int main(int argc, char** argv) remote_serializer = new RemoteSerializer(); event_registry = new EventRegistry(); log_mgr = new LogMgr(); + + input_mgr = new InputMgr(); if ( events_file ) event_player = new EventPlayer(events_file); diff --git a/src/types.bif b/src/types.bif index da6bd6e031..ee43207ddd 100644 --- a/src/types.bif +++ b/src/types.bif @@ -167,4 +167,11 @@ enum ID %{ Unknown, %} +module Input; + +enum Reader %{ + READER_DEFAULT, + READER_ASCII, +%} + module GLOBAL; From 0eafeb03693e3134bb47219df43066a390a42d55 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 13:16:09 -0700 Subject: [PATCH 002/149] works (thanks to robin) --- scripts/base/init-bare.bro | 2 ++ src/NetVar.h | 1 + src/input.bif | 1 - 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 859a69f2dc..8d1d2a312f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1508,3 +1508,5 @@ const parse_udp_tunnels = F &redef; # Load the logging framework here because it uses fairly deep integration with # BiFs and script-land defined types. @load base/frameworks/logging + +@load base/input.bif diff --git a/src/NetVar.h b/src/NetVar.h index f8def230c0..957a86aeb3 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -264,6 +264,7 @@ extern void init_net_var(); #include "types.bif.netvar_h" #include "event.bif.netvar_h" #include "logging.bif.netvar_h" +#include "input.bif.netvar_h" #include "reporter.bif.netvar_h" #endif diff --git a/src/input.bif b/src/input.bif index edb6b4e9bb..3ff5284c63 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,7 +9,6 @@ module Input; function Input::__create_reader%(reader: Input::Reader, source: string%) : bool %{ - exit(5); InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); return new Val( the_reader != 0, TYPE_BOOL ); %} From f8be3519c7b868cd23829141b9588396df50e6ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 15:41:07 -0700 Subject: [PATCH 003/149] well, it compiles. and perhaps it sends an event. billiant. --- src/InputMgr.cc | 20 +++++++++++++++++--- src/InputMgr.h | 10 +++++++++- src/InputReader.cc | 19 +++++++++++++++++++ src/InputReader.h | 15 ++++++++++++++- src/input.bif | 6 ++++-- 5 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index cacc512bcf..5ab602f4f3 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,11 +35,17 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source) +InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription) { InputReaderDefinition* ir = input_readers; - exit(12); - + + RecordType* rtype = eventDescription->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) + { + reporter->Error("eventDescription argument not of right type"); + return 0; + } + while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { @@ -86,5 +92,13 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source) return reader_obj; } + +void InputMgr::Error(InputReader* reader, const char* msg) +{ + reporter->Error(fmt("error with input reader for %s: %s", + reader->Source().c_str(), msg)); +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h index 978481afba..3d3d0f2d3f 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -10,12 +10,20 @@ #include "EventHandler.h" #include "RemoteSerializer.h" +class InputReader; class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source); + InputReader* CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription); + +protected: + friend class InputReader; + + // Reports an error for the given reader. + void Error(InputReader* reader, const char* msg); + }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index f2b3b05801..ef47bb1e10 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,5 +1,7 @@ #include "InputReader.h" +#include "EventRegistry.h" +#include "Event.h" InputReader::InputReader() { @@ -9,4 +11,21 @@ InputReader::InputReader() InputReader::~InputReader() { +} + +void InputReader::Error(const char *msg) +{ + input_mgr->Error(this, msg); +} + +bool InputReader::Init(string source, string eventName) { + EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + if ( handler == 0 ) { + reporter->Error("Event %s not found", eventName.c_str()); + return false; + } + + mgr.Dispatch(new Event(handler, 0)); + return true; } \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h index ce8303383a..58a56e8221 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -6,16 +6,29 @@ #ifndef INPUTREADER_H #define INPUTREADER_H +#include "InputMgr.h" +#include "BroString.h" + class InputReader { public: InputReader(); virtual ~InputReader(); + + bool Init(string source, string eventName); protected: // Methods that have to be overwritten by the individual readers - + + // Reports an error to the user. + void Error(const char *msg); + + // The following methods return the information as passed to Init(). + const string Source() const { return source; } + private: friend class InputMgr; + + string source; // When an error occurs, this method is called to set a flag marking the // writer as disabled. diff --git a/src/input.bif b/src/input.bif index 3ff5284c63..e2fc2ae91e 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,9 +7,11 @@ module Input; #include "NetVar.h" %%} -function Input::__create_reader%(reader: Input::Reader, source: string%) : bool +type Event: record; + +function Input::__create_reader%(reader: Input::Reader, source: string, eventName: string, eventDescription: Input::Event%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventName->AsString()->CheckString(), eventDescription->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From 9c8b0dec3b8622a592aa1c787d9bde7d53f45f5f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 16:35:34 -0700 Subject: [PATCH 004/149] event from c++ to script works (at last...) --- scripts/base/frameworks/input/__load__.bro | 1 + scripts/base/frameworks/input/main.bro | 11 +++++++++++ scripts/base/init-bare.bro | 3 ++- src/InputMgr.cc | 7 ++++--- src/InputMgr.h | 2 +- src/InputReader.cc | 19 +++++++++++-------- src/LogMgr.cc | 2 +- src/input.bif | 4 ++-- 8 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 scripts/base/frameworks/input/__load__.bro create mode 100644 scripts/base/frameworks/input/main.bro diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/frameworks/input/__load__.bro @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro new file mode 100644 index 0000000000..cb071872ac --- /dev/null +++ b/scripts/base/frameworks/input/main.bro @@ -0,0 +1,11 @@ + +module Input; + +export { + type Event: record { + name: string; + columns: any; + }; +} + +@load base/input.bif diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 8d1d2a312f..ade92bfd6e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1509,4 +1509,5 @@ const parse_udp_tunnels = F &redef; # BiFs and script-land defined types. @load base/frameworks/logging -@load base/input.bif +@load base/frameworks/input + diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5ab602f4f3..f3fb2ff935 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,11 +35,11 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription) +InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* event) { InputReaderDefinition* ir = input_readers; - RecordType* rtype = eventDescription->Type()->AsRecordType(); + RecordType* rtype = InputReaderDefinition->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) { reporter->Error("eventDescription argument not of right type"); @@ -49,7 +49,6 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string event while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { - DBG_LOG(DBG_LOGGING, "unknown reader when creating reader"); reporter->Error("unknown reader when creating reader"); return 0; } @@ -89,6 +88,8 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string event InputReader* reader_obj = (*ir->factory)(); assert(reader_obj); + reader_obj->Init(source, eventName); + return reader_obj; } diff --git a/src/InputMgr.h b/src/InputMgr.h index 3d3d0f2d3f..e98f3d77b7 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -16,7 +16,7 @@ class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription); + InputReader* CreateReader(EnumVal* reader, string source, RecordVal* event); protected: friend class InputReader; diff --git a/src/InputReader.cc b/src/InputReader.cc index ef47bb1e10..fc9be7f2b6 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,7 +1,7 @@ #include "InputReader.h" -#include "EventRegistry.h" -#include "Event.h" +// #include "EventRegistry.h" +// #include "Event.h" InputReader::InputReader() { @@ -19,13 +19,16 @@ void InputReader::Error(const char *msg) } bool InputReader::Init(string source, string eventName) { - EventHandler* handler = event_registry->Lookup(eventName.c_str()); + //EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", eventName.c_str()); + // return false; + //} - if ( handler == 0 ) { - reporter->Error("Event %s not found", eventName.c_str()); - return false; - } + //val_list* vl = new val_list; + //vl->append(new Val(12, TYPE_COUNT)); - mgr.Dispatch(new Event(handler, 0)); + //mgr.Dispatch(new Event(handler, vl)); return true; } \ No newline at end of file diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 9e320f8810..fc4b89b0ed 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -475,7 +475,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) +bool LogMgr::(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); diff --git a/src/input.bif b/src/input.bif index e2fc2ae91e..88d4e32129 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,9 +9,9 @@ module Input; type Event: record; -function Input::__create_reader%(reader: Input::Reader, source: string, eventName: string, eventDescription: Input::Event%) : bool +function Input::__create_reader%(reader: Input::Reader, source: string, eventDescription: Input::Event%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventName->AsString()->CheckString(), eventDescription->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventDescription->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From 365406024627c8c2b0b3d41263840a7fbaf029ed Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 21 Oct 2011 14:01:18 -0700 Subject: [PATCH 005/149] compiles. sill doesn't do much. --- scripts/base/frameworks/input/main.bro | 8 ++++-- src/InputMgr.cc | 35 ++++++++++++++++++++---- src/InputMgr.h | 8 +++++- src/InputReader.cc | 29 ++++++++++---------- src/InputReader.h | 10 ++++++- src/InputReaderAscii.cc | 38 +++++++++++++++++++++++++- src/InputReaderAscii.h | 10 +++++++ src/LogMgr.cc | 2 +- src/input.bif | 6 ++-- 9 files changed, 117 insertions(+), 29 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index cb071872ac..b5a05af3b6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -2,9 +2,11 @@ module Input; export { - type Event: record { - name: string; - columns: any; + type ReaderDescription: record { + source: string; + idx: any; + val: any; + destination: any; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index f3fb2ff935..500a325249 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,14 +35,14 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* event) +InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) { InputReaderDefinition* ir = input_readers; - RecordType* rtype = InputReaderDefinition->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) + RecordType* rtype = description->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::ReaderDescription, 0) ) { - reporter->Error("eventDescription argument not of right type"); + reporter->Error("readerDescription argument not of right type"); return 0; } @@ -88,7 +88,11 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* e InputReader* reader_obj = (*ir->factory)(); assert(reader_obj); - reader_obj->Init(source, eventName); + // get the source... + const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); + string source((const char*) bsource->Bytes(), bsource->Len()); + + reader_obj->Init(source, 0, NULL); return reader_obj; @@ -100,6 +104,27 @@ void InputMgr::Error(InputReader* reader, const char* msg) reader->Source().c_str(), msg)); } +/* + TODO: + +void InputMgr::SendEvent(string name) { + //EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", eventName.c_str()); + // return false; + //} + + //val_list* vl = new val_list; + //vl->append(new Val(12, TYPE_COUNT)); + + //mgr.Dispatch(new Event(handler, vl)); + + +} + +*/ + diff --git a/src/InputMgr.h b/src/InputMgr.h index e98f3d77b7..255d61fe5e 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -9,20 +9,26 @@ #include "Val.h" #include "EventHandler.h" #include "RemoteSerializer.h" +#include "LogMgr.h" // for the LogVal and LogType data types class InputReader; + class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source, RecordVal* event); + InputReader* CreateReader(EnumVal* reader, RecordVal* description); protected: friend class InputReader; // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); + +private: + // required functionality + // InputValsToRecord to convert received inputvals back to bro records / tables / whatever }; diff --git a/src/InputReader.cc b/src/InputReader.cc index fc9be7f2b6..4d29040e60 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -5,7 +5,7 @@ InputReader::InputReader() { - + disabled = true; // disabled will be set correcty in init. } InputReader::~InputReader() @@ -18,17 +18,18 @@ void InputReader::Error(const char *msg) input_mgr->Error(this, msg); } -bool InputReader::Init(string source, string eventName) { - //EventHandler* handler = event_registry->Lookup(eventName.c_str()); - - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", eventName.c_str()); - // return false; - //} - - //val_list* vl = new val_list; - //vl->append(new Val(12, TYPE_COUNT)); - - //mgr.Dispatch(new Event(handler, vl)); - return true; +bool InputReader::Init(string arg_source, int arg_num_fields, + const LogField* const * arg_fields) +{ + source = arg_source; + num_fields = arg_num_fields; + fields = arg_fields; + + // disable if DoInit returns error. + disabled = !DoInit(arg_source, arg_num_fields, arg_fields); + return !disabled; +} + +void InputReader::Finish() { + DoFinish(); } \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h index 58a56e8221..f3638c7246 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -8,16 +8,22 @@ #include "InputMgr.h" #include "BroString.h" +#include "LogMgr.h" class InputReader { public: InputReader(); virtual ~InputReader(); - bool Init(string source, string eventName); + bool Init(string arg_source, int num_fields, const LogField* const* fields); + void Finish(); + protected: // Methods that have to be overwritten by the individual readers + virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; + + virtual void DoFinish() = 0; // Reports an error to the user. void Error(const char *msg); @@ -29,6 +35,8 @@ private: friend class InputMgr; string source; + int num_fields; + const LogField* const * fields; // When an error occurs, this method is called to set a flag marking the // writer as disabled. diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 97933c2a6e..b9aab16815 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -2,11 +2,47 @@ #include "InputReaderAscii.h" #include "DebugLogger.h" +#include + InputReaderAscii::InputReaderAscii() { - DBG_LOG(DBG_LOGGING, "input reader initialized"); + //DBG_LOG(DBG_LOGGING, "input reader initialized"); + file = 0; } InputReaderAscii::~InputReaderAscii() { +} + +void InputReaderAscii::DoFinish() +{ +} + +bool InputReaderAscii::DoInit(string path, int num_fields, + const LogField* const * fields) +{ + fname = path; + + file = new ifstream(path.c_str()); + if ( !file->is_open() ) { + return false; + } + + // try to read the header line... + string line; + if ( !getline(*file, line) ) + return false; + + // split on tabs... + istringstream ss(line); + while ( ss ) { + string s; + if ( !getline(ss, s, '\t')) + break; + + + } + + + return false; } \ No newline at end of file diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c471639fd4..0d2008ed7f 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -3,6 +3,9 @@ #define INPUTREADERASCII_H #include "InputReader.h" +#include +#include + class InputReaderAscii : public InputReader { public: @@ -12,8 +15,15 @@ public: static InputReader* Instantiate() { return new InputReaderAscii; } protected: + + virtual bool DoInit(string path, int num_fields, + const LogField* const * fields); + virtual void DoFinish(); private: + + ifstream* file; + string fname; }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index fc4b89b0ed..9e320f8810 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -475,7 +475,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::(EnumVal* id, RecordVal* sval) +bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); diff --git a/src/input.bif b/src/input.bif index 88d4e32129..3da869ea08 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,11 +7,11 @@ module Input; #include "NetVar.h" %%} -type Event: record; +type ReaderDescription: record; -function Input::__create_reader%(reader: Input::Reader, source: string, eventDescription: Input::Event%) : bool +function Input::__create_reader%(reader: Input::Reader, description: Input::ReaderDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventDescription->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From d7a3b85fcda42a1485ef7a53c5419100e4b5ea95 Mon Sep 17 00:00:00 2001 From: amannb Date: Tue, 25 Oct 2011 11:47:23 -0700 Subject: [PATCH 006/149] many helper functions --- src/InputMgr.cc | 71 +++++++++++++++------ src/InputMgr.h | 4 +- src/InputReader.cc | 33 +++++++++- src/InputReader.h | 12 ++++ src/InputReaderAscii.cc | 134 +++++++++++++++++++++++++++++++++++++--- src/InputReaderAscii.h | 22 +++++++ 6 files changed, 245 insertions(+), 31 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 500a325249..ec46c55813 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -100,31 +100,62 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) void InputMgr::Error(InputReader* reader, const char* msg) { - reporter->Error(fmt("error with input reader for %s: %s", - reader->Source().c_str(), msg)); + reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -/* - TODO: -void InputMgr::SendEvent(string name) { - //EventHandler* handler = event_registry->Lookup(eventName.c_str()); - - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", eventName.c_str()); - // return false; - //} - - //val_list* vl = new val_list; - //vl->append(new Val(12, TYPE_COUNT)); - - //mgr.Dispatch(new Event(handler, vl)); +void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +{ + EventHandler* handler = event_registry->Lookup(name.c_str()); + if ( handler == 0 ) { + reporter->Error("Event %s not found", name.c_str()); + return; + } + val_list* vl = new val_list; + for ( int i = 0; i < num_vals; i++) { + vl->append(LogValToVal(vals[i])); + } + + mgr.Dispatch(new Event(handler, vl)); +} + +Val* InputMgr::LogValToVal(const LogVal* val) { + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + return new Val(val->val.int_val, val->type); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + return new Val(val->val.uint_val, val->type); + break; + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return new Val(val->val.double_val, val->type); + break; + + case TYPE_STRING: + { + BroString *s = new BroString(*(val->val.string_val)); + return new StringVal(s); + break; + } + + case TYPE_PORT: + return new PortVal(val->val.uint_val); + break; + + default: + reporter->InternalError("unsupported type for input_read"); + } + + + reporter->InternalError("Impossible error"); + return NULL; } - -*/ - - diff --git a/src/InputMgr.h b/src/InputMgr.h index 255d61fe5e..79f76a1e6f 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -13,7 +13,6 @@ class InputReader; - class InputMgr { public: InputMgr(); @@ -29,6 +28,9 @@ protected: private: // required functionality // InputValsToRecord to convert received inputvals back to bro records / tables / whatever + Val* LogValToVal(const LogVal* val); + + void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); }; diff --git a/src/InputReader.cc b/src/InputReader.cc index 4d29040e60..d812349733 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -32,4 +32,35 @@ bool InputReader::Init(string arg_source, int arg_num_fields, void InputReader::Finish() { DoFinish(); -} \ No newline at end of file +} + +bool InputReader::Update() { + return DoUpdate(); +} + +// stolen from logwriter +const char* InputReader::Fmt(const char* format, ...) + { + if ( ! buf ) + buf = (char*) malloc(buf_len); + + va_list al; + va_start(al, format); + int n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + + if ( (unsigned int) n >= buf_len ) + { // Not enough room, grow the buffer. + buf_len = n + 32; + buf = (char*) realloc(buf, buf_len); + + // Is it portable to restart? + va_start(al, format); + n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + } + + return buf; + } + + diff --git a/src/InputReader.h b/src/InputReader.h index f3638c7246..9d776276fa 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -18,12 +18,17 @@ public: bool Init(string arg_source, int num_fields, const LogField* const* fields); void Finish(); + + bool Update(); protected: // Methods that have to be overwritten by the individual readers virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; virtual void DoFinish() = 0; + + // update file contents to logmgr + virtual bool DoUpdate() = 0; // Reports an error to the user. void Error(const char *msg); @@ -31,6 +36,9 @@ protected: // The following methods return the information as passed to Init(). const string Source() const { return source; } + // A thread-safe version of fmt(). (stolen from logwriter) + const char* Fmt(const char* format, ...); + private: friend class InputMgr; @@ -44,6 +52,10 @@ private: bool disabled; bool Disabled() { return disabled; } + + // For implementing Fmt(). + char* buf; + unsigned int buf_len; }; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index b9aab16815..4bc4b81cda 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -1,9 +1,22 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include "InputReaderAscii.h" #include "DebugLogger.h" #include +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) + : name(arg_name), type(arg_type) +{ + position = arg_position; +} + +FieldMapping::FieldMapping(const FieldMapping& arg) + : name(arg.name), type(arg.type) +{ + position = arg.position; +} + InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); @@ -18,31 +31,134 @@ void InputReaderAscii::DoFinish() { } -bool InputReaderAscii::DoInit(string path, int num_fields, - const LogField* const * fields) +bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const * fields) { fname = path; file = new ifstream(path.c_str()); if ( !file->is_open() ) { + Error(Fmt("cannot open %s", path.c_str())); return false; } // try to read the header line... string line; - if ( !getline(*file, line) ) + if ( !getline(*file, line) ) { + Error("could not read first line"); return false; + } // split on tabs... - istringstream ss(line); - while ( ss ) { + istringstream splitstream(line); + unsigned int currTab = 0; + int wantFields = 0; + while ( splitstream ) { string s; - if ( !getline(ss, s, '\t')) + if ( !getline(splitstream, s, '\t')) break; - + // current found heading in s... compare if we want it + for ( int i = 0; i < num_fields; i++ ) { + const LogField* field = fields[i]; + if ( field->name == s ) { + // cool, found field. note position + FieldMapping f(field->name, field->type, i); + columnMap.push_back(f); + wantFields++; + break; // done with searching + } + } + + // look if we did push something... + if ( columnMap.size() == currTab ) { + // no, we didn't. note that... + FieldMapping empty; + columnMap.push_back(empty); + } + + // done + currTab++; } + + if ( wantFields != num_fields ) { + // we did not find all fields? + // :( + Error("wantFields != num_fields"); + return false; + } + + this->num_fields = num_fields; - return false; -} \ No newline at end of file + // well, that seems to have worked... + return true; +} + +// read the entire file and send appropriate thingies back to InputMgr +bool InputReaderAscii::DoUpdate() { + // TODO: all the stuff we need for a second reading. + // *cough* + // + + + string line; + while ( getline(*file, line ) ) { + // split on tabs + + istringstream splitstream(line); + string s; + + LogVal fields[num_fields]; + + unsigned int currTab = 0; + unsigned int currField = 0; + while ( splitstream ) { + if ( !getline(splitstream, s, '\t') ) + break; + + + if ( currTab >= columnMap.size() ) { + Error("Tabs in heading do not match tabs in data?"); + //disabled = true; + return false; + } + + FieldMapping currMapping = columnMap[currTab]; + currTab++; + + if ( currMapping.IsEmpty() ) { + // well, that was easy + continue; + } + + if ( currField >= num_fields ) { + Error("internal error - fieldnum greater as possible"); + return false; + } + + LogVal val(currMapping.type, true); + + switch ( currMapping.type ) { + case TYPE_STRING: + val.val.string_val = new string(s); + + default: + Error(Fmt("unsupported field format %d for %s", currMapping.type, + currMapping.name.c_str())); + return false; + } + + currField++; + } + + if ( currField != num_fields ) { + Error("curr_field != num_fields in DoUpdate"); + return false; + } + + // ok, now we have built our line. send it back to... whomever. + + } + + return true; +} diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 0d2008ed7f..551a08b02e 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -1,3 +1,4 @@ +// See the file "COPYING" in the main distribution directory for copyright. #ifndef INPUTREADERASCII_H #define INPUTREADERASCII_H @@ -5,6 +6,19 @@ #include "InputReader.h" #include #include +#include + +// Description for input field mapping +struct FieldMapping { + string name; + TypeTag type; + int position; + + FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const FieldMapping& arg); + FieldMapping() { position = -1; } + bool IsEmpty() { return position == -1; } +}; class InputReaderAscii : public InputReader { @@ -19,11 +33,19 @@ protected: virtual bool DoInit(string path, int num_fields, const LogField* const * fields); virtual void DoFinish(); + + virtual bool DoUpdate(); private: ifstream* file; string fname; + + unsigned int num_fields; + + // map columns in the file to columns to send back to the manager + vector columnMap; + }; From 5b0c307f87a7213951e586f6e881501dc658f03d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 25 Oct 2011 14:11:21 -0700 Subject: [PATCH 007/149] very basic input to event working... --- scripts/base/frameworks/input/main.bro | 4 ++-- src/InputMgr.cc | 18 ++++++++++++++++-- src/InputReader.cc | 11 +++++++++++ src/InputReader.h | 3 +++ src/InputReaderAscii.cc | 14 ++++++++++---- 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index b5a05af3b6..4bea9d73d2 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,8 +5,8 @@ export { type ReaderDescription: record { source: string; idx: any; - val: any; - destination: any; + val: any &optional; + destination: any &optional; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index ec46c55813..5cd82b613f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -91,8 +91,22 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) // get the source... const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - - reader_obj->Init(source, 0, NULL); + + RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + LogField** fields = new LogField*[idx->NumFields()]; + for ( int i = 0; i < idx->NumFields(); i++ ) + { + // FIXME: do type checking... + LogField* field = new LogField(); + field->name = idx->FieldName(i); + field->type = idx->FieldType(i)->Tag(); + fields[i] = field; + } + + + reader_obj->Init(source, idx->NumFields(), fields); + reader_obj->Update(); return reader_obj; diff --git a/src/InputReader.cc b/src/InputReader.cc index d812349733..d512bc1699 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -5,6 +5,8 @@ InputReader::InputReader() { + buf = 0; + buf_len = 1024; disabled = true; // disabled will be set correcty in init. } @@ -18,6 +20,11 @@ void InputReader::Error(const char *msg) input_mgr->Error(this, msg); } +void InputReader::Error(const string &msg) + { + input_mgr->Error(this, msg.c_str()); + } + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { @@ -38,6 +45,10 @@ bool InputReader::Update() { return DoUpdate(); } +void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { + input_mgr->SendEvent(name, num_vals, vals); +} + // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) { diff --git a/src/InputReader.h b/src/InputReader.h index 9d776276fa..3bfb0adf91 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -31,6 +31,7 @@ protected: virtual bool DoUpdate() = 0; // Reports an error to the user. + void Error(const string &msg); void Error(const char *msg); // The following methods return the information as passed to Init(). @@ -39,6 +40,8 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); + void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4bc4b81cda..5d6b23416a 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -37,7 +37,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const file = new ifstream(path.c_str()); if ( !file->is_open() ) { - Error(Fmt("cannot open %s", path.c_str())); + Error(Fmt("cannot open %s", fname.c_str())); return false; } @@ -108,7 +108,7 @@ bool InputReaderAscii::DoUpdate() { istringstream splitstream(line); string s; - LogVal fields[num_fields]; + LogVal** fields = new LogVal*[num_fields]; unsigned int currTab = 0; unsigned int currField = 0; @@ -136,11 +136,12 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal val(currMapping.type, true); + LogVal* val = new LogVal(currMapping.type, true); switch ( currMapping.type ) { case TYPE_STRING: - val.val.string_val = new string(s); + val->val.string_val = new string(s); + break; default: Error(Fmt("unsupported field format %d for %s", currMapping.type, @@ -148,6 +149,8 @@ bool InputReaderAscii::DoUpdate() { return false; } + fields[currField] = val; + currField++; } @@ -157,6 +160,9 @@ bool InputReaderAscii::DoUpdate() { } // ok, now we have built our line. send it back to... whomever. + // for testing purposes: fixed event. + + SendEvent("inputEvent", num_fields, fields); } From b245d4168a6c9d9ce9243a34f7fcc704d526a7d9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 26 Oct 2011 17:02:57 -0700 Subject: [PATCH 008/149] yay, basic table assignment. --- scripts/base/frameworks/input/main.bro | 4 +- src/InputMgr.cc | 60 ++++++++++++++++++++++++-- src/InputMgr.h | 8 +++- src/InputReader.cc | 5 +++ src/InputReader.h | 2 + src/InputReaderAscii.cc | 16 +++++++ 6 files changed, 87 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4bea9d73d2..b5a05af3b6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,8 +5,8 @@ export { type ReaderDescription: record { source: string; idx: any; - val: any &optional; - destination: any &optional; + val: any; + destination: any; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5cd82b613f..5c30922863 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -13,6 +13,16 @@ #include "InputReaderAscii.h" +struct InputMgr::ReaderInfo { + EnumVal* id; + EnumVal* type; + InputReader* reader; + unsigned int num_idx_fields; + unsigned int num_val_fields; + + TableVal* tab; + + }; struct InputReaderDefinition { bro_int_t type; // the type @@ -30,7 +40,7 @@ InputReaderDefinition input_readers[] = { InputMgr::InputMgr() { - DBG_LOG(DBG_LOGGING, "this has to happen"); + //DBG_LOG(DBG_LOGGING, "this has to happen"); } @@ -93,8 +103,10 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) string source((const char*) bsource->Bytes(), bsource->Len()); RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - LogField** fields = new LogField*[idx->NumFields()]; + LogField** fields = new LogField*[idx->NumFields() + val->NumFields()]; for ( int i = 0; i < idx->NumFields(); i++ ) { // FIXME: do type checking... @@ -103,15 +115,43 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) field->type = idx->FieldType(i)->Tag(); fields[i] = field; } + for ( int i = 0; i < val->NumFields(); i++ ) + { + // FIXME: do type checking... + LogField* field = new LogField(); + field->name = val->FieldName(i); + field->type = val->FieldType(i)->Tag(); + fields[idx->NumFields() + i] = field; + } + + ReaderInfo* info = new ReaderInfo; + info->reader = reader_obj; + info->type = reader; + info->num_idx_fields = idx->NumFields(); + info->num_val_fields = val->NumFields(); + info->tab = dst; + readers.push_back(info); - reader_obj->Init(source, idx->NumFields(), fields); + reader_obj->Init(source, idx->NumFields() + val->NumFields(), fields); reader_obj->Update(); return reader_obj; } +void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + i->tab->Assign(LogValToVal(vals[0]), LogValToVal(vals[1])); + reporter->Error("assigned"); +} + + void InputMgr::Error(InputReader* reader, const char* msg) { reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); @@ -172,4 +212,16 @@ Val* InputMgr::LogValToVal(const LogVal* val) { return NULL; } - +InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) + { + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->reader == reader ) + { + return *s; + } + } + + return 0; + } + diff --git a/src/InputMgr.h b/src/InputMgr.h index 79f76a1e6f..136be2d608 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -24,14 +24,18 @@ protected: // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); + + void Put(const InputReader* reader, const LogVal* const *vals); private: - // required functionality - // InputValsToRecord to convert received inputvals back to bro records / tables / whatever + struct ReaderInfo; + Val* LogValToVal(const LogVal* val); void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + ReaderInfo* FindReader(const InputReader* reader); + vector readers; }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index d512bc1699..6502fdb421 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -25,6 +25,11 @@ void InputReader::Error(const string &msg) input_mgr->Error(this, msg.c_str()); } +void InputReader::Put(const LogVal* const *val) +{ + input_mgr->Put(this, val); +} + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { diff --git a/src/InputReader.h b/src/InputReader.h index 3bfb0adf91..3725c3d461 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -42,6 +42,8 @@ protected: void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + void Put(const LogVal* const *val); + private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 5d6b23416a..d0d4a3014c 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -143,6 +143,21 @@ bool InputReaderAscii::DoUpdate() { val->val.string_val = new string(s); break; + case TYPE_BOOL: + case TYPE_INT: + val->val.int_val = atoi(s.c_str()); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = atof(s.c_str()); + break; + + case TYPE_COUNT: + val->val.uint_val = atoi(s.c_str()); + break; + default: Error(Fmt("unsupported field format %d for %s", currMapping.type, currMapping.name.c_str())); @@ -163,6 +178,7 @@ bool InputReaderAscii::DoUpdate() { // for testing purposes: fixed event. SendEvent("inputEvent", num_fields, fields); + Put(fields); } From 86730c13dda7df06fa96b151c17ec437aa742b4f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 26 Oct 2011 17:46:43 -0700 Subject: [PATCH 009/149] more complex types... --- scripts/base/frameworks/input/main.bro | 3 + src/InputMgr.cc | 314 +++++++++++++++++++++++-- src/InputMgr.h | 17 +- src/InputReader.cc | 11 + src/InputReader.h | 2 + src/InputReaderAscii.cc | 35 ++- src/input.bif | 17 +- src/types.bif | 4 + 8 files changed, 372 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index b5a05af3b6..4bb7129d03 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -2,11 +2,14 @@ module Input; export { + const default_reader = READER_ASCII &redef; + type ReaderDescription: record { source: string; idx: any; val: any; destination: any; + reader: Reader &default=default_reader; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5c30922863..648a933a22 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -21,6 +21,8 @@ struct InputMgr::ReaderInfo { unsigned int num_val_fields; TableVal* tab; + RecordType* rtype; + RecordType* itype; }; @@ -45,7 +47,7 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) +InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) { InputReaderDefinition* ir = input_readers; @@ -55,6 +57,8 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) reporter->Error("readerDescription argument not of right type"); return 0; } + + EnumVal* reader = description->Lookup(rtype->FieldOffset("reader"))->AsEnumVal(); while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) @@ -106,38 +110,191 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - LogField** fields = new LogField*[idx->NumFields() + val->NumFields()]; - for ( int i = 0; i < idx->NumFields(); i++ ) - { - // FIXME: do type checking... - LogField* field = new LogField(); - field->name = idx->FieldName(i); - field->type = idx->FieldType(i)->Tag(); - fields[i] = field; + + vector fieldsV; // vector, because we don't know the length beforehands + + + bool status = !UnrollRecordType(&fieldsV, idx, ""); + + int idxfields = fieldsV.size(); + + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + + if ( status ) { + reporter->Error("Problem unrolling"); + return 0; } - for ( int i = 0; i < val->NumFields(); i++ ) - { - // FIXME: do type checking... - LogField* field = new LogField(); - field->name = val->FieldName(i); - field->type = val->FieldType(i)->Tag(); - fields[idx->NumFields() + i] = field; + + + LogField** fields = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + fields[i] = fieldsV[i]; } ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader; - info->num_idx_fields = idx->NumFields(); - info->num_val_fields = val->NumFields(); + Ref(reader); + info->num_idx_fields = idxfields; + info->num_val_fields = valfields; info->tab = dst; + Ref(dst); + info->rtype = val; + Ref(val); // we save a pointer of it... I really hope that this wasn't already done anywhere. + info->id = id; + Ref(id); // ditto... + info->itype = idx; + Ref(idx); readers.push_back(info); - reader_obj->Init(source, idx->NumFields() + val->NumFields(), fields); + reader_obj->Init(source, fieldsV.size(), fields); reader_obj->Update(); return reader_obj; +} +bool InputMgr::IsCompatibleType(BroType* t) + { + if ( ! t ) + return false; + + switch ( t->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_SUBNET: + case TYPE_ADDR: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_RECORD: + // for record: check, if all elements are compatible? But... LogMgr also doesn't do this. + // ^ recursive checking is done in UnrollRecordType. + return true; + + case TYPE_FILE: + case TYPE_FUNC: + return false; + + + case TYPE_TABLE: + return false; + + case TYPE_VECTOR: + { + return IsCompatibleType(t->AsVectorType()->YieldType()); + } + + default: + return false; + } + + return false; + } + + +bool InputMgr::RemoveReader(EnumVal* id) { + ReaderInfo *i = 0; + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->id == id ) + { + i = (*s); + readers.erase(s); // remove from vector + break; + } + } + + if ( i == 0 ) { + return false; // not found + } + + Unref(i->type); + Unref(i->tab); + Unref(i->itype); + Unref(i->rtype); + Unref(i->id); + + delete(i->reader); + delete(i); + + return true; +} + +bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { + for ( int i = 0; i < rec->NumFields(); i++ ) + { + + if ( !IsCompatibleType(rec->FieldType(i)) ) { + reporter->Error("Incompatible type \"%s\" in table definition for InputReader", type_name(rec->FieldType(i)->Tag())); + return false; + } + + if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) + { + + string prep = nameprepend + rec->FieldName(i) + "."; + + if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) + { + return false; + } + + } else { + LogField* field = new LogField(); + field->name = nameprepend + rec->FieldName(i); + field->type = rec->FieldType(i)->Tag(); + + fields->push_back(field); + } + } + + return true; + +} + +bool InputMgr::ForceUpdate(EnumVal* id) +{ + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Reader not found"); + return false; + } + + i->reader->Update(); + return true; +} + +Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { + Val* idxval; + int position = 0; + + + if ( num_fields == 1 ) { + idxval = LogValToVal(vals[0]); + } else { + ListVal *l = new ListVal(TYPE_ANY); + for ( int j = 0 ; j < type->NumFields(); j++ ) { + if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { + l->Append(LogValToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); + } else { + l->Append(LogValToVal(vals[position], type->FieldType(j)->Tag())); + position++; + } + } + idxval = l; + } + + assert ( position == num_fields ); + + return idxval; + } void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { @@ -147,10 +304,65 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { return; } - i->tab->Assign(LogValToVal(vals[0]), LogValToVal(vals[1])); - reporter->Error("assigned"); + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* valval; + + int position = i->num_idx_fields; + if ( i->num_val_fields == 1 ) { + valval = LogValToVal(vals[i->num_idx_fields]); + } else { + RecordVal * r = new RecordVal(i->rtype); + + /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { + reporter->InternalError("Type mismatch"); + return; + } */ + + for ( int j = 0; j < i->rtype->NumFields(); j++) { + + Val* val = 0; + if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + position++; + } + + if ( val == 0 ) { + reporter->InternalError("conversion error"); + return; + } + + r->Assign(j,val); + + } + valval = r; + } + + i->tab->Assign(idxval, valval); } +void InputMgr::Clear(const InputReader* reader) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + i->tab->RemoveAll(); +} + +bool InputMgr::Delete(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return false; + } + + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + + return ( i->tab->Delete(idxval) != 0 ); +} void InputMgr::Error(InputReader* reader, const char* msg) { @@ -174,7 +386,46 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } -Val* InputMgr::LogValToVal(const LogVal* val) { + +Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { + if ( position == 0 ) { + reporter->InternalError("Need position"); + return 0; + } + + /* + if ( request_type->Tag() != TYPE_RECORD ) { + reporter->InternalError("I only work with records"); + return 0; + } */ + + + RecordVal* rec = new RecordVal(request_type->AsRecordType()); + for ( int i = 0; i < request_type->NumFields(); i++ ) { + + Val* fieldVal = 0; + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + fieldVal = LogValToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); + } else { + fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)->Tag()); + (*position)++; + } + + rec->Assign(i, fieldVal); + } + + return rec; + +} + +Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { + + if ( request_type != TYPE_ANY && request_type != val->type ) { + reporter->InternalError("Typetags don't match: %d vs %d", request_type, val->type); + return 0; + } + + switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: @@ -203,6 +454,10 @@ Val* InputMgr::LogValToVal(const LogVal* val) { return new PortVal(val->val.uint_val); break; + case TYPE_ADDR: + return new AddrVal(val->val.addr_val); + break; + default: reporter->InternalError("unsupported type for input_read"); } @@ -225,3 +480,18 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) return 0; } + +InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) + { + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->id == id ) + { + return *s; + } + } + + return 0; + } + + diff --git a/src/InputMgr.h b/src/InputMgr.h index 136be2d608..d5f732935c 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -11,13 +11,17 @@ #include "RemoteSerializer.h" #include "LogMgr.h" // for the LogVal and LogType data types +#include + class InputReader; class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, RecordVal* description); + InputReader* CreateReader(EnumVal* id, RecordVal* description); + bool ForceUpdate(EnumVal* id); + bool RemoveReader(EnumVal* id); protected: friend class InputReader; @@ -26,14 +30,23 @@ protected: void Error(InputReader* reader, const char* msg); void Put(const InputReader* reader, const LogVal* const *vals); + void Clear(const InputReader* reader); + bool Delete(const InputReader* reader, const LogVal* const *vals); private: struct ReaderInfo; - Val* LogValToVal(const LogVal* val); + bool IsCompatibleType(BroType* t); + + bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + + Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); + Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); + Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); + ReaderInfo* FindReader(const EnumVal* id); vector readers; }; diff --git a/src/InputReader.cc b/src/InputReader.cc index 6502fdb421..1facc57c7f 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -30,6 +30,17 @@ void InputReader::Put(const LogVal* const *val) input_mgr->Put(this, val); } +void InputReader::Clear() +{ + input_mgr->Clear(this); +} + +void InputReader::Delete(const LogVal* const *val) +{ + input_mgr->Delete(this, val); +} + + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { diff --git a/src/InputReader.h b/src/InputReader.h index 3725c3d461..0e93344e1a 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -43,6 +43,8 @@ protected: void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); void Put(const LogVal* const *val); + void Clear(); + void Delete(const LogVal* const *val); private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index d0d4a3014c..e4d581c0d8 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -144,6 +144,13 @@ bool InputReaderAscii::DoUpdate() { break; case TYPE_BOOL: + if ( s == "T" ) { + val->val.int_val = 1; + } else { + val->val.int_val = 0; + } + break; + case TYPE_INT: val->val.int_val = atoi(s.c_str()); break; @@ -155,16 +162,37 @@ bool InputReaderAscii::DoUpdate() { break; case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: val->val.uint_val = atoi(s.c_str()); break; + case TYPE_SUBNET: { + int pos = s.find("/"); + string width = s.substr(pos); + val->val.subnet_val.width = atoi(width.c_str()); + string addr = s.substr(0, pos); + s = addr; + // fallthrough + } + case TYPE_ADDR: { + addr_type t = dotted_to_addr(s.c_str()); +#ifdef BROv6 + copy_addr(t, val->val.addr_val); +#else + copy_addr(&t, val->val.addr_val); +#endif + break; + } + + default: Error(Fmt("unsupported field format %d for %s", currMapping.type, currMapping.name.c_str())); return false; } - fields[currField] = val; + fields[currMapping.position] = val; currField++; } @@ -174,10 +202,7 @@ bool InputReaderAscii::DoUpdate() { return false; } - // ok, now we have built our line. send it back to... whomever. - // for testing purposes: fixed event. - - SendEvent("inputEvent", num_fields, fields); + // ok, now we have built our line. send it back to the input manager Put(fields); } diff --git a/src/input.bif b/src/input.bif index 3da869ea08..4e2cbf07b5 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,9 +9,22 @@ module Input; type ReaderDescription: record; -function Input::__create_reader%(reader: Input::Reader, description: Input::ReaderDescription%) : bool +function Input::__create_reader%(id: ID, description: Input::ReaderDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), description->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} + +function Input::__force_update%(id: ID%) : bool + %{ + bool res = input_mgr->ForceUpdate(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_reader%(id: ID%) : bool + %{ + bool res = input_mgr->RemoveReader(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + diff --git a/src/types.bif b/src/types.bif index ee43207ddd..7b81a7f631 100644 --- a/src/types.bif +++ b/src/types.bif @@ -174,4 +174,8 @@ enum Reader %{ READER_ASCII, %} +enum ID %{ + Unknown, +%} + module GLOBAL; From f20125d22de20a7147026602f83fa1ed710700ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 28 Oct 2011 13:11:47 -0700 Subject: [PATCH 010/149] little snag with hashing functionality... --- src/InputMgr.cc | 232 +++++++++++++++++++++++++++++++++++++++- src/InputMgr.h | 11 +- src/InputReader.cc | 13 ++- src/InputReader.h | 9 +- src/InputReaderAscii.cc | 80 ++++++++++++-- src/InputReaderAscii.h | 5 +- src/Val.h | 5 +- 7 files changed, 333 insertions(+), 22 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 648a933a22..5270c83d5a 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -13,6 +13,18 @@ #include "InputReaderAscii.h" +#include "CompHash.h" + + +class InputHash { +public: + HashKey* valhash; + HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. +}; + +declare(PDict, InputHash); + + struct InputMgr::ReaderInfo { EnumVal* id; EnumVal* type; @@ -24,6 +36,9 @@ struct InputMgr::ReaderInfo { RecordType* rtype; RecordType* itype; + PDict(InputHash)* currDict; + PDict(InputHash)* lastDict; + }; struct InputReaderDefinition { @@ -147,10 +162,20 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->itype = idx; Ref(idx); readers.push_back(info); + info->currDict = new PDict(InputHash); + info->lastDict = new PDict(InputHash); - reader_obj->Init(source, fieldsV.size(), fields); - reader_obj->Update(); + int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); + if ( success == false ) { + RemoveReader(id); + return 0; + } + success = reader_obj->Update(); + if ( success == false ) { + RemoveReader(id); + return 0; + } return reader_obj; @@ -198,7 +223,6 @@ bool InputMgr::IsCompatibleType(BroType* t) return false; } - bool InputMgr::RemoveReader(EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) @@ -215,6 +239,9 @@ bool InputMgr::RemoveReader(EnumVal* id) { return false; // not found } + i->reader->Finish(); + + Unref(i->type); Unref(i->tab); Unref(i->itype); @@ -267,8 +294,7 @@ bool InputMgr::ForceUpdate(EnumVal* id) return false; } - i->reader->Update(); - return true; + return i->reader->Update(); } Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { @@ -297,6 +323,105 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo } + +void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + + InputHash *h = i->lastDict->Lookup(idxhash); + if ( h != 0 ) { + // seen before + if ( h->valhash->Hash() == valhash->Hash() ) { + // ok, double. + i->lastDict->Remove(idxhash); + i->currDict->Insert(idxhash, h); + return; + } else { + // updated + i->lastDict->Remove(idxhash); + delete(h); + } + } + + + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* valval; + + int position = i->num_idx_fields; + if ( i->num_val_fields == 1 ) { + valval = LogValToVal(vals[i->num_idx_fields]); + } else { + RecordVal * r = new RecordVal(i->rtype); + + /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { + reporter->InternalError("Type mismatch"); + return; + } */ + + for ( int j = 0; j < i->rtype->NumFields(); j++) { + + Val* val = 0; + if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + position++; + } + + if ( val == 0 ) { + reporter->InternalError("conversion error"); + return; + } + + r->Assign(j,val); + + } + valval = r; + } + + //i->tab->Assign(idxval, valval); + HashKey* k = i->tab->ComputeHash(idxval); + if ( !k ) { + reporter->InternalError("could not hash"); + return; + } + + i->tab->Assign(idxval, k, valval); + InputHash* ih = new InputHash(); + ih->idxkey = k; + ih->valhash = valhash; + + i->currDict->Insert(idxhash, ih); + +} + +void InputMgr::EndCurrentSend(const InputReader* reader) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + // lastdict contains all deleted entries + IterCookie *c = i->lastDict->InitForIteration(); + InputHash* ih; + while ( ( ih = i->lastDict->NextEntry(c )) ) { + i->tab->Delete(ih->idxkey); + } + + i->lastDict->Clear(); + delete(i->lastDict); + + i->lastDict = i->currDict; + i->currDict = new PDict(InputHash); +} + void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -418,6 +543,95 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ } +HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals) { + int length = 0; + + for ( int i = 0; i < num_elements; i++ ) { + const LogVal* val = vals[i]; + switch (val->type) { + case TYPE_BOOL: + case TYPE_INT: + length += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + length += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + length += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + { + length += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + length += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + default: + reporter->InternalError("unsupported type for hashlogvals"); + } + + } + + int position = 0; + char *data = (char*) malloc(length); + for ( int i = 0; i < num_elements; i++ ) { + const LogVal* val = vals[i]; + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + *(data+position) = val->val.int_val; + position += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + *(data+position) = val->val.uint_val; + position += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + *(data+position) = val->val.double_val; + position += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + { + memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); + position += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + memcpy(data+position, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); + position += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + default: + reporter->InternalError("unsupported type for hashlogvals2"); + } + + + } + + assert(position == length); + return new HashKey(data, length); + + +} + Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { if ( request_type != TYPE_ANY && request_type != val->type ) { @@ -495,3 +709,11 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) } +string InputMgr::Hash(const string &input) { + unsigned char digest[16]; + hash_md5(input.length(), (const unsigned char*) input.c_str(), digest); + string out((const char*) digest, 16); + return out; +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h index d5f732935c..d147fa262a 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -15,9 +15,10 @@ class InputReader; + class InputMgr { public: - InputMgr(); + InputMgr(); InputReader* CreateReader(EnumVal* id, RecordVal* description); bool ForceUpdate(EnumVal* id); @@ -32,6 +33,9 @@ protected: void Put(const InputReader* reader, const LogVal* const *vals); void Clear(const InputReader* reader); bool Delete(const InputReader* reader, const LogVal* const *vals); + + void SendEntry(const InputReader* reader, const LogVal* const *vals); + void EndCurrentSend(const InputReader* reader); private: struct ReaderInfo; @@ -40,6 +44,8 @@ private: bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); + Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); @@ -49,6 +55,9 @@ private: ReaderInfo* FindReader(const EnumVal* id); vector readers; + + string Hash(const string &input); + }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index 1facc57c7f..494df3fb81 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -41,20 +41,22 @@ void InputReader::Delete(const LogVal* const *val) } -bool InputReader::Init(string arg_source, int arg_num_fields, +bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * arg_fields) { source = arg_source; num_fields = arg_num_fields; + index_fields = arg_idx_fields; fields = arg_fields; // disable if DoInit returns error. - disabled = !DoInit(arg_source, arg_num_fields, arg_fields); + disabled = !DoInit(arg_source, arg_num_fields, arg_idx_fields, arg_fields); return !disabled; } void InputReader::Finish() { DoFinish(); + disabled = true; } bool InputReader::Update() { @@ -91,3 +93,10 @@ const char* InputReader::Fmt(const char* format, ...) } +void InputReader::SendEntry(const LogVal* const *vals) { + input_mgr->SendEntry(this, vals); +} + +void InputReader::EndCurrentSend() { + input_mgr->EndCurrentSend(this); +} diff --git a/src/InputReader.h b/src/InputReader.h index 0e93344e1a..b547d29506 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -15,7 +15,7 @@ public: InputReader(); virtual ~InputReader(); - bool Init(string arg_source, int num_fields, const LogField* const* fields); + bool Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const* fields); void Finish(); @@ -23,7 +23,7 @@ public: protected: // Methods that have to be overwritten by the individual readers - virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; + virtual bool DoInit(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * fields) = 0; virtual void DoFinish() = 0; @@ -46,11 +46,16 @@ protected: void Clear(); void Delete(const LogVal* const *val); + void SendEntry(const LogVal* const *vals); + void EndCurrentSend(); + + private: friend class InputMgr; string source; int num_fields; + int index_fields; const LogField* const * fields; // When an error occurs, this method is called to set a flag marking the diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e4d581c0d8..e434f7e750 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -21,17 +21,26 @@ InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); file = 0; + + //keyMap = new map(); } InputReaderAscii::~InputReaderAscii() { + DoFinish(); } void InputReaderAscii::DoFinish() { + columnMap.empty(); + if ( file != 0 ) { + file->close(); + delete(file); + file = 0; + } } -bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const * fields) +bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const LogField* const * fields) { fname = path; @@ -47,6 +56,9 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const Error("could not read first line"); return false; } + + this->num_fields = num_fields; + this->idx_fields = idx_fields; // split on tabs... istringstream splitstream(line); @@ -83,12 +95,10 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const if ( wantFields != num_fields ) { // we did not find all fields? // :( - Error("wantFields != num_fields"); + Error("One of the requested fields could not be found in the input data file"); return false; } - - this->num_fields = num_fields; // well, that seems to have worked... return true; @@ -101,6 +111,9 @@ bool InputReaderAscii::DoUpdate() { // + // new keymap + //map *newKeyMap = new map(); + string line; while ( getline(*file, line ) ) { // split on tabs @@ -109,10 +122,12 @@ bool InputReaderAscii::DoUpdate() { string s; LogVal** fields = new LogVal*[num_fields]; + //string string_fields[num_fields]; unsigned int currTab = 0; unsigned int currField = 0; while ( splitstream ) { + if ( !getline(splitstream, s, '\t') ) break; @@ -146,8 +161,11 @@ bool InputReaderAscii::DoUpdate() { case TYPE_BOOL: if ( s == "T" ) { val->val.int_val = 1; - } else { + } else if ( s == "F" ) { val->val.int_val = 0; + } else { + Error(Fmt("Invalid value for boolean: %s", s.c_str())); + return false; } break; @@ -173,9 +191,15 @@ bool InputReaderAscii::DoUpdate() { val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; - // fallthrough + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + val->val.subnet_val.net = dotted_to_addr(s.c_str()); + break; + } case TYPE_ADDR: { + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... addr_type t = dotted_to_addr(s.c_str()); #ifdef BROv6 copy_addr(t, val->val.addr_val); @@ -193,19 +217,57 @@ bool InputReaderAscii::DoUpdate() { } fields[currMapping.position] = val; + //string_fields[currMapping.position] = s; currField++; } if ( currField != num_fields ) { - Error("curr_field != num_fields in DoUpdate"); + Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); return false; } - // ok, now we have built our line. send it back to the input manager - Put(fields); + + SendEntry(fields); + + /* + string indexstring = ""; + string valstring = ""; + for ( unsigned int i = 0; i < idx_fields; i++ ) { + indexstring.append(string_fields[i]); + } + + for ( unsigned int i = idx_fields; i < num_fields; i++ ) { + valstring.append(string_fields[i]); + } + + string valhash = Hash(valstring); + string indexhash = Hash(indexstring); + + if ( keyMap->find(indexhash) == keyMap->end() ) { + // new key + Put(fields); + } else if ( (*keyMap)[indexhash] != valhash ) { + // changed key + Put(fields); + keyMap->erase(indexhash); + } else { + // field not changed + keyMap->erase(indexhash); + } + + + (*newKeyMap)[indexhash] = valhash; + */ + + for ( unsigned int i = 0; i < num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; } + + EndCurrentSend(); return true; } diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 551a08b02e..c26a139dcd 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -30,7 +30,7 @@ public: protected: - virtual bool DoInit(string path, int num_fields, + virtual bool DoInit(string path, int arg_num_fields, int arg_idx_fields, const LogField* const * fields); virtual void DoFinish(); @@ -42,9 +42,12 @@ private: string fname; unsigned int num_fields; + unsigned int idx_fields; // map columns in the file to columns to send back to the manager vector columnMap; + + //map *keyMap; }; diff --git a/src/Val.h b/src/Val.h index d851be311b..3ae0bc3334 100644 --- a/src/Val.h +++ b/src/Val.h @@ -841,6 +841,9 @@ public: timer = 0; } + HashKey* ComputeHash(const Val* index) const + { return table_hash->ComputeHash(index, 1); } + protected: friend class Val; friend class StateAccess; @@ -851,8 +854,6 @@ protected: void CheckExpireAttr(attr_tag at); int ExpandCompoundAndInit(val_list* vl, int k, Val* new_val); int CheckAndAssign(Val* index, Val* new_val, Opcode op = OP_ASSIGN); - HashKey* ComputeHash(const Val* index) const - { return table_hash->ComputeHash(index, 1); } bool AddProperties(Properties arg_state); bool RemoveProperties(Properties arg_state); From 638976791efb3b7d9d491266d83bb30b73d4bd53 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 2 Nov 2011 14:00:19 -0700 Subject: [PATCH 011/149] hashing seems to work _correctly_ now... --- src/InputMgr.cc | 44 ++++++++++++++++++++++++++++++----------- src/InputReaderAscii.cc | 41 +++++++++++++++++++++++++++++++++----- src/InputReaderAscii.h | 3 +++ 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5270c83d5a..e3acfb9505 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -290,10 +290,10 @@ bool InputMgr::ForceUpdate(EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->Error("Reader not found"); + reporter->InternalError("Reader not found"); return false; } - + return i->reader->Update(); } @@ -302,8 +302,9 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo int position = 0; - if ( num_fields == 1 ) { + if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { idxval = LogValToVal(vals[0]); + position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); for ( int j = 0 ; j < type->NumFields(); j++ ) { @@ -317,6 +318,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo idxval = l; } + //reporter->Error("Position: %d, num_fields: %d", position, num_fields); assert ( position == num_fields ); return idxval; @@ -331,8 +333,15 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { return; } + + reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); + reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + reporter->Error("Result: %d", (uint64_t) valhash->Hash()); + + //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); InputHash *h = i->lastDict->Lookup(idxhash); if ( h != 0 ) { @@ -393,9 +402,12 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } i->tab->Assign(idxval, k, valval); + InputHash* ih = new InputHash(); + k = i->tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; + //i->tab->Delete(k); i->currDict->Insert(idxhash, ih); @@ -407,11 +419,12 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { reporter->InternalError("Unknown reader"); return; } - - // lastdict contains all deleted entries + // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; - while ( ( ih = i->lastDict->NextEntry(c )) ) { + reporter->Error("ending"); + while ( ( ih = i->lastDict->NextEntry(c) ) ) { + reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } @@ -582,28 +595,37 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } + //reporter->Error("Length: %d", length); + int position = 0; char *data = (char*) malloc(length); + if ( data == 0 ) { + reporter->InternalError("Could not malloc?"); + } for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: - *(data+position) = val->val.int_val; + //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, position); + memcpy(data+position, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); + //*(data+position) = val->val.int_val; position += sizeof(val->val.int_val); break; case TYPE_COUNT: case TYPE_COUNTER: case TYPE_PORT: - *(data+position) = val->val.uint_val; + //*(data+position) = val->val.uint_val; + memcpy(data+position, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); position += sizeof(val->val.uint_val); break; case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - *(data+position) = val->val.double_val; + //*(data+position) = val->val.double_val; + memcpy(data+position, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); position += sizeof(val->val.double_val); break; @@ -685,7 +707,7 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->reader == reader ) + if ( (*s)->reader && (*s)->reader == reader ) { return *s; } @@ -699,7 +721,7 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->id == id ) + if ( (*s)->id && (*s)->id->AsEnum() == id->AsEnum() ) { return *s; } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e434f7e750..4755b2b74f 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -50,6 +50,16 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } + + this->num_fields = num_fields; + this->idx_fields = idx_fields; + this->fields = fields; + + return true; +} + + +bool InputReaderAscii::ReadHeader() { // try to read the header line... string line; if ( !getline(*file, line) ) { @@ -57,9 +67,6 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } - this->num_fields = num_fields; - this->idx_fields = idx_fields; - // split on tabs... istringstream splitstream(line); unsigned int currTab = 0; @@ -70,7 +77,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const break; // current found heading in s... compare if we want it - for ( int i = 0; i < num_fields; i++ ) { + for ( unsigned int i = 0; i < num_fields; i++ ) { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position @@ -92,7 +99,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const currTab++; } - if ( wantFields != num_fields ) { + if ( wantFields != (int) num_fields ) { // we did not find all fields? // :( Error("One of the requested fields could not be found in the input data file"); @@ -106,9 +113,30 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { + + + // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) + if ( file && file->is_open() ) { + file->close(); + } + file = new ifstream(fname.c_str()); + if ( !file->is_open() ) { + Error(Fmt("cannot open %s", fname.c_str())); + return false; + } + // + + // file->seekg(0, ios::beg); // do not forget clear. + + + if ( ReadHeader() == false ) { + return false; + } + // TODO: all the stuff we need for a second reading. // *cough* // + // // new keymap @@ -152,6 +180,7 @@ bool InputReaderAscii::DoUpdate() { } LogVal* val = new LogVal(currMapping.type, true); + //bzero(val, sizeof(LogVal)); switch ( currMapping.type ) { case TYPE_STRING: @@ -267,6 +296,8 @@ bool InputReaderAscii::DoUpdate() { } + //file->clear(); // remove end of file evil bits + //file->seekg(0, ios::beg); // and seek to start. EndCurrentSend(); return true; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c26a139dcd..673a2cdae2 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -37,6 +37,8 @@ protected: virtual bool DoUpdate(); private: + + bool ReadHeader(); ifstream* file; string fname; @@ -46,6 +48,7 @@ private: // map columns in the file to columns to send back to the manager vector columnMap; + const LogField* const * fields; // raw mapping //map *keyMap; From b5a77aa77baf69b788adb7b40f4246f0692939a0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 2 Nov 2011 14:29:58 -0700 Subject: [PATCH 012/149] reading seems to work with all atomic types + records... --- src/InputMgr.cc | 25 ++++++++++++++++++++----- src/InputReaderAscii.cc | 2 +- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e3acfb9505..bbfd354189 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -334,12 +334,12 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } - reporter->Error("Hashing %d index fields", i->num_idx_fields); + //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); - reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); - reporter->Error("Hashing %d val fields", i->num_val_fields); + //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); + //reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); - reporter->Error("Result: %d", (uint64_t) valhash->Hash()); + //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); @@ -422,7 +422,6 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; - reporter->Error("ending"); while ( ( ih = i->lastDict->NextEntry(c) ) ) { reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); @@ -589,6 +588,11 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals length += NUM_ADDR_WORDS*sizeof(uint32_t); break; + case TYPE_SUBNET: + length += sizeof(val->val.subnet_val.width); + length += sizeof(val->val.subnet_val.net); + break; + default: reporter->InternalError("unsupported type for hashlogvals"); } @@ -641,6 +645,13 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals position += NUM_ADDR_WORDS*sizeof(uint32_t); break; + case TYPE_SUBNET: + memcpy(data+position,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); + position += sizeof(val->val.subnet_val.width); + memcpy(data+position, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + position += sizeof(val->val.subnet_val.net); + break; + default: reporter->InternalError("unsupported type for hashlogvals2"); } @@ -694,6 +705,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new AddrVal(val->val.addr_val); break; + case TYPE_SUBNET: + return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); + break; + default: reporter->InternalError("unsupported type for input_read"); } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4755b2b74f..39635de407 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -216,7 +216,7 @@ bool InputReaderAscii::DoUpdate() { case TYPE_SUBNET: { int pos = s.find("/"); - string width = s.substr(pos); + string width = s.substr(pos+1); val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; From 4845c3a9a61d95dd0246f3beb6fcaa6e74c88f8c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 3 Nov 2011 14:04:13 -0700 Subject: [PATCH 013/149] send events when input entries change --- src/InputMgr.cc | 97 +++++++++++++++++++++++++++++++++++++++++++++- src/InputMgr.h | 8 +++- src/InputReader.cc | 24 +++++++----- src/input.bif | 11 ++++++ src/types.bif | 7 ++++ 5 files changed, 133 insertions(+), 14 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index bbfd354189..e987dda81e 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -38,6 +38,8 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; + + list* events; // events we fire when "something" happens }; @@ -165,6 +167,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + info->events = new list(); int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { @@ -223,7 +226,7 @@ bool InputMgr::IsCompatibleType(BroType* t) return false; } -bool InputMgr::RemoveReader(EnumVal* id) { +bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -254,6 +257,43 @@ bool InputMgr::RemoveReader(EnumVal* id) { return true; } +bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + i->events->push_back(eventName); + + return true; +} + +bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + bool erased = false; + + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) + { + if ( *it == eventName ) { + it = i->events->erase(it); + erased = true; + } + else + ++it; + } + + + return erased; +} + + bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { @@ -286,7 +326,7 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec } -bool InputMgr::ForceUpdate(EnumVal* id) +bool InputMgr::ForceUpdate(const EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -333,6 +373,8 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { return; } + bool updated = false; + //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); @@ -355,6 +397,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { // updated i->lastDict->Remove(idxhash); delete(h); + updated = true; } } @@ -411,8 +454,25 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->currDict->Insert(idxhash, ih); + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) { + EnumVal* ev; + if ( updated ) { + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + } + + + Ref(idxval); + Ref(valval); + SendEvent(*it, ev, idxval, valval); + ++it; + } + } + void InputMgr::EndCurrentSend(const InputReader* reader) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -423,6 +483,23 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; while ( ( ih = i->lastDict->NextEntry(c) ) ) { + + if ( i->events->size() > 0 ) { + ListVal *idx = i->tab->RecoverIndex(ih->idxkey); + assert(idx != 0); + Val *val = i->tab->Lookup(idx); + assert(val != 0); + + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(*it, ev, idx, val); + ++it; + } + } + reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } @@ -523,6 +600,22 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } +void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) +{ + EventHandler* handler = event_registry->Lookup(name.c_str()); + if ( handler == 0 ) { + reporter->Error("Event %s not found", name.c_str()); + return; + } + + val_list* vl = new val_list; + vl->append(event); + vl->append(left); + vl->append(right); + + mgr.Dispatch(new Event(handler, vl)); +} + Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { diff --git a/src/InputMgr.h b/src/InputMgr.h index d147fa262a..378838c2cf 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -21,8 +21,11 @@ public: InputMgr(); InputReader* CreateReader(EnumVal* id, RecordVal* description); - bool ForceUpdate(EnumVal* id); - bool RemoveReader(EnumVal* id); + bool ForceUpdate(const EnumVal* id); + bool RemoveReader(const EnumVal* id); + bool RegisterEvent(const EnumVal* id, string eventName); + bool UnregisterEvent(const EnumVal* id, string eventName); + protected: friend class InputReader; @@ -43,6 +46,7 @@ private: bool IsCompatibleType(BroType* t); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); diff --git a/src/InputReader.cc b/src/InputReader.cc index 494df3fb81..994f8b9b97 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,13 +1,12 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include "InputReader.h" -// #include "EventRegistry.h" -// #include "Event.h" InputReader::InputReader() { buf = 0; buf_len = 1024; - disabled = true; // disabled will be set correcty in init. + disabled = true; // disabled will be set correcty in init. } InputReader::~InputReader() @@ -21,9 +20,9 @@ void InputReader::Error(const char *msg) } void InputReader::Error(const string &msg) - { +{ input_mgr->Error(this, msg.c_str()); - } +} void InputReader::Put(const LogVal* const *val) { @@ -54,16 +53,19 @@ bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields return !disabled; } -void InputReader::Finish() { +void InputReader::Finish() +{ DoFinish(); disabled = true; } -bool InputReader::Update() { +bool InputReader::Update() +{ return DoUpdate(); } -void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { +void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +{ input_mgr->SendEvent(name, num_vals, vals); } @@ -93,10 +95,12 @@ const char* InputReader::Fmt(const char* format, ...) } -void InputReader::SendEntry(const LogVal* const *vals) { +void InputReader::SendEntry(const LogVal* const *vals) +{ input_mgr->SendEntry(this, vals); } -void InputReader::EndCurrentSend() { +void InputReader::EndCurrentSend() +{ input_mgr->EndCurrentSend(this); } diff --git a/src/input.bif b/src/input.bif index 4e2cbf07b5..90dd2386b1 100644 --- a/src/input.bif +++ b/src/input.bif @@ -21,6 +21,17 @@ function Input::__force_update%(id: ID%) : bool return new Val( res, TYPE_BOOL ); %} +function Input::__add_event%(id: ID, name: string%) : bool + %{ + bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_event%(id: ID, name: string%) : bool + %{ + bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL ); + %} function Input::__remove_reader%(id: ID%) : bool %{ diff --git a/src/types.bif b/src/types.bif index 7b81a7f631..f90a954224 100644 --- a/src/types.bif +++ b/src/types.bif @@ -174,6 +174,13 @@ enum Reader %{ READER_ASCII, %} +enum Event %{ + EVENT_NEW, + EVENT_CHANGED, + EVENT_REMOVED, +%} + + enum ID %{ Unknown, %} From 2e3874331d0540724a47e91c8b0146b7b3fa6846 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 12:41:10 -0700 Subject: [PATCH 014/149] support for filters and little event fix --- scripts/base/frameworks/input/main.bro | 10 ++ src/InputMgr.cc | 204 +++++++++++++++++++++---- src/InputMgr.h | 5 + src/input.bif | 22 ++- 4 files changed, 210 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4bb7129d03..35ca6dfa4e 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -11,6 +11,16 @@ export { destination: any; reader: Reader &default=default_reader; }; + + type Filter: record { + name: string; + ## descriptive name. for later removal + + + pred: function(typ: Input::Event, left: any, right: any): bool &optional; + ## decision function, that decides if an inserton, update or removal should really be executed + }; + } @load base/input.bif diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e987dda81e..0b87a561ad 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -24,6 +24,11 @@ public: declare(PDict, InputHash); +struct InputMgr::Filter { + EnumVal* id; + string name; + Func* pred; +}; struct InputMgr::ReaderInfo { EnumVal* id; @@ -39,10 +44,17 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; - list* events; // events we fire when "something" happens + list events; // events we fire when "something" happens + list filters; // events we fire when "something" happens + +// ~ReaderInfo(); }; +//void InputMgr::~ReaderInfo() { +// +//} + struct InputReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -167,8 +179,6 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); - info->events = new list(); - int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { RemoveReader(id); @@ -264,7 +274,7 @@ bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { return false; } - i->events->push_back(eventName); + i->events.push_back(eventName); return true; } @@ -276,21 +286,23 @@ bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { return false; } - bool erased = false; + //bool erased = false; - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) + std::list::iterator it = i->events.begin(); + while ( it != i->events.end() ) { if ( *it == eventName ) { - it = i->events->erase(it); - erased = true; + it = i->events.erase(it); + return true; + // erased = true; } else ++it; } - return erased; + return false; + //return erased; } @@ -337,6 +349,59 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } +bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + + Filter filter; + filter.name = name->AsString()->CheckString(); + filter.id = id->Ref()->AsEnumVal(); + filter.pred = pred ? pred->AsFunc() : 0; + + i->filters.push_back(filter); + + return true; +} + +bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) + { + if ( (*it).name == name ) { + it = i->filters.erase(it); + return true; + break; + } + else + ++it; + } + + return false;; +} + + + Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { Val* idxval; int position = 0; @@ -398,6 +463,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->lastDict->Remove(idxhash); delete(h); updated = true; + } } @@ -437,6 +503,48 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { valval = r; } + + Val* oldval = 0; + if ( updated == true ) { + // in that case, we need the old value to send the event (if we send an event). + oldval = i->tab->Lookup(idxval); + } + + + // call filters first do determine if we really add / change the entry + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) { + if (! (*it).pred ) { + continue; + } + + EnumVal* ev; + Ref(idxval); + Ref(valval); + + if ( updated ) { + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + } + + val_list vl(3); + vl.append(ev); + vl.append(idxval); + vl.append(valval); + Val* v = (*it).pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // throw away. Hence - we quit. + return; + } + + ++it; + } + + //i->tab->Assign(idxval, valval); HashKey* k = i->tab->ComputeHash(idxval); if ( !k ) { @@ -454,21 +562,28 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->currDict->Insert(idxhash, ih); - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) { + // send events now that we are kind of finished. + std::list::iterator filter_iterator = i->events.begin(); + while ( filter_iterator != i->events.end() ) { EnumVal* ev; - if ( updated ) { + Ref(idxval); + + if ( updated ) { // in case of update send back the old value. ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + assert ( oldval != 0 ); + Ref(oldval); + SendEvent(*filter_iterator, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + Ref(valval); + SendEvent(*filter_iterator, ev, idxval, valval); } - Ref(idxval); - Ref(valval); - SendEvent(*it, ev, idxval, valval); - ++it; + ++filter_iterator; } + + } @@ -483,24 +598,61 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; while ( ( ih = i->lastDict->NextEntry(c) ) ) { + + if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event + { - if ( i->events->size() > 0 ) { ListVal *idx = i->tab->RecoverIndex(ih->idxkey); assert(idx != 0); Val *val = i->tab->Lookup(idx); assert(val != 0); - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) { - Ref(idx); - Ref(val); - EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(*it, ev, idx, val); - ++it; + + { + // ask filter, if we want to expire this element... + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) { + if (! (*it).pred ) { + continue; + } + + EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + Ref(idx); + Ref(val); + + val_list vl(3); + vl.append(ev); + vl.append(idx); + vl.append(val); + Val* v = (*it).pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // throw away. Hence - we quit and simply go to the next entry of lastDict + continue; + } + + ++it; + } } + + // + + { + std::list::iterator it = i->events.begin(); + while ( it != i->events.end() ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(*it, ev, idx, val); + ++it; + } + } + } - reporter->Error("Expiring element"); + //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } diff --git a/src/InputMgr.h b/src/InputMgr.h index 378838c2cf..74914e65ad 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -26,6 +26,9 @@ public: bool RegisterEvent(const EnumVal* id, string eventName); bool UnregisterEvent(const EnumVal* id, string eventName); + bool AddFilter(EnumVal *id, RecordVal* filter); + bool RemoveFilter(EnumVal* id, const string &name); + protected: friend class InputReader; @@ -62,6 +65,8 @@ private: string Hash(const string &input); + struct Filter; + }; extern InputMgr* input_mgr; diff --git a/src/input.bif b/src/input.bif index 90dd2386b1..2301482506 100644 --- a/src/input.bif +++ b/src/input.bif @@ -8,34 +8,46 @@ module Input; %%} type ReaderDescription: record; +type Filter: record; -function Input::__create_reader%(id: ID, description: Input::ReaderDescription%) : bool +function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescription%) : bool %{ InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__force_update%(id: ID%) : bool +function Input::__force_update%(id: Log::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_event%(id: ID, name: string%) : bool +function Input::__add_event%(id: Log::ID, name: string%) : bool %{ bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_event%(id: ID, name: string%) : bool +function Input::__remove_event%(id: Log::ID, name: string%) : bool %{ bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_reader%(id: ID%) : bool +function Input::__remove_reader%(id: Log::ID%) : bool %{ bool res = input_mgr->RemoveReader(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} +function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool + %{ + bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_filter%(id: Log::ID, name: string%) : bool + %{ + bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL); + %} From 5f37040c962a9f3d524ddcccfaeba982acb0d500 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 13:59:43 -0700 Subject: [PATCH 015/149] filters really working as intented (though probably still memleaky) --- src/InputMgr.cc | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0b87a561ad..7ca1e1b485 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -537,8 +537,14 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { Unref(v); if ( result == false ) { - // throw away. Hence - we quit. - return; + if ( !updated ) { + // throw away. Hence - we quit. + return; + } else { + // keep old one + i->currDict->Insert(idxhash, h); + return; + } } ++it; @@ -596,8 +602,11 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { } // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); + i->lastDict->MakeRobustCookie(c); InputHash* ih; - while ( ( ih = i->lastDict->NextEntry(c) ) ) { + HashKey *lastDictIdxKey; + //while ( ( ih = i->lastDict->NextEntry(c) ) ) { + while ( ( ih = i->lastDict->NextEntry(lastDictIdxKey, c) ) ) { if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event { @@ -609,6 +618,7 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { { + bool doBreak = false; // ask filter, if we want to expire this element... std::list::iterator it = i->filters.begin(); while ( it != i->filters.end() ) { @@ -627,13 +637,21 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { Val* v = (*it).pred->Call(&vl); bool result = v->AsBool(); Unref(v); + + ++it; if ( result == false ) { - // throw away. Hence - we quit and simply go to the next entry of lastDict + // Keep it. Hence - we quit and simply go to the next entry of lastDict + // ah well - and we have to add the entry to currDict... + i->currDict->Insert(lastDictIdxKey, i->lastDict->RemoveEntry(lastDictIdxKey)); + doBreak = true; continue; } - ++it; + } + + if ( doBreak ) { + continue; } } @@ -654,6 +672,8 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); + i->lastDict->Remove(lastDictIdxKey); + delete(ih); } i->lastDict->Clear(); From 2aa0f6da5795aa90c83c9fa1872776737f1e7416 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 14:33:34 -0700 Subject: [PATCH 016/149] beautify script calls, track filters --- scripts/base/frameworks/input/main.bro | 63 +++++++++++++++++++++++++- src/input.bif | 12 ++--- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 35ca6dfa4e..e0c41f19be 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -20,7 +20,68 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## decision function, that decides if an inserton, update or removal should really be executed }; - + + const no_filter: Filter = [$name=""]; # Sentinel. + + global create_reader: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global remove_reader: function(id: Log::ID) : bool; + global force_update: function(id: Log::ID) : bool; + global add_event: function(id: Log::ID, name: string) : bool; + global remove_event: function(id: Log::ID, name: string) : bool; + global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; + global remove_filter: function(id: Log::ID, name: string) : bool; + global get_filter: function(id: ID, name: string) : Filter; + } @load base/input.bif + + +module Input; + +global filters: table[ID, string] of Filter; + +function create_reader(id: Log::ID, description: Input::ReaderDescription) : bool + { + return __create_reader(id, description); + } + +function remove_reader(id: Log::ID) : bool + { + return __remove_reader(id); + } + +function force_update(id: Log::ID) : bool + { + return __force_update(id); + } + +function add_event(id: Log::ID, name: string) : bool + { + return __add_event(id, name); + } + +function remove_event(id: Log::ID, name: string) : bool + { + return __remove_event(id, name); + } + +function add_filter(id: Log::ID, filter: Input::Filter) : bool + { + filters[id, filter$name] = filter; + return __add_filter(id, filter); + } + +function remove_filter(id: Log::ID, name: string) : bool + { + delete filters[id, name]; + return __remove_filter(id, name); + } + +function get_filter(id: ID, name: string) : Filter + { + if ( [id, name] in filters ) + return filters[id, name]; + + return no_filter; + } diff --git a/src/input.bif b/src/input.bif index 2301482506..c4bf14e3ed 100644 --- a/src/input.bif +++ b/src/input.bif @@ -16,6 +16,12 @@ function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescript return new Val( the_reader != 0, TYPE_BOOL ); %} +function Input::__remove_reader%(id: Log::ID%) : bool + %{ + bool res = input_mgr->RemoveReader(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + function Input::__force_update%(id: Log::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); @@ -34,12 +40,6 @@ function Input::__remove_event%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__remove_reader%(id: Log::ID%) : bool - %{ - bool res = input_mgr->RemoveReader(id->AsEnumVal()); - return new Val( res, TYPE_BOOL ); - %} - function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool %{ bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); From 1d39eaf32dff51ef779188f3123e3d63a28f71f8 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 15:03:40 -0700 Subject: [PATCH 017/149] small fixes, less leakiness --- scripts/base/frameworks/input/main.bro | 1 - src/InputMgr.cc | 98 +++++++++++--------------- 2 files changed, 41 insertions(+), 58 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index e0c41f19be..6e19452f7a 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -16,7 +16,6 @@ export { name: string; ## descriptive name. for later removal - pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## decision function, that decides if an inserton, update or removal should really be executed }; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 7ca1e1b485..18b17c7576 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -28,8 +28,14 @@ struct InputMgr::Filter { EnumVal* id; string name; Func* pred; + + ~Filter(); }; +InputMgr::Filter::~Filter() { + Unref(id); +} + struct InputMgr::ReaderInfo { EnumVal* id; EnumVal* type; @@ -45,15 +51,20 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* lastDict; list events; // events we fire when "something" happens - list filters; // events we fire when "something" happens - -// ~ReaderInfo(); + list filters; // filters that can prevent our actions + ~ReaderInfo(); }; -//void InputMgr::~ReaderInfo() { -// -//} +InputMgr::ReaderInfo::~ReaderInfo() { + Unref(type); + Unref(tab); + Unref(itype); + Unref(rtype); + Unref(id); + + delete(reader); +} struct InputReaderDefinition { bro_int_t type; // the type @@ -71,10 +82,8 @@ InputReaderDefinition input_readers[] = { InputMgr::InputMgr() { - //DBG_LOG(DBG_LOGGING, "this has to happen"); } - // create a new input reader object to be used at whomevers leisure lateron. InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) { @@ -163,30 +172,26 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; - info->type = reader; - Ref(reader); + info->type = reader->Ref()->AsEnumVal(); info->num_idx_fields = idxfields; info->num_val_fields = valfields; - info->tab = dst; - Ref(dst); - info->rtype = val; - Ref(val); // we save a pointer of it... I really hope that this wasn't already done anywhere. - info->id = id; - Ref(id); // ditto... - info->itype = idx; - Ref(idx); - readers.push_back(info); + info->tab = dst->Ref()->AsTableVal(); + info->rtype = val->Ref()->AsRecordType(); + info->id = id->Ref()->AsEnumVal(); + info->itype = idx->Ref()->AsRecordType(); info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + readers.push_back(info); + int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { - RemoveReader(id); + assert( RemoveReader(id) ); return 0; } success = reader_obj->Update(); if ( success == false ) { - RemoveReader(id); + assert ( RemoveReader(id) ); return 0; } @@ -240,12 +245,12 @@ bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->id == id ) - { - i = (*s); - readers.erase(s); // remove from vector - break; - } + if ( (*s)->id == id ) + { + i = (*s); + readers.erase(s); // remove from vector + break; + } } if ( i == 0 ) { @@ -254,14 +259,6 @@ bool InputMgr::RemoveReader(const EnumVal* id) { i->reader->Finish(); - - Unref(i->type); - Unref(i->tab); - Unref(i->itype); - Unref(i->rtype); - Unref(i->id); - - delete(i->reader); delete(i); return true; @@ -279,6 +276,8 @@ bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { return true; } +// remove first event with name eventName +// (though there shouldn't really be several events with the same name... bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -286,23 +285,18 @@ bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { return false; } - //bool erased = false; - std::list::iterator it = i->events.begin(); while ( it != i->events.end() ) { if ( *it == eventName ) { it = i->events.erase(it); return true; - // erased = true; } else ++it; } - return false; - //return erased; } @@ -335,14 +329,13 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec } return true; - } bool InputMgr::ForceUpdate(const EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -352,7 +345,7 @@ bool InputMgr::ForceUpdate(const EnumVal* id) bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -380,7 +373,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -427,7 +420,6 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo assert ( position == num_fields ); return idxval; - } @@ -538,7 +530,8 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { if ( result == false ) { if ( !updated ) { - // throw away. Hence - we quit. + // throw away. Hence - we quit. And remove the entry from the current dictionary... + delete(i->currDict->RemoveEntry(idxhash)); return; } else { // keep old one @@ -588,9 +581,6 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { ++filter_iterator; } - - - } @@ -670,13 +660,12 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { } - //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); - i->lastDict->Remove(lastDictIdxKey); + i->lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->lastDict->Clear(); + i->lastDict->Clear(); // should be empty... but... well... who knows... delete(i->lastDict); i->lastDict = i->currDict; @@ -699,11 +688,6 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { } else { RecordVal * r = new RecordVal(i->rtype); - /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { - reporter->InternalError("Type mismatch"); - return; - } */ - for ( int j = 0; j < i->rtype->NumFields(); j++) { Val* val = 0; From 5983d44d950d2b93d91c36846668b35fb24d437a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 8 Nov 2011 15:33:32 -0800 Subject: [PATCH 018/149] read header line in bro logfile format --- src/InputReaderAscii.cc | 19 +++++++++++++++++-- src/InputReaderAscii.h | 3 ++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 39635de407..e4770421c6 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -62,7 +62,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const bool InputReaderAscii::ReadHeader() { // try to read the header line... string line; - if ( !getline(*file, line) ) { + if ( !GetLine(line) ) { Error("could not read first line"); return false; } @@ -111,6 +111,21 @@ bool InputReaderAscii::ReadHeader() { return true; } +bool InputReaderAscii::GetLine(string& str) { + while ( getline(*file, str) ) { + if ( str[0] != '#' ) { + return true; + } + + if ( str.compare(0,8, "#fields\t") == 0 ) { + str = str.substr(8); + return true; + } + } + + return false; +} + // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { @@ -143,7 +158,7 @@ bool InputReaderAscii::DoUpdate() { //map *newKeyMap = new map(); string line; - while ( getline(*file, line ) ) { + while ( GetLine(line ) ) { // split on tabs istringstream splitstream(line); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 673a2cdae2..f86cfd0062 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -37,8 +37,9 @@ protected: virtual bool DoUpdate(); private: - bool ReadHeader(); + + bool GetLine(string& str); ifstream* file; string fname; From 1a642f3568c3c8c445f76a5b7438f385b880bbe7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 14 Nov 2011 17:18:28 -0800 Subject: [PATCH 019/149] tried enum support - doesn't yet work due to internal bro interface problems... --- src/InputMgr.cc | 6 ++++++ src/InputReaderAscii.cc | 2 +- src/input.bif | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 18b17c7576..8db5a70c66 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -828,6 +828,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals break; case TYPE_STRING: + case TYPE_ENUM: { length += val->val.string_val->size(); break; @@ -883,6 +884,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals break; case TYPE_STRING: + case TYPE_ENUM: { memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); position += val->val.string_val->size(); @@ -958,6 +960,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); break; + case TYPE_ENUM: + reporter->InternalError("Sorry, Enums reading does not yet work, missing internal inferface"); + + default: reporter->InternalError("unsupported type for input_read"); } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e4770421c6..f91c55b666 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -128,7 +128,6 @@ bool InputReaderAscii::GetLine(string& str) { // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { - // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) if ( file && file->is_open() ) { @@ -198,6 +197,7 @@ bool InputReaderAscii::DoUpdate() { //bzero(val, sizeof(LogVal)); switch ( currMapping.type ) { + case TYPE_ENUM: case TYPE_STRING: val->val.string_val = new string(s); break; diff --git a/src/input.bif b/src/input.bif index c4bf14e3ed..7b051fba16 100644 --- a/src/input.bif +++ b/src/input.bif @@ -51,3 +51,4 @@ function Input::__remove_filter%(id: Log::ID, name: string%) : bool bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); %} + From cde8153c1868f6bd3984a2c35aeb645b1023968d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 08:36:03 -0800 Subject: [PATCH 020/149] switch to set if record or simple value is desired. --- scripts/base/frameworks/input/main.bro | 1 + src/InputMgr.cc | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 6e19452f7a..2b87ac980c 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -9,6 +9,7 @@ export { idx: any; val: any; destination: any; + want_record: bool &default=T; reader: Reader &default=default_reader; }; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 8db5a70c66..5f99583c10 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -42,6 +42,7 @@ struct InputMgr::ReaderInfo { InputReader* reader; unsigned int num_idx_fields; unsigned int num_val_fields; + bool want_record; TableVal* tab; RecordType* rtype; @@ -147,6 +148,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + Val *want_record = description->Lookup(rtype->FieldOffset("want_record")); vector fieldsV; // vector, because we don't know the length beforehands @@ -181,6 +183,11 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->itype = idx->Ref()->AsRecordType(); info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + info->want_record = ( want_record->InternalInt() == 1 ); + + if ( valfields > 1 ) { + assert(info->want_record); + } readers.push_back(info); @@ -311,7 +318,6 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) { - string prep = nameprepend + rec->FieldName(i) + "."; if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) @@ -464,7 +470,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { Val* valval; int position = i->num_idx_fields; - if ( i->num_val_fields == 1 ) { + if ( i->num_val_fields == 1 && !i->want_record ) { valval = LogValToVal(vals[i->num_idx_fields]); } else { RecordVal * r = new RecordVal(i->rtype); @@ -683,7 +689,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { Val* valval; int position = i->num_idx_fields; - if ( i->num_val_fields == 1 ) { + if ( i->num_val_fields == 1 && !i->want_record ) { valval = LogValToVal(vals[i->num_idx_fields]); } else { RecordVal * r = new RecordVal(i->rtype); @@ -961,7 +967,7 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { break; case TYPE_ENUM: - reporter->InternalError("Sorry, Enums reading does not yet work, missing internal inferface"); + reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); default: From 4a3c9923253c8c26973a24c537a3ca231961db5e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 10:57:45 -0800 Subject: [PATCH 021/149] InputReader can read Sets. --- src/InputMgr.cc | 244 +++++++++++++++++++++++++--------------- src/InputMgr.h | 2 + src/InputReaderAscii.cc | 211 ++++++++++++++++++++++------------ src/InputReaderAscii.h | 5 + src/LogMgr.cc | 6 +- src/LogMgr.h | 4 +- 6 files changed, 310 insertions(+), 162 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5f99583c10..ba17d8448f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -234,11 +234,14 @@ bool InputMgr::IsCompatibleType(BroType* t) case TYPE_TABLE: - return false; + { + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + } case TYPE_VECTOR: { - return IsCompatibleType(t->AsVectorType()->YieldType()); + return false; // do me... + //return IsCompatibleType(t->AsVectorType()->YieldType()); } default: @@ -329,6 +332,9 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec LogField* field = new LogField(); field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); + if ( field->type == TYPE_TABLE ) { + field->set_type = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + } fields->push_back(field); } @@ -810,49 +816,133 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ } + +int InputMgr::GetLogValLength(const LogVal* val) { + int length = 0; + + switch (val->type) { + case TYPE_BOOL: + case TYPE_INT: + length += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + length += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + length += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + length += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + length += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + case TYPE_SUBNET: + length += sizeof(val->val.subnet_val.width); + length += sizeof(val->val.subnet_val.net); + break; + + case TYPE_TABLE: { + for ( int i = 0; i < val->val.set_val.size; i++ ) { + length += GetLogValLength(val->val.set_val.vals[i]); + } + break; + } + + default: + reporter->InternalError("unsupported type %d for GetLogValLength", val->type); + } + + return length; + +} + +int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, startpos); + memcpy(data+startpos, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); + //*(data+startpos) = val->val.int_val; + return sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + //*(data+startpos) = val->val.uint_val; + memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); + return sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + //*(data+startpos) = val->val.double_val; + memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); + return sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + memcpy(data+startpos, val->val.string_val->c_str(), val->val.string_val->length()); + return val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + memcpy(data+startpos, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); + return NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + case TYPE_SUBNET: { + int length = 0; + memcpy(data+startpos,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); + length += sizeof(val->val.subnet_val.width); + memcpy(data+startpos, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + length += sizeof(val->val.subnet_val.net); + return length; + break; + } + + case TYPE_TABLE: { + int length = 0; + for ( int i = 0; i < val->val.set_val.size; i++ ) { + length += CopyLogVal(data, startpos+length, val->val.set_val.vals[i]); + } + return length; + break; + } + + default: + reporter->InternalError("unsupported type %d for CopyLogVal", val->type); + return 0; + } + + reporter->InternalError("internal error"); + return 0; + +} + HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals) { int length = 0; for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; - switch (val->type) { - case TYPE_BOOL: - case TYPE_INT: - length += sizeof(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - length += sizeof(val->val.uint_val); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - length += sizeof(val->val.double_val); - break; - - case TYPE_STRING: - case TYPE_ENUM: - { - length += val->val.string_val->size(); - break; - } - - case TYPE_ADDR: - length += NUM_ADDR_WORDS*sizeof(uint32_t); - break; - - case TYPE_SUBNET: - length += sizeof(val->val.subnet_val.width); - length += sizeof(val->val.subnet_val.net); - break; - - default: - reporter->InternalError("unsupported type for hashlogvals"); - } - + length += GetLogValLength(val); } //reporter->Error("Length: %d", length); @@ -864,56 +954,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; - switch ( val->type ) { - case TYPE_BOOL: - case TYPE_INT: - //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, position); - memcpy(data+position, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); - //*(data+position) = val->val.int_val; - position += sizeof(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - //*(data+position) = val->val.uint_val; - memcpy(data+position, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); - position += sizeof(val->val.uint_val); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - //*(data+position) = val->val.double_val; - memcpy(data+position, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); - position += sizeof(val->val.double_val); - break; - - case TYPE_STRING: - case TYPE_ENUM: - { - memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); - position += val->val.string_val->size(); - break; - } - - case TYPE_ADDR: - memcpy(data+position, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); - position += NUM_ADDR_WORDS*sizeof(uint32_t); - break; - - case TYPE_SUBNET: - memcpy(data+position,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); - position += sizeof(val->val.subnet_val.width); - memcpy(data+position, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); - position += sizeof(val->val.subnet_val.net); - break; - - default: - reporter->InternalError("unsupported type for hashlogvals2"); - } - - + position += CopyLogVal(data, position, val); } assert(position == length); @@ -966,6 +1007,29 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); break; + case TYPE_TABLE: { + if ( val->val.set_val.size == 0 ) { + // empty table + TypeList* set_index = new TypeList(base_type(TYPE_ANY)); + // iim quite sure this does not work... we probably need the internal set type for this... + reporter->InternalError("Implement me."); + return new TableVal(new SetType(set_index, 0)); + } else { + // all entries have to have the same type... + TypeTag type = val->val.set_val.vals[0]->type; + TypeList* set_index = new TypeList(base_type(type)); + set_index->Append(base_type(type)); + SetType* s = new SetType(set_index, 0); + TableVal* t = new TableVal(s); + for ( int i = 0; i < val->val.set_val.size; i++ ) { + assert( val->val.set_val.vals[i]->type == type); + t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return t; + } + break; + } + case TYPE_ENUM: reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); diff --git a/src/InputMgr.h b/src/InputMgr.h index 74914e65ad..d4bfa5c355 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -52,6 +52,8 @@ private: void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); + int GetLogValLength(const LogVal* val); + int CopyLogVal(char *data, const int startpos, const LogVal* val); Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index f91c55b666..60a8c5685a 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -11,12 +11,22 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int position = arg_position; } +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position) + : name(arg_name), type(arg_type), set_type(arg_set_type) +{ + position = arg_position; +} + FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type) + : name(arg.name), type(arg.type), set_type(arg.set_type) { position = arg.position; } +FieldMapping FieldMapping::setType() { + return FieldMapping(name, set_type, position); +} + InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); @@ -81,7 +91,7 @@ bool InputReaderAscii::ReadHeader() { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position - FieldMapping f(field->name, field->type, i); + FieldMapping f(field->name, field->type, field->set_type, i); columnMap.push_back(f); wantFields++; break; // done with searching @@ -126,6 +136,132 @@ bool InputReaderAscii::GetLine(string& str) { return false; } + +LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { + + LogVal* val = new LogVal(field.type, true); + //bzero(val, sizeof(LogVal)); + + switch ( field.type ) { + case TYPE_ENUM: + case TYPE_STRING: + val->val.string_val = new string(s); + break; + + case TYPE_BOOL: + if ( s == "T" ) { + val->val.int_val = 1; + } else if ( s == "F" ) { + val->val.int_val = 0; + } else { + Error(Fmt("Invalid value for boolean: %s", s.c_str())); + return false; + } + break; + + case TYPE_INT: + val->val.int_val = atoi(s.c_str()); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = atof(s.c_str()); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + val->val.uint_val = atoi(s.c_str()); + break; + + case TYPE_SUBNET: { + int pos = s.find("/"); + string width = s.substr(pos+1); + val->val.subnet_val.width = atoi(width.c_str()); + string addr = s.substr(0, pos); + s = addr; + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + val->val.subnet_val.net = dotted_to_addr(s.c_str()); + break; + + } + case TYPE_ADDR: { + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + addr_type t = dotted_to_addr(s.c_str()); +#ifdef BROv6 + copy_addr(t, val->val.addr_val); +#else + copy_addr(&t, val->val.addr_val); +#endif + break; + } + + case TYPE_TABLE: { + // construct a table from entry... + // for the moment assume, that entries are split by ",". + + if ( s == "-" ) { + // empty + val->val.set_val.size = 0; + break; + } + + // how many entries do we have... + unsigned int length = 1; + for ( unsigned int i = 0; i < s.size(); i++ ) + if ( s[i] == ',') length++; + + unsigned int pos = 0; + LogVal** lvals = new LogVal* [length]; + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + + istringstream splitstream(s); + while ( splitstream ) { + string element; + + if ( pos >= length ) { + Error(Fmt("Internal error while parsing set. pos %d > length %d", pos, length)); + break; + } + + if ( !getline(splitstream, element, ',') ) + break; + + + LogVal* newval = EntryToVal(element, field.setType()); + if ( newval == 0 ) { + Error("Error while reading set"); + return 0; + } + lvals[pos] = newval; + + pos++; + + } + + if ( pos != length ) { + Error("Internal error while parsing set: did not find all elements"); + return 0; + } + + break; + } + + + default: + Error(Fmt("unsupported field format %d for %s", field.type, + field.name.c_str())); + return 0; + } + + return val; + +} + // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { @@ -161,7 +297,6 @@ bool InputReaderAscii::DoUpdate() { // split on tabs istringstream splitstream(line); - string s; LogVal** fields = new LogVal*[num_fields]; //string string_fields[num_fields]; @@ -170,6 +305,7 @@ bool InputReaderAscii::DoUpdate() { unsigned int currField = 0; while ( splitstream ) { + string s; if ( !getline(splitstream, s, '\t') ) break; @@ -193,73 +329,10 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal* val = new LogVal(currMapping.type, true); - //bzero(val, sizeof(LogVal)); - - switch ( currMapping.type ) { - case TYPE_ENUM: - case TYPE_STRING: - val->val.string_val = new string(s); - break; - - case TYPE_BOOL: - if ( s == "T" ) { - val->val.int_val = 1; - } else if ( s == "F" ) { - val->val.int_val = 0; - } else { - Error(Fmt("Invalid value for boolean: %s", s.c_str())); - return false; - } - break; - - case TYPE_INT: - val->val.int_val = atoi(s.c_str()); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - val->val.double_val = atof(s.c_str()); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - val->val.uint_val = atoi(s.c_str()); - break; - - case TYPE_SUBNET: { - int pos = s.find("/"); - string width = s.substr(pos+1); - val->val.subnet_val.width = atoi(width.c_str()); - string addr = s.substr(0, pos); - s = addr; - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... - val->val.subnet_val.net = dotted_to_addr(s.c_str()); - break; - - } - case TYPE_ADDR: { - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... - addr_type t = dotted_to_addr(s.c_str()); -#ifdef BROv6 - copy_addr(t, val->val.addr_val); -#else - copy_addr(&t, val->val.addr_val); -#endif - break; - } - - - default: - Error(Fmt("unsupported field format %d for %s", currMapping.type, - currMapping.name.c_str())); + LogVal* val = EntryToVal(s, currMapping); + if ( val == 0 ) { return false; - } - + } fields[currMapping.position] = val; //string_fields[currMapping.position] = s; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index f86cfd0062..ab2b89339c 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,11 +12,15 @@ struct FieldMapping { string name; TypeTag type; + TypeTag set_type; int position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position); FieldMapping(const FieldMapping& arg); FieldMapping() { position = -1; } + + FieldMapping setType(); bool IsEmpty() { return position == -1; } }; @@ -38,6 +42,7 @@ protected: private: bool ReadHeader(); + LogVal* EntryToVal(string s, FieldMapping type); bool GetLine(string& str); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index f78a2a19e0..9818d9cdfb 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -81,16 +81,18 @@ struct LogMgr::Stream { bool LogField::Read(SerializationFormat* fmt) { int t; + int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type")); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "set_type") ); type = (TypeTag) t; + set_type = (TypeTag) it; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)set_type, "set_type")); } LogVal::~LogVal() diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..40dab8677b 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,10 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; + // needed by input framework. otherwise it cannot determine the inner type of a set. + TypeTag set_type; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type) { } + : name(other.name), type(other.type), set_type(other.set_type) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); From 3d0162bcdc2e2a3e4923ac22ec5597cb4d167304 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 11:18:48 -0800 Subject: [PATCH 022/149] isCompatibleType works correctly for tables. --- src/InputMgr.cc | 27 +++++++++++++++++---------- src/InputMgr.h | 2 +- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index ba17d8448f..4506501c94 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -205,7 +205,8 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) return reader_obj; } -bool InputMgr::IsCompatibleType(BroType* t) + +bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) { if ( ! t ) return false; @@ -223,24 +224,29 @@ bool InputMgr::IsCompatibleType(BroType* t) case TYPE_INTERVAL: case TYPE_ENUM: case TYPE_STRING: - case TYPE_RECORD: - // for record: check, if all elements are compatible? But... LogMgr also doesn't do this. - // ^ recursive checking is done in UnrollRecordType. return true; - - case TYPE_FILE: - case TYPE_FUNC: - return false; + case TYPE_RECORD: + return ! atomic_only; case TYPE_TABLE: { - return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + if ( atomic_only ) + return false; + + if ( ! t->IsSet() ) + return false; + + return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); } case TYPE_VECTOR: { return false; // do me... + + //if ( atomic_only ) + // return false; + // //return IsCompatibleType(t->AsVectorType()->YieldType()); } @@ -249,7 +255,8 @@ bool InputMgr::IsCompatibleType(BroType* t) } return false; - } +} + bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; diff --git a/src/InputMgr.h b/src/InputMgr.h index d4bfa5c355..93c6447467 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -46,7 +46,7 @@ protected: private: struct ReaderInfo; - bool IsCompatibleType(BroType* t); + bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); From fb5f26e7fcf5d3bda5c25472e2933e7ed975ddb1 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 15:23:46 -0800 Subject: [PATCH 023/149] make default values work (thanks to robin) --- src/InputMgr.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 4506501c94..a0a9d6a35c 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -97,7 +97,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) return 0; } - EnumVal* reader = description->Lookup(rtype->FieldOffset("reader"))->AsEnumVal(); + EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) @@ -148,7 +148,6 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - Val *want_record = description->Lookup(rtype->FieldOffset("want_record")); vector fieldsV; // vector, because we don't know the length beforehands @@ -163,9 +162,11 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) if ( status ) { reporter->Error("Problem unrolling"); + Unref(reader); return 0; } + Val *want_record = description->LookupWithDefault(rtype->FieldOffset("want_record")); LogField** fields = new LogField*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { @@ -174,7 +175,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; - info->type = reader->Ref()->AsEnumVal(); + info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault info->num_idx_fields = idxfields; info->num_val_fields = valfields; info->tab = dst->Ref()->AsTableVal(); @@ -184,6 +185,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); info->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { assert(info->want_record); From 821878835a800edb794c25c32a59ef67958a87da Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 16:32:35 -0800 Subject: [PATCH 024/149] read vector. still missing: enums, empty fields for optional parameters. --- src/InputMgr.cc | 45 +++++++++++++++++++++++++++------ src/InputReaderAscii.cc | 56 ++++++++++++++++++++++++++--------------- src/InputReaderAscii.h | 6 ++--- src/LogMgr.cc | 6 ++--- src/LogMgr.h | 6 ++--- 5 files changed, 83 insertions(+), 36 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index a0a9d6a35c..4c21268c84 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -244,12 +244,10 @@ bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) case TYPE_VECTOR: { - return false; // do me... - - //if ( atomic_only ) - // return false; - // - //return IsCompatibleType(t->AsVectorType()->YieldType()); + if ( atomic_only ) + return false; + + return IsCompatibleType(t->AsVectorType()->YieldType(), true); } default: @@ -342,7 +340,9 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); if ( field->type == TYPE_TABLE ) { - field->set_type = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + } else if ( field->type == TYPE_VECTOR ) { + field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); } fields->push_back(field); @@ -870,6 +870,13 @@ int InputMgr::GetLogValLength(const LogVal* val) { break; } + case TYPE_VECTOR: { + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + length += GetLogValLength(val->val.vector_val.vals[i]); + } + break; + } + default: reporter->InternalError("unsupported type %d for GetLogValLength", val->type); } @@ -936,6 +943,15 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { break; } + case TYPE_VECTOR: { + int length = 0; + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + length += CopyLogVal(data, startpos+length, val->val.vector_val.vals[i]); + } + return length; + break; + } + default: reporter->InternalError("unsupported type %d for CopyLogVal", val->type); return 0; @@ -1039,6 +1055,21 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { break; } + case TYPE_VECTOR: { + assert ( val->val.vector_val.size > 1 ); // implement empty vector... + + // all entries have to have the same type... + TypeTag type = val->val.vector_val.vals[0]->type; + VectorType* vt = new VectorType(base_type(type)); + VectorVal* v = new VectorVal(vt); + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + assert( val->val.vector_val.vals[i]->type == type); + v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return v; + + } + case TYPE_ENUM: reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 60a8c5685a..3b4409e652 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -11,20 +11,20 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int position = arg_position; } -FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position) - : name(arg_name), type(arg_type), set_type(arg_set_type) +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) + : name(arg_name), type(arg_type), subtype(arg_subtype) { position = arg_position; } FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type), set_type(arg.set_type) + : name(arg.name), type(arg.type), subtype(arg.subtype) { position = arg.position; } -FieldMapping FieldMapping::setType() { - return FieldMapping(name, set_type, position); +FieldMapping FieldMapping::subType() { + return FieldMapping(name, subtype, position); } InputReaderAscii::InputReaderAscii() @@ -91,7 +91,7 @@ bool InputReaderAscii::ReadHeader() { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position - FieldMapping f(field->name, field->type, field->set_type, i); + FieldMapping f(field->name, field->type, field->subtype, i); columnMap.push_back(f); wantFields++; break; // done with searching @@ -112,7 +112,7 @@ bool InputReaderAscii::ReadHeader() { if ( wantFields != (int) num_fields ) { // we did not find all fields? // :( - Error("One of the requested fields could not be found in the input data file"); + Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d", wantFields, num_fields)); return false; } @@ -199,25 +199,40 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; } - case TYPE_TABLE: { - // construct a table from entry... - // for the moment assume, that entries are split by ",". - - if ( s == "-" ) { - // empty - val->val.set_val.size = 0; - break; - } - + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { // how many entries do we have... unsigned int length = 1; for ( unsigned int i = 0; i < s.size(); i++ ) if ( s[i] == ',') length++; unsigned int pos = 0; + LogVal** lvals = new LogVal* [length]; - val->val.set_val.vals = lvals; - val->val.set_val.size = length; + + if ( field.type == TYPE_TABLE ) { + // construct a table from entry... + // for the moment assume, that entries are split by ",". + + /* Fix support for emtyp tables if ( s == "-" ) { + // empty + val->val.set_val.size = 0; + break; + } */ + + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } else if ( field.type == TYPE_VECTOR ) { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } else { + assert(false); + } istringstream splitstream(s); while ( splitstream ) { @@ -232,7 +247,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; - LogVal* newval = EntryToVal(element, field.setType()); + LogVal* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { Error("Error while reading set"); return 0; @@ -243,6 +258,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { } + if ( pos != length ) { Error("Internal error while parsing set: did not find all elements"); return 0; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index ab2b89339c..56c1001acb 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,15 +12,15 @@ struct FieldMapping { string name; TypeTag type; - TypeTag set_type; + TypeTag subtype; int position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); - FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); FieldMapping(const FieldMapping& arg); FieldMapping() { position = -1; } - FieldMapping setType(); + FieldMapping subType(); bool IsEmpty() { return position == -1; } }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 9818d9cdfb..6eaace3893 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -83,16 +83,16 @@ bool LogField::Read(SerializationFormat* fmt) int t; int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "set_type") ); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); type = (TypeTag) t; - set_type = (TypeTag) it; + subtype = (TypeTag) it; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)set_type, "set_type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); } LogVal::~LogVal() diff --git a/src/LogMgr.h b/src/LogMgr.h index 40dab8677b..b8530d29ab 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,12 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; - // needed by input framework. otherwise it cannot determine the inner type of a set. - TypeTag set_type; + // needed by input framework. otherwise it cannot determine the inner type of a set or vector. + TypeTag subtype; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type), set_type(other.set_type) { } + : name(other.name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); From ab68d8400789156a23b1e6bbaf57828e7eac7c83 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 22:13:36 -0800 Subject: [PATCH 025/149] reading of enum types (thanks, Seth) --- src/InputMgr.cc | 50 +++++++++++++++++++++++++++------------------- src/InputMgr.h | 4 ++-- src/InputReader.cc | 3 ++- src/InputReader.h | 2 +- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 4c21268c84..64f3e2eb9b 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -422,7 +422,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { - idxval = LogValToVal(vals[0]); + idxval = LogValToVal(vals[0], type->FieldType(0)); position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); @@ -430,7 +430,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { l->Append(LogValToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); } else { - l->Append(LogValToVal(vals[position], type->FieldType(j)->Tag())); + l->Append(LogValToVal(vals[position], type->FieldType(j))); position++; } } @@ -486,7 +486,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { int position = i->num_idx_fields; if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields]); + valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); } else { RecordVal * r = new RecordVal(i->rtype); @@ -501,7 +501,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + val = LogValToVal(vals[position], i->rtype->FieldType(j)); position++; } @@ -705,7 +705,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { int position = i->num_idx_fields; if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields]); + valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); } else { RecordVal * r = new RecordVal(i->rtype); @@ -715,7 +715,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + val = LogValToVal(vals[position], i->rtype->FieldType(j)); position++; } @@ -760,7 +760,7 @@ void InputMgr::Error(InputReader* reader, const char* msg) reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } - +/* Does not work atm, because LogValToVal needs BroType void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); @@ -775,7 +775,7 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c } mgr.Dispatch(new Event(handler, vl)); -} +} */ void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) { @@ -814,7 +814,7 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { fieldVal = LogValToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); } else { - fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)->Tag()); + fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)); (*position)++; } @@ -988,10 +988,10 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } -Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { +Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { - if ( request_type != TYPE_ANY && request_type != val->type ) { - reporter->InternalError("Typetags don't match: %d vs %d", request_type, val->type); + if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { + reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; } @@ -1041,13 +1041,12 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new TableVal(new SetType(set_index, 0)); } else { // all entries have to have the same type... - TypeTag type = val->val.set_val.vals[0]->type; - TypeList* set_index = new TypeList(base_type(type)); - set_index->Append(base_type(type)); + BroType* type = request_type->AsTableType()->Indices()->PureType(); + TypeList* set_index = new TypeList(type->Ref()); + set_index->Append(type->Ref()); SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); for ( int i = 0; i < val->val.set_val.size; i++ ) { - assert( val->val.set_val.vals[i]->type == type); t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); } return t; @@ -1059,19 +1058,28 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { assert ( val->val.vector_val.size > 1 ); // implement empty vector... // all entries have to have the same type... - TypeTag type = val->val.vector_val.vals[0]->type; - VectorType* vt = new VectorType(base_type(type)); + BroType* type = request_type->AsVectorType()->YieldType(); + VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); for ( int i = 0; i < val->val.vector_val.size; i++ ) { - assert( val->val.vector_val.vals[i]->type == type); v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); } return v; } - case TYPE_ENUM: - reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); + case TYPE_ENUM: { + // well, this is kind of stupid, because EnumType just mangles the module name and the var name together again... + // but well + string module = extract_module_name(val->val.string_val->c_str()); + string var = extract_var_name(val->val.string_val->c_str()); + bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str()); + if ( index == -1 ) { + reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", module.c_str(), var.c_str()); + } + return new EnumVal(index, request_type->Ref()->AsEnumType() ); + break; + } default: diff --git a/src/InputMgr.h b/src/InputMgr.h index 93c6447467..17b7e2e804 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -55,11 +55,11 @@ private: int GetLogValLength(const LogVal* val); int CopyLogVal(char *data, const int startpos, const LogVal* val); - Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); + Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReader.cc b/src/InputReader.cc index 994f8b9b97..7403d1f989 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -64,10 +64,11 @@ bool InputReader::Update() return DoUpdate(); } +/* void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { input_mgr->SendEvent(name, num_vals, vals); -} +} */ // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) diff --git a/src/InputReader.h b/src/InputReader.h index b547d29506..41000e4c0c 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -40,7 +40,7 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); void Put(const LogVal* const *val); void Clear(); From 4fef1e3f8c5fe7005f23f91d6001cd1d79ffcdd2 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 22:47:28 -0800 Subject: [PATCH 026/149] set & entry separator configuration (with the restriction that they have to be exactly one character long) --- .../base/frameworks/input/readers/ascii.bro | 19 +++++++++ src/InputReaderAscii.cc | 40 +++++++++++++++++-- src/InputReaderAscii.h | 15 ++++++- src/input.bif | 9 +++++ 4 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 scripts/base/frameworks/input/readers/ascii.bro diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro new file mode 100644 index 0000000000..9f630975a2 --- /dev/null +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -0,0 +1,19 @@ +##! Interface for the ascii input reader. + +module InputAscii; + +export { + ## Separator between fields. + ## Please note that the separator has to be exactly one character long + const separator = "\t" &redef; + + ## Separator between set elements. + ## Please note that the separator has to be exactly one character long + const set_separator = "," &redef; + + ## String to use for empty fields. + const empty_field = "-" &redef; + + ## String to use for an unset &optional field. + const unset_field = "-" &redef; +} diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 3b4409e652..22cdcfdcf0 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -2,6 +2,7 @@ #include "InputReaderAscii.h" #include "DebugLogger.h" +#include "NetVar.h" #include @@ -29,15 +30,46 @@ FieldMapping FieldMapping::subType() { InputReaderAscii::InputReaderAscii() { - //DBG_LOG(DBG_LOGGING, "input reader initialized"); file = 0; //keyMap = new map(); + + separator_len = BifConst::LogAscii::separator->Len(); + separator = new char[separator_len]; + memcpy(separator, BifConst::LogAscii::separator->Bytes(), + separator_len); + if ( separator_len != 1 ) { + Error("separator length has to be 1. Separator will be truncated."); + } + + set_separator_len = BifConst::LogAscii::set_separator->Len(); + set_separator = new char[set_separator_len]; + memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), + set_separator_len); + if ( set_separator_len != 1 ) { + Error("set_separator length has to be 1. Separator will be truncated."); + } + + empty_field_len = BifConst::LogAscii::empty_field->Len(); + empty_field = new char[empty_field_len]; + memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), + empty_field_len); + + unset_field_len = BifConst::LogAscii::unset_field->Len(); + unset_field = new char[unset_field_len]; + memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), + unset_field_len); + } InputReaderAscii::~InputReaderAscii() { DoFinish(); + + delete [] separator; + delete [] set_separator; + delete [] empty_field; + delete [] unset_field; } void InputReaderAscii::DoFinish() @@ -83,7 +115,7 @@ bool InputReaderAscii::ReadHeader() { int wantFields = 0; while ( splitstream ) { string s; - if ( !getline(splitstream, s, '\t')) + if ( !getline(splitstream, s, separator[0])) break; // current found heading in s... compare if we want it @@ -243,7 +275,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; } - if ( !getline(splitstream, element, ',') ) + if ( !getline(splitstream, element, set_separator[0]) ) break; @@ -322,7 +354,7 @@ bool InputReaderAscii::DoUpdate() { while ( splitstream ) { string s; - if ( !getline(splitstream, s, '\t') ) + if ( !getline(splitstream, s, separator[0]) ) break; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 56c1001acb..d69b8c04bc 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -57,7 +57,20 @@ private: const LogField* const * fields; // raw mapping //map *keyMap; - + // + // Options set from the script-level. + char* separator; + int separator_len; + + char* set_separator; + int set_separator_len; + + char* empty_field; + int empty_field_len; + + char* unset_field; + int unset_field_len; + }; diff --git a/src/input.bif b/src/input.bif index 7b051fba16..aaef25dcc3 100644 --- a/src/input.bif +++ b/src/input.bif @@ -52,3 +52,12 @@ function Input::__remove_filter%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL); %} +# Options for Ascii Reader + +module InputAscii; + +const separator: string; +const set_separator: string; +const empty_field: string; +const unset_field: string; + From 4dd95fcf3cf1efc83ec4d2c53fee765d2a800674 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 23:51:51 -0800 Subject: [PATCH 027/149] support for uninitialized fields & empty sets and tables. The only snag is... with the default output format of the log-file writer, the input reader cannot tell if a table or set is empty or uninitialized (both cases use the same character by default). In this case, by default it is assumed that the field/vector is uninitalized. --- scripts/base/frameworks/input/__load__.bro | 2 + src/InputMgr.cc | 61 ++++++++++------------ src/InputReaderAscii.cc | 61 ++++++++-------------- src/InputReaderAscii.h | 12 ++--- 4 files changed, 57 insertions(+), 79 deletions(-) diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro index a10fe855df..a3315186d5 100644 --- a/scripts/base/frameworks/input/__load__.bro +++ b/scripts/base/frameworks/input/__load__.bro @@ -1 +1,3 @@ @load ./main +@load ./readers/ascii + diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 64f3e2eb9b..9c0e9c12b5 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -505,10 +505,10 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { position++; } - if ( val == 0 ) { + /* if ( val == 0 ) { reporter->InternalError("conversion error"); return; - } + } */ r->Assign(j,val); @@ -871,7 +871,9 @@ int InputMgr::GetLogValLength(const LogVal* val) { } case TYPE_VECTOR: { - for ( int i = 0; i < val->val.vector_val.size; i++ ) { + int j = val->val.vector_val.size; + for ( int i = 0; i < j; i++ ) { + reporter->Error("size is %d", val->val.vector_val.size); length += GetLogValLength(val->val.vector_val.vals[i]); } break; @@ -945,7 +947,8 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { case TYPE_VECTOR: { int length = 0; - for ( int i = 0; i < val->val.vector_val.size; i++ ) { + int j = val->val.vector_val.size; + for ( int i = 0; i < j; i++ ) { length += CopyLogVal(data, startpos+length, val->val.vector_val.vals[i]); } return length; @@ -994,6 +997,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; } + + if ( !val->present ) { + return 0; // unset field + } switch ( val->type ) { @@ -1033,38 +1040,28 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { break; case TYPE_TABLE: { - if ( val->val.set_val.size == 0 ) { - // empty table - TypeList* set_index = new TypeList(base_type(TYPE_ANY)); - // iim quite sure this does not work... we probably need the internal set type for this... - reporter->InternalError("Implement me."); - return new TableVal(new SetType(set_index, 0)); - } else { - // all entries have to have the same type... - BroType* type = request_type->AsTableType()->Indices()->PureType(); - TypeList* set_index = new TypeList(type->Ref()); - set_index->Append(type->Ref()); - SetType* s = new SetType(set_index, 0); - TableVal* t = new TableVal(s); - for ( int i = 0; i < val->val.set_val.size; i++ ) { - t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); - } - return t; - } + // all entries have to have the same type... + BroType* type = request_type->AsTableType()->Indices()->PureType(); + TypeList* set_index = new TypeList(type->Ref()); + set_index->Append(type->Ref()); + SetType* s = new SetType(set_index, 0); + TableVal* t = new TableVal(s); + for ( int i = 0; i < val->val.set_val.size; i++ ) { + t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return t; break; } case TYPE_VECTOR: { - assert ( val->val.vector_val.size > 1 ); // implement empty vector... - - // all entries have to have the same type... - BroType* type = request_type->AsVectorType()->YieldType(); - VectorType* vt = new VectorType(type->Ref()); - VectorVal* v = new VectorVal(vt); - for ( int i = 0; i < val->val.vector_val.size; i++ ) { - v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); - } - return v; + // all entries have to have the same type... + BroType* type = request_type->AsVectorType()->YieldType(); + VectorType* vt = new VectorType(type->Ref()); + VectorVal* v = new VectorVal(vt); + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return v; } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 22cdcfdcf0..4a0d4157bc 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -34,31 +34,19 @@ InputReaderAscii::InputReaderAscii() //keyMap = new map(); - separator_len = BifConst::LogAscii::separator->Len(); - separator = new char[separator_len]; - memcpy(separator, BifConst::LogAscii::separator->Bytes(), - separator_len); - if ( separator_len != 1 ) { + separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), BifConst::InputAscii::separator->Len()); + if ( separator.size() != 1 ) { Error("separator length has to be 1. Separator will be truncated."); } - set_separator_len = BifConst::LogAscii::set_separator->Len(); - set_separator = new char[set_separator_len]; - memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), - set_separator_len); - if ( set_separator_len != 1 ) { + set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), BifConst::InputAscii::set_separator->Len()); + if ( set_separator.size() != 1 ) { Error("set_separator length has to be 1. Separator will be truncated."); } - empty_field_len = BifConst::LogAscii::empty_field->Len(); - empty_field = new char[empty_field_len]; - memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), - empty_field_len); - - unset_field_len = BifConst::LogAscii::unset_field->Len(); - unset_field = new char[unset_field_len]; - memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), - unset_field_len); + empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), BifConst::InputAscii::empty_field->Len()); + + unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); } @@ -66,10 +54,6 @@ InputReaderAscii::~InputReaderAscii() { DoFinish(); - delete [] separator; - delete [] set_separator; - delete [] empty_field; - delete [] unset_field; } void InputReaderAscii::DoFinish() @@ -172,7 +156,10 @@ bool InputReaderAscii::GetLine(string& str) { LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { LogVal* val = new LogVal(field.type, true); - //bzero(val, sizeof(LogVal)); + + if ( s.compare(unset_field) == 0 ) { // field is not set... + return new LogVal(field.type, false); + } switch ( field.type ) { case TYPE_ENUM: @@ -244,19 +231,13 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { if ( s[i] == ',') length++; unsigned int pos = 0; + + if ( s.compare(empty_field) == 0 ) + length = 0; LogVal** lvals = new LogVal* [length]; if ( field.type == TYPE_TABLE ) { - // construct a table from entry... - // for the moment assume, that entries are split by ",". - - /* Fix support for emtyp tables if ( s == "-" ) { - // empty - val->val.set_val.size = 0; - break; - } */ - val->val.set_val.vals = lvals; val->val.set_val.size = length; } else if ( field.type == TYPE_VECTOR ) { @@ -266,18 +247,20 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { assert(false); } + if ( length == 0 ) + break; //empty + istringstream splitstream(s); while ( splitstream ) { string element; - if ( pos >= length ) { - Error(Fmt("Internal error while parsing set. pos %d > length %d", pos, length)); - break; - } - if ( !getline(splitstream, element, set_separator[0]) ) break; - + + if ( pos >= length ) { + Error(Fmt("Internal error while parsing set. pos %d >= length %d. Element: %s", pos, length, element.c_str())); + break; + } LogVal* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index d69b8c04bc..c848c17110 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -59,17 +59,13 @@ private: //map *keyMap; // // Options set from the script-level. - char* separator; - int separator_len; + string separator; - char* set_separator; - int set_separator_len; + string set_separator; - char* empty_field; - int empty_field_len; + string empty_field; - char* unset_field; - int unset_field_len; + string unset_field; }; From e2c521fc4e6ef9eb66eb6cd431f9186468b14b70 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 18 Nov 2011 10:49:20 -0800 Subject: [PATCH 028/149] start reworking input framework... does not compile at the moment, but there are a few uncommitted changes that will be reverted in the next commit. --- scripts/base/frameworks/input/main.bro | 46 ++--- src/InputMgr.cc | 263 ++++++++++++------------- src/InputMgr.h | 18 +- src/InputReader.cc | 39 ++-- src/InputReader.h | 37 ++-- src/input.bif | 22 +-- 6 files changed, 208 insertions(+), 217 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 2b87ac980c..d9c0812498 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,30 +4,36 @@ module Input; export { const default_reader = READER_ASCII &redef; - type ReaderDescription: record { + type StreamDescription: record { source: string; - idx: any; - val: any; - destination: any; - want_record: bool &default=T; reader: Reader &default=default_reader; }; type Filter: record { - name: string; ## descriptive name. for later removal + name: string; + ## for tables + idx: any &optional; + val: any &optional; + destination: any &optional; + want_record: bool &default=T; + table_ev: any &optional; # event containing idx, val as values. + + ## decision function, that decides if an insertion, update or removal should really be executed. + ## or events should be thought pred: function(typ: Input::Event, left: any, right: any): bool &optional; - ## decision function, that decides if an inserton, update or removal should really be executed + + ## for "normalized" events + ev: any &optional; + ev_description: any &optional; }; const no_filter: Filter = [$name=""]; # Sentinel. - global create_reader: function(id: Log::ID, description: Input::ReaderDescription) : bool; - global remove_reader: function(id: Log::ID) : bool; + global create_stream: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; - global add_event: function(id: Log::ID, name: string) : bool; - global remove_event: function(id: Log::ID, name: string) : bool; global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; global remove_filter: function(id: Log::ID, name: string) : bool; global get_filter: function(id: ID, name: string) : Filter; @@ -41,14 +47,14 @@ module Input; global filters: table[ID, string] of Filter; -function create_reader(id: Log::ID, description: Input::ReaderDescription) : bool +function create_stream(id: Log::ID, description: Input::ReaderDescription) : bool { - return __create_reader(id, description); + return __create_stream(id, description); } -function remove_reader(id: Log::ID) : bool +function remove_stream(id: Log::ID) : bool { - return __remove_reader(id); + return __remove_stream(id); } function force_update(id: Log::ID) : bool @@ -56,16 +62,6 @@ function force_update(id: Log::ID) : bool return __force_update(id); } -function add_event(id: Log::ID, name: string) : bool - { - return __add_event(id, name); - } - -function remove_event(id: Log::ID, name: string) : bool - { - return __remove_event(id, name); - } - function add_filter(id: Log::ID, filter: Input::Filter) : bool { filters[id, filter$name] = filter; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 9c0e9c12b5..f9250f6f0f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -27,22 +27,11 @@ declare(PDict, InputHash); struct InputMgr::Filter { EnumVal* id; string name; - Func* pred; - ~Filter(); -}; - -InputMgr::Filter::~Filter() { - Unref(id); -} - -struct InputMgr::ReaderInfo { - EnumVal* id; - EnumVal* type; - InputReader* reader; unsigned int num_idx_fields; unsigned int num_val_fields; bool want_record; + EventHandlerPtr table_event; TableVal* tab; RecordType* rtype; @@ -50,18 +39,42 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; - - list events; // events we fire when "something" happens - list filters; // filters that can prevent our actions + + Func* pred; + + EventHandlerPtr event; + RecordType* event_type; + + ~Filter(); +}; + +InputMgr::Filter::~Filter() { + Unref(id); + if ( tab ) + Unref(tab); + if ( itype ) + Unref(itype); + if ( rtype ) + Unref(rtype); + if ( event_type) + Unref(event_type); +} + +struct InputMgr::ReaderInfo { + EnumVal* id; + EnumVal* type; + InputReader* reader; + + //list events; // events we fire when "something" happens + map filters; // filters that can prevent our actions ~ReaderInfo(); }; InputMgr::ReaderInfo::~ReaderInfo() { + // all the contents of filters should delete themselves automatically... + Unref(type); - Unref(tab); - Unref(itype); - Unref(rtype); Unref(id); delete(reader); @@ -86,14 +99,14 @@ InputMgr::InputMgr() } // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) +InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) { InputReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::ReaderDescription, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) { - reporter->Error("readerDescription argument not of right type"); + reporter->Error("Streamdescription argument not of right type"); return 0; } @@ -145,55 +158,15 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); - RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); - TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - - - vector fieldsV; // vector, because we don't know the length beforehands - - - bool status = !UnrollRecordType(&fieldsV, idx, ""); - - int idxfields = fieldsV.size(); - - status = status || !UnrollRecordType(&fieldsV, val, ""); - int valfields = fieldsV.size() - idxfields; - - if ( status ) { - reporter->Error("Problem unrolling"); - Unref(reader); - return 0; - } - - Val *want_record = description->LookupWithDefault(rtype->FieldOffset("want_record")); - - LogField** fields = new LogField*[fieldsV.size()]; - for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { - fields[i] = fieldsV[i]; - } ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault - info->num_idx_fields = idxfields; - info->num_val_fields = valfields; - info->tab = dst->Ref()->AsTableVal(); - info->rtype = val->Ref()->AsRecordType(); info->id = id->Ref()->AsEnumVal(); - info->itype = idx->Ref()->AsRecordType(); - info->currDict = new PDict(InputHash); - info->lastDict = new PDict(InputHash); - info->want_record = ( want_record->InternalInt() == 1 ); - Unref(want_record); // ref'd by lookupwithdefault - - if ( valfields > 1 ) { - assert(info->want_record); - } readers.push_back(info); - int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); + int success = reader_obj->Init(source); if ( success == false ) { assert( RemoveReader(id) ); return 0; @@ -208,6 +181,86 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) } +bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Stream not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + + RecordType *idx = fval->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + + vector fieldsV; // vector, because we don't know the length beforehands + + bool status = !UnrollRecordType(&fieldsV, idx, ""); + + int idxfields = fieldsV.size(); + + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + + if ( status ) { + reporter->Error("Problem unrolling"); + return false; + } + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + LogField** fields = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + fields[i] = fieldsV[i]; + } + + Filter filter; + filter.name = name->AsString()->CheckString(); + filter.id = id->Ref()->AsEnumVal(); + filter.pred = pred ? pred->AsFunc() : 0; + filter.num_idx_fields = idxfields; + filter.num_val_fields = valfields; + filter.tab = dst ? dst->Ref()->AsTableVal() : 0; + filter.rtype = rtype ? val->Ref()->AsRecordType() : 0; + filter.itype = itype ? idx->Ref()->AsRecordType() : 0; + // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted + filter.currDict = new PDict(InputHash); + filter.lastDict = new PDict(InputHash); + filter.want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + + if ( valfields > 1 ) { + assert(info->want_record); + } + + i->filters[id->InternalInt()] = filter; + + // ok, now we have to alert the reader of our new filter with our funky new fields + // the id is handled in a ... well, to be honest, a little bit sneaky manner. + // the "problem" is, that we can have several filters in the reader for one filter in the log manager. + // this is due to the fact, that a filter can either output it's result as a table, as an event... + // ...or as an table _and_ an event. And... if we have a table and an event, we actually need two different sets + // of filters in the reader, because the fields for the table and the event may differ and I absolutely do not want + // to build a union of these values and figure it out later. + // hence -> filter id is multiplicated with 2. + // filterId*2 -> results for table + // filterId*2+1 -> results for event + i->AddFilter( id->InternalInt() * 2, fieldsV.size(), idxfields, fields ); + + return true; +} + + bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) { if ( ! t ) @@ -258,7 +311,7 @@ bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) } -bool InputMgr::RemoveReader(const EnumVal* id) { +bool InputMgr::RemoveStream(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -281,42 +334,6 @@ bool InputMgr::RemoveReader(const EnumVal* id) { return true; } -bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->InternalError("Reader not found"); - return false; - } - - i->events.push_back(eventName); - - return true; -} - -// remove first event with name eventName -// (though there shouldn't really be several events with the same name... -bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->InternalError("Reader not found"); - return false; - } - - std::list::iterator it = i->events.begin(); - while ( it != i->events.end() ) - { - if ( *it == eventName ) { - it = i->events.erase(it); - return true; - } - else - ++it; - } - - return false; -} - - bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { @@ -363,34 +380,6 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } -bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) - { - reporter->Error("filter argument not of right type"); - return false; - } - - - Val* name = fval->Lookup(rtype->FieldOffset("name")); - Val* pred = fval->Lookup(rtype->FieldOffset("pred")); - - Filter filter; - filter.name = name->AsString()->CheckString(); - filter.id = id->Ref()->AsEnumVal(); - filter.pred = pred ? pred->AsFunc() : 0; - - i->filters.push_back(filter); - - return true; -} - bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -398,7 +387,7 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } - +/* std::list::iterator it = i->filters.begin(); while ( it != i->filters.end() ) { @@ -410,8 +399,15 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { else ++it; } + */ - return false;; + map::iterator it = i->filters.find(id->InternalInt()); + if ( it == i->filters.end() ) { + return false; + } + + it->filters.erase(it); + return true; } @@ -444,7 +440,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo } -void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { +void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -605,7 +601,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } -void InputMgr::EndCurrentSend(const InputReader* reader) { +void InputMgr::EndCurrentSend(const InputReader* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -693,7 +689,7 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { i->currDict = new PDict(InputHash); } -void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { +void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -733,7 +729,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { i->tab->Assign(idxval, valval); } -void InputMgr::Clear(const InputReader* reader) { +void InputMgr::Clear(const InputReader* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -873,7 +869,6 @@ int InputMgr::GetLogValLength(const LogVal* val) { case TYPE_VECTOR: { int j = val->val.vector_val.size; for ( int i = 0; i < j; i++ ) { - reporter->Error("size is %d", val->val.vector_val.size); length += GetLogValLength(val->val.vector_val.vals[i]); } break; diff --git a/src/InputMgr.h b/src/InputMgr.h index 17b7e2e804..1cacf89143 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -20,11 +20,9 @@ class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* id, RecordVal* description); + InputReader* CreateStream(EnumVal* id, RecordVal* description); bool ForceUpdate(const EnumVal* id); - bool RemoveReader(const EnumVal* id); - bool RegisterEvent(const EnumVal* id, string eventName); - bool UnregisterEvent(const EnumVal* id, string eventName); + bool RemoveStream(const EnumVal* id); bool AddFilter(EnumVal *id, RecordVal* filter); bool RemoveFilter(EnumVal* id, const string &name); @@ -36,12 +34,14 @@ protected: // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); - void Put(const InputReader* reader, const LogVal* const *vals); - void Clear(const InputReader* reader); - bool Delete(const InputReader* reader, const LogVal* const *vals); + // for readers to write to input stream in direct mode (reporting new/deleted values directly) + void Put(const InputReader* reader, int id. const LogVal* const *vals); + void Clear(const InputReader* reader, int id); + bool Delete(const InputReader* reader, int id, const LogVal* const *vals); - void SendEntry(const InputReader* reader, const LogVal* const *vals); - void EndCurrentSend(const InputReader* reader); + // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) + void SendEntry(const InputReader* reader, int id, const LogVal* const *vals); + void EndCurrentSend(const InputReader* reader, int id); private: struct ReaderInfo; diff --git a/src/InputReader.cc b/src/InputReader.cc index 7403d1f989..1008cf1b67 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -24,35 +24,42 @@ void InputReader::Error(const string &msg) input_mgr->Error(this, msg.c_str()); } -void InputReader::Put(const LogVal* const *val) +void InputReader::Put(int id, const LogVal* const *val) { - input_mgr->Put(this, val); + input_mgr->Put(this, int id, val); } -void InputReader::Clear() +void InputReader::Clear(int id) { - input_mgr->Clear(this); + input_mgr->Clear(this, int id); } -void InputReader::Delete(const LogVal* const *val) +void InputReader::Delete(int id, const LogVal* const *val) { - input_mgr->Delete(this, val); + input_mgr->Delete(this, int id, val); } -bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields, - const LogField* const * arg_fields) +bool InputReader::Init(string arg_source) { source = arg_source; - num_fields = arg_num_fields; - index_fields = arg_idx_fields; - fields = arg_fields; // disable if DoInit returns error. - disabled = !DoInit(arg_source, arg_num_fields, arg_idx_fields, arg_fields); + disabled = !DoInit(arg_source); return !disabled; } +bool InputReader::AddFilter(int id, int arg_num_fields, + const LogField* const * arg_fields) +{ + return DoAddFilter(int id, arg_num_fields, arg_fields); +} + +bool InputReader::RemoveFilter(int id) +{ + return DoRemoveFilter(int id); +} + void InputReader::Finish() { DoFinish(); @@ -96,12 +103,12 @@ const char* InputReader::Fmt(const char* format, ...) } -void InputReader::SendEntry(const LogVal* const *vals) +void InputReader::SendEntry(int id, const LogVal* const *vals) { - input_mgr->SendEntry(this, vals); + input_mgr->SendEntry(this, int id, vals); } -void InputReader::EndCurrentSend() +void InputReader::EndCurrentSend(int id) { - input_mgr->EndCurrentSend(this); + input_mgr->EndCurrentSend(this, int id); } diff --git a/src/InputReader.h b/src/InputReader.h index 41000e4c0c..12f0bc9db4 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -15,7 +15,11 @@ public: InputReader(); virtual ~InputReader(); - bool Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const* fields); + bool Init(string arg_source); + + bool AddFilter( int id, int arg_num_fields, const LogField* const* fields ); + + bool RemoveFilter ( int id ); void Finish(); @@ -23,8 +27,11 @@ public: protected: // Methods that have to be overwritten by the individual readers - virtual bool DoInit(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * fields) = 0; - + virtual bool DoInit(string arg_sources) = 0; + + virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) = 0; + virtual bool DoRemoveFilter( int id ); + virtual void DoFinish() = 0; // update file contents to logmgr @@ -42,28 +49,26 @@ protected: //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); - void Put(const LogVal* const *val); - void Clear(); - void Delete(const LogVal* const *val); + // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table + void Put(int id, const LogVal* const *val); + void Delete(int id, const LogVal* const *val); + void Clear(int id); - void SendEntry(const LogVal* const *vals); - void EndCurrentSend(); + // Table-functions (tracking mode): Only changed lines are propagated. + void SendEntry(int id, const LogVal* const *vals); + void EndCurrentSend(int id); private: friend class InputMgr; string source; - int num_fields; - int index_fields; - const LogField* const * fields; - // When an error occurs, this method is called to set a flag marking the - // writer as disabled. + // When an error occurs, this method is called to set a flag marking the + // writer as disabled. - bool disabled; - - bool Disabled() { return disabled; } + bool disabled; + bool Disabled() { return disabled; } // For implementing Fmt(). char* buf; diff --git a/src/input.bif b/src/input.bif index aaef25dcc3..ef069316ab 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,18 +7,18 @@ module Input; #include "NetVar.h" %%} -type ReaderDescription: record; +type StreamDescription: record; type Filter: record; -function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescription%) : bool +function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__remove_reader%(id: Log::ID%) : bool +function Input::__remove_stream%(id: Log::ID%) : bool %{ - bool res = input_mgr->RemoveReader(id->AsEnumVal()); + bool res = input_mgr->RemoveStream(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} @@ -28,18 +28,6 @@ function Input::__force_update%(id: Log::ID%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__add_event%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL ); - %} - -function Input::__remove_event%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL ); - %} - function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool %{ bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); From b3f01915fbca99ddda434ae791b143150d27fcf7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 12:07:50 -0800 Subject: [PATCH 029/149] compiles with basic new filter framework - but crashes on use. --- scripts/base/frameworks/input/main.bro | 16 +- src/InputMgr.cc | 255 +++++++++++------------- src/InputMgr.h | 2 +- src/InputReader.cc | 14 +- src/InputReader.h | 3 +- src/InputReaderAscii.cc | 265 ++++++++++++------------- src/InputReaderAscii.h | 30 ++- 7 files changed, 283 insertions(+), 302 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index d9c0812498..9d83d73ec6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -14,9 +14,9 @@ export { name: string; ## for tables - idx: any &optional; - val: any &optional; - destination: any &optional; + idx: any; + val: any; + destination: any; want_record: bool &default=T; table_ev: any &optional; # event containing idx, val as values. @@ -25,13 +25,13 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## for "normalized" events - ev: any &optional; - ev_description: any &optional; + # ev: any &optional; + # ev_description: any &optional; }; - const no_filter: Filter = [$name=""]; # Sentinel. + const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - global create_stream: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; @@ -47,7 +47,7 @@ module Input; global filters: table[ID, string] of Filter; -function create_stream(id: Log::ID, description: Input::ReaderDescription) : bool +function create_stream(id: Log::ID, description: Input::StreamDescription) : bool { return __create_stream(id, description); } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index f9250f6f0f..403d656140 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -16,18 +16,20 @@ #include "CompHash.h" -class InputHash { -public: +struct InputHash { HashKey* valhash; HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. }; declare(PDict, InputHash); -struct InputMgr::Filter { +class InputMgr::Filter { +public: EnumVal* id; string name; + //int filter_type; // to distinguish between event and table filters + unsigned int num_idx_fields; unsigned int num_val_fields; bool want_record; @@ -68,6 +70,8 @@ struct InputMgr::ReaderInfo { //list events; // events we fire when "something" happens map filters; // filters that can prevent our actions + bool HasFilter(int id); + ~ReaderInfo(); }; @@ -80,6 +84,15 @@ InputMgr::ReaderInfo::~ReaderInfo() { delete(reader); } +bool InputMgr::ReaderInfo::HasFilter(int id) { + map::iterator it = filters.find(id); + if ( it == filters.end() ) { + return false; + } + return true; +} + + struct InputReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -168,12 +181,12 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) int success = reader_obj->Init(source); if ( success == false ) { - assert( RemoveReader(id) ); + assert( RemoveStream(id) ); return 0; } success = reader_obj->Update(); if ( success == false ) { - assert ( RemoveReader(id) ); + assert ( RemoveStream(id) ); return 0; } @@ -224,6 +237,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } + // FIXME: remove those funky 0-tests again as the idea was changed. Filter filter; filter.name = name->AsString()->CheckString(); filter.id = id->Ref()->AsEnumVal(); @@ -231,8 +245,8 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { filter.num_idx_fields = idxfields; filter.num_val_fields = valfields; filter.tab = dst ? dst->Ref()->AsTableVal() : 0; - filter.rtype = rtype ? val->Ref()->AsRecordType() : 0; - filter.itype = itype ? idx->Ref()->AsRecordType() : 0; + filter.rtype = val ? val->Ref()->AsRecordType() : 0; + filter.itype = idx ? idx->Ref()->AsRecordType() : 0; // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted filter.currDict = new PDict(InputHash); filter.lastDict = new PDict(InputHash); @@ -240,22 +254,11 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { - assert(info->want_record); + assert(filter.want_record); } i->filters[id->InternalInt()] = filter; - - // ok, now we have to alert the reader of our new filter with our funky new fields - // the id is handled in a ... well, to be honest, a little bit sneaky manner. - // the "problem" is, that we can have several filters in the reader for one filter in the log manager. - // this is due to the fact, that a filter can either output it's result as a table, as an event... - // ...or as an table _and_ an event. And... if we have a table and an event, we actually need two different sets - // of filters in the reader, because the fields for the table and the event may differ and I absolutely do not want - // to build a union of these values and figure it out later. - // hence -> filter id is multiplicated with 2. - // filterId*2 -> results for table - // filterId*2+1 -> results for event - i->AddFilter( id->InternalInt() * 2, fieldsV.size(), idxfields, fields ); + i->reader->AddFilter( id->InternalInt(), fieldsV.size(), fields ); return true; } @@ -387,31 +390,15 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } -/* - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) - { - if ( (*it).name == name ) { - it = i->filters.erase(it); - return true; - break; - } - else - ++it; - } - */ - map::iterator it = i->filters.find(id->InternalInt()); if ( it == i->filters.end() ) { return false; } - it->filters.erase(it); + i->filters.erase(it); return true; } - - Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { Val* idxval; int position = 0; @@ -449,27 +436,28 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const bool updated = false; + assert(i->HasFilter(id)); //reporter->Error("Hashing %d index fields", i->num_idx_fields); - HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + HashKey* idxhash = HashLogVals(i->filters[id].num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + HashKey* valhash = HashLogVals(i->filters[id].num_val_fields, vals+i->filters[id].num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = i->lastDict->Lookup(idxhash); + InputHash *h = i->filters[id].lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before if ( h->valhash->Hash() == valhash->Hash() ) { // ok, double. - i->lastDict->Remove(idxhash); - i->currDict->Insert(idxhash, h); + i->filters[id].lastDict->Remove(idxhash); + i->filters[id].currDict->Insert(idxhash, h); return; } else { // updated - i->lastDict->Remove(idxhash); + i->filters[id].lastDict->Remove(idxhash); delete(h); updated = true; @@ -477,27 +465,22 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const } - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); Val* valval; - int position = i->num_idx_fields; - if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); + int position = i->filters[id].num_idx_fields; + if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { + valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->rtype); + RecordVal * r = new RecordVal(i->filters[id].rtype); - /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { - reporter->InternalError("Type mismatch"); - return; - } */ - - for ( int j = 0; j < i->rtype->NumFields(); j++) { + for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { Val* val = 0; - if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)); + val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); position++; } @@ -516,17 +499,12 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const Val* oldval = 0; if ( updated == true ) { // in that case, we need the old value to send the event (if we send an event). - oldval = i->tab->Lookup(idxval); + oldval = i->filters[id].tab->Lookup(idxval); } - // call filters first do determine if we really add / change the entry - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) { - if (! (*it).pred ) { - continue; - } - + // call filter first to determine if we really add / change the entry + if ( i->filters[id].pred ) { EnumVal* ev; Ref(idxval); Ref(valval); @@ -541,44 +519,45 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const vl.append(ev); vl.append(idxval); vl.append(valval); - Val* v = (*it).pred->Call(&vl); + Val* v = i->filters[id].pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... - delete(i->currDict->RemoveEntry(idxhash)); + delete(i->filters[id].currDict->RemoveEntry(idxhash)); return; } else { // keep old one - i->currDict->Insert(idxhash, h); + i->filters[id].currDict->Insert(idxhash, h); return; } } - ++it; } //i->tab->Assign(idxval, valval); - HashKey* k = i->tab->ComputeHash(idxval); + HashKey* k = i->filters[id].tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); return; } - i->tab->Assign(idxval, k, valval); + i->filters[id].tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); - k = i->tab->ComputeHash(idxval); + k = i->filters[id].tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; //i->tab->Delete(k); - i->currDict->Insert(idxhash, ih); + i->filters[id].currDict->Insert(idxhash, ih); // send events now that we are kind of finished. + + /* FIXME: fix me. std::list::iterator filter_iterator = i->events.begin(); while ( filter_iterator != i->events.end() ) { EnumVal* ev; @@ -597,7 +576,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ++filter_iterator; - } + } */ } @@ -607,86 +586,74 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { reporter->InternalError("Unknown reader"); return; } + + assert(i->HasFilter(id)); + // lastdict contains all deleted entries and should be empty apart from that - IterCookie *c = i->lastDict->InitForIteration(); - i->lastDict->MakeRobustCookie(c); + IterCookie *c = i->filters[id].lastDict->InitForIteration(); + i->filters[id].lastDict->MakeRobustCookie(c); InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = i->lastDict->NextEntry(lastDictIdxKey, c) ) ) { - - if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event - { + while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { - ListVal *idx = i->tab->RecoverIndex(ih->idxkey); + if ( i->filters[id].pred ) { + ListVal *idx = i->filters[id].tab->RecoverIndex(ih->idxkey); assert(idx != 0); - Val *val = i->tab->Lookup(idx); + Val *val = i->filters[id].tab->Lookup(idx); assert(val != 0); - { - bool doBreak = false; - // ask filter, if we want to expire this element... - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) { - if (! (*it).pred ) { - continue; - } + bool doBreak = false; + // ask predicate, if we want to expire this element... - EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - Ref(idx); - Ref(val); + EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + Ref(idx); + Ref(val); - val_list vl(3); - vl.append(ev); - vl.append(idx); - vl.append(val); - Val* v = (*it).pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - - ++it; - - if ( result == false ) { - // Keep it. Hence - we quit and simply go to the next entry of lastDict - // ah well - and we have to add the entry to currDict... - i->currDict->Insert(lastDictIdxKey, i->lastDict->RemoveEntry(lastDictIdxKey)); - doBreak = true; - continue; - } - - } - - if ( doBreak ) { - continue; - } + val_list vl(3); + vl.append(ev); + vl.append(idx); + vl.append(val); + Val* v = i->filters[id].pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // Keep it. Hence - we quit and simply go to the next entry of lastDict + // ah well - and we have to add the entry to currDict... + i->filters[id].currDict->Insert(lastDictIdxKey, i->filters[id].lastDict->RemoveEntry(lastDictIdxKey)); + continue; } - + + // { - std::list::iterator it = i->events.begin(); - while ( it != i->events.end() ) { + /* FIXME: events + std::list::iterator it = i->filters[id].events.begin(); + while ( it != i->filters[id].events.end() ) { Ref(idx); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); SendEvent(*it, ev, idx, val); ++it; } + */ } } - i->tab->Delete(ih->idxkey); - i->lastDict->Remove(lastDictIdxKey); // deletex in next line + i->filters[id].tab->Delete(ih->idxkey); + i->filters[id].lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->lastDict->Clear(); // should be empty... but... well... who knows... - delete(i->lastDict); + i->filters[id].lastDict->Clear(); // should be empty... but... well... who knows... + delete(i->filters[id].lastDict); - i->lastDict = i->currDict; - i->currDict = new PDict(InputHash); + i->filters[id].lastDict = i->filters[id].currDict; + i->filters[id].currDict = new PDict(InputHash); } void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { @@ -696,22 +663,24 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) return; } - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + assert(i->HasFilter(id)); + + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); Val* valval; - int position = i->num_idx_fields; - if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); + int position = i->filters[id].num_idx_fields; + if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { + valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->rtype); + RecordVal * r = new RecordVal(i->filters[id].rtype); - for ( int j = 0; j < i->rtype->NumFields(); j++) { + for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { Val* val = 0; - if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)); + val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); position++; } @@ -726,7 +695,7 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) valval = r; } - i->tab->Assign(idxval, valval); + i->filters[id].tab->Assign(idxval, valval); } void InputMgr::Clear(const InputReader* reader, int id) { @@ -735,20 +704,24 @@ void InputMgr::Clear(const InputReader* reader, int id) { reporter->InternalError("Unknown reader"); return; } - - i->tab->RemoveAll(); + + assert(i->HasFilter(id)); + + i->filters[id].tab->RemoveAll(); } -bool InputMgr::Delete(const InputReader* reader, const LogVal* const *vals) { +bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); return false; } - - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); - return ( i->tab->Delete(idxval) != 0 ); + assert(i->HasFilter(id)); + + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + + return ( i->filters[id].tab->Delete(idxval) != 0 ); } void InputMgr::Error(InputReader* reader, const char* msg) diff --git a/src/InputMgr.h b/src/InputMgr.h index 1cacf89143..5d531cd6fc 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -35,7 +35,7 @@ protected: void Error(InputReader* reader, const char* msg); // for readers to write to input stream in direct mode (reporting new/deleted values directly) - void Put(const InputReader* reader, int id. const LogVal* const *vals); + void Put(const InputReader* reader, int id, const LogVal* const *vals); void Clear(const InputReader* reader, int id); bool Delete(const InputReader* reader, int id, const LogVal* const *vals); diff --git a/src/InputReader.cc b/src/InputReader.cc index 1008cf1b67..1c65985fd6 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -26,17 +26,17 @@ void InputReader::Error(const string &msg) void InputReader::Put(int id, const LogVal* const *val) { - input_mgr->Put(this, int id, val); + input_mgr->Put(this, id, val); } void InputReader::Clear(int id) { - input_mgr->Clear(this, int id); + input_mgr->Clear(this, id); } void InputReader::Delete(int id, const LogVal* const *val) { - input_mgr->Delete(this, int id, val); + input_mgr->Delete(this, id, val); } @@ -52,12 +52,12 @@ bool InputReader::Init(string arg_source) bool InputReader::AddFilter(int id, int arg_num_fields, const LogField* const * arg_fields) { - return DoAddFilter(int id, arg_num_fields, arg_fields); + return DoAddFilter(id, arg_num_fields, arg_fields); } bool InputReader::RemoveFilter(int id) { - return DoRemoveFilter(int id); + return DoRemoveFilter(id); } void InputReader::Finish() @@ -105,10 +105,10 @@ const char* InputReader::Fmt(const char* format, ...) void InputReader::SendEntry(int id, const LogVal* const *vals) { - input_mgr->SendEntry(this, int id, vals); + input_mgr->SendEntry(this, id, vals); } void InputReader::EndCurrentSend(int id) { - input_mgr->EndCurrentSend(this, int id); + input_mgr->EndCurrentSend(this, id); } diff --git a/src/InputReader.h b/src/InputReader.h index 12f0bc9db4..6e3d689750 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -30,7 +30,8 @@ protected: virtual bool DoInit(string arg_sources) = 0; virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) = 0; - virtual bool DoRemoveFilter( int id ); + + virtual bool DoRemoveFilter( int id ) = 0; virtual void DoFinish() = 0; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4a0d4157bc..84feb74e61 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -28,6 +28,7 @@ FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } + InputReaderAscii::InputReaderAscii() { file = 0; @@ -58,7 +59,7 @@ InputReaderAscii::~InputReaderAscii() void InputReaderAscii::DoFinish() { - columnMap.empty(); + filters.empty(); if ( file != 0 ) { file->close(); delete(file); @@ -66,7 +67,7 @@ void InputReaderAscii::DoFinish() } } -bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const LogField* const * fields) +bool InputReaderAscii::DoInit(string path) { fname = path; @@ -76,11 +77,39 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } + return true; +} - this->num_fields = num_fields; - this->idx_fields = idx_fields; - this->fields = fields; +bool InputReaderAscii::DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) { + if ( HasFilter(id) ) { + return false; // no, we don't want to add this a second time + } + Filter f; + f.num_fields = arg_num_fields; + f.fields = fields; + + filters[id] = f; + + return true; +} + +bool InputReaderAscii::DoRemoveFilter ( int id ) { + if (!HasFilter(id) ) { + return false; + } + + assert ( filters.erase(id) == 1 ); + + return true; +} + + +bool InputReaderAscii::HasFilter(int id) { + map::iterator it = filters.find(id); + if ( it == filters.end() ) { + return false; + } return true; } @@ -93,46 +122,47 @@ bool InputReaderAscii::ReadHeader() { return false; } - // split on tabs... - istringstream splitstream(line); - unsigned int currTab = 0; - int wantFields = 0; - while ( splitstream ) { - string s; - if ( !getline(splitstream, s, separator[0])) - break; - - // current found heading in s... compare if we want it - for ( unsigned int i = 0; i < num_fields; i++ ) { - const LogField* field = fields[i]; - if ( field->name == s ) { - // cool, found field. note position - FieldMapping f(field->name, field->type, field->subtype, i); - columnMap.push_back(f); - wantFields++; - break; // done with searching + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + // split on tabs... + istringstream splitstream(line); + unsigned int currTab = 0; + int wantFields = 0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0])) + break; + + // current found heading in s... compare if we want it + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + const LogField* field = (*it).second.fields[i]; + if ( field->name == s ) { + // cool, found field. note position + FieldMapping f(field->name, field->type, field->subtype, i); + (*it).second.columnMap.push_back(f); + wantFields++; + break; // done with searching + } } + + // look if we did push something... + if ( (*it).second.columnMap.size() == currTab ) { + // no, we didn't. note that... + FieldMapping empty; + (*it).second.columnMap.push_back(empty); + } + + // done + currTab++; + } + + if ( wantFields != (int) (*it).second.num_fields ) { + // we did not find all fields? + // :( + Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d. Filternum: %d", wantFields, (*it).second.num_fields, (*it).first)); + return false; } - - // look if we did push something... - if ( columnMap.size() == currTab ) { - // no, we didn't. note that... - FieldMapping empty; - columnMap.push_back(empty); - } - - // done - currTab++; - } - - if ( wantFields != (int) num_fields ) { - // we did not find all fields? - // :( - Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d", wantFields, num_fields)); - return false; } - // well, that seems to have worked... return true; } @@ -314,110 +344,77 @@ bool InputReaderAscii::DoUpdate() { return false; } - // TODO: all the stuff we need for a second reading. - // *cough* - // - // - - - // new keymap - //map *newKeyMap = new map(); - string line; while ( GetLine(line ) ) { - // split on tabs + + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - istringstream splitstream(line); - - LogVal** fields = new LogVal*[num_fields]; - //string string_fields[num_fields]; - - unsigned int currTab = 0; - unsigned int currField = 0; - while ( splitstream ) { - - string s; - if ( !getline(splitstream, s, separator[0]) ) - break; - + // split on tabs - if ( currTab >= columnMap.size() ) { - Error("Tabs in heading do not match tabs in data?"); - //disabled = true; - return false; - } - - FieldMapping currMapping = columnMap[currTab]; - currTab++; - - if ( currMapping.IsEmpty() ) { - // well, that was easy - continue; - } - - if ( currField >= num_fields ) { - Error("internal error - fieldnum greater as possible"); - return false; - } - - LogVal* val = EntryToVal(s, currMapping); - if ( val == 0 ) { - return false; - } - fields[currMapping.position] = val; - //string_fields[currMapping.position] = s; - - currField++; - } - - if ( currField != num_fields ) { - Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); - return false; - } - - - SendEntry(fields); - - /* - string indexstring = ""; - string valstring = ""; - for ( unsigned int i = 0; i < idx_fields; i++ ) { - indexstring.append(string_fields[i]); - } - - for ( unsigned int i = idx_fields; i < num_fields; i++ ) { - valstring.append(string_fields[i]); - } - - string valhash = Hash(valstring); - string indexhash = Hash(indexstring); - - if ( keyMap->find(indexhash) == keyMap->end() ) { - // new key - Put(fields); - } else if ( (*keyMap)[indexhash] != valhash ) { - // changed key - Put(fields); - keyMap->erase(indexhash); - } else { - // field not changed - keyMap->erase(indexhash); - } - - - (*newKeyMap)[indexhash] = valhash; - */ + istringstream splitstream(line); - for ( unsigned int i = 0; i < num_fields; i++ ) { - delete fields[i]; + LogVal** fields = new LogVal*[(*it).second.num_fields]; + //string string_fields[num_fields]; + + unsigned int currTab = 0; + unsigned int currField = 0; + while ( splitstream ) { + + string s; + if ( !getline(splitstream, s, separator[0]) ) + break; + + + if ( currTab >= (*it).second.columnMap.size() ) { + Error("Tabs in heading do not match tabs in data?"); + //disabled = true; + return false; + } + + FieldMapping currMapping = (*it).second.columnMap[currTab]; + currTab++; + + if ( currMapping.IsEmpty() ) { + // well, that was easy + continue; + } + + if ( currField >= (*it).second.num_fields ) { + Error("internal error - fieldnum greater as possible"); + return false; + } + + LogVal* val = EntryToVal(s, currMapping); + if ( val == 0 ) { + return false; + } + fields[currMapping.position] = val; + //string_fields[currMapping.position] = s; + + currField++; + } + + if ( currField != (*it).second.num_fields ) { + Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); + return false; + } + + + SendEntry((*it).first, fields); + + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; } - delete [] fields; } //file->clear(); // remove end of file evil bits //file->seekg(0, ios::beg); // and seek to start. - EndCurrentSend(); + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + EndCurrentSend((*it).first); + } return true; } diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c848c17110..01169a3cfc 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -34,13 +34,30 @@ public: protected: - virtual bool DoInit(string path, int arg_num_fields, int arg_idx_fields, - const LogField* const * fields); + virtual bool DoInit(string path); + + virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ); + + virtual bool DoRemoveFilter ( int id ); + virtual void DoFinish(); virtual bool DoUpdate(); private: + + struct Filter { + unsigned int num_fields; + + const LogField* const * fields; // raw mapping + + // map columns in the file to columns to send back to the manager + vector columnMap; + + }; + + bool HasFilter(int id); + bool ReadHeader(); LogVal* EntryToVal(string s, FieldMapping type); @@ -49,15 +66,8 @@ private: ifstream* file; string fname; - unsigned int num_fields; - unsigned int idx_fields; + map filters; - // map columns in the file to columns to send back to the manager - vector columnMap; - const LogField* const * fields; // raw mapping - - //map *keyMap; - // // Options set from the script-level. string separator; From 7eb4d9934168fa0d37e01e3dbdd6feb60a88c485 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 12:27:34 -0800 Subject: [PATCH 030/149] very basic functionality kind of works again --- src/InputMgr.cc | 52 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 403d656140..b51cd6fd27 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -47,10 +47,50 @@ public: EventHandlerPtr event; RecordType* event_type; - ~Filter(); + // ~Filter(); + // Filter(); + // Filter(const InputMgr::Filter& filter); + + void DoCleanup(); }; -InputMgr::Filter::~Filter() { +/* +InputMgr::Filter::Filter() { + tab = 0; + itype = 0; + rtype = 0; + event_type = 0; +} + +InputMgr::Filter::Filter(const InputMgr::Filter& f) { + id = f.id; + id->Ref(); + + tab = f.tab; + if ( tab ) + tab->Ref(); + + itype = f.itype; + if ( itype ) + itype->Ref(); + + rtype = f.rtype; + if ( rtype ) + Ref(rtype); + + event_type = f.event_type; + if ( event_type ) + Ref(event_type); + + name = f.name; + num_idx_fields = f.num_idx_fields; + num_val_fields = f.num_val_fields; + want_record = f.want_record; + + +} */ + +void InputMgr::Filter::DoCleanup() { Unref(id); if ( tab ) Unref(tab); @@ -60,7 +100,10 @@ InputMgr::Filter::~Filter() { Unref(rtype); if ( event_type) Unref(event_type); -} + + delete currDict; + delete lastDict; +} struct InputMgr::ReaderInfo { EnumVal* id; @@ -395,6 +438,8 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } + i->filters[id->InternalInt()].DoCleanup(); + i->filters.erase(it); return true; } @@ -545,6 +590,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const return; } + reporter->Error("assigning"); i->filters[id].tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); From 029871e48c4c63add07d63a52127bb7b50f47189 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 13:42:02 -0800 Subject: [PATCH 031/149] first test. --- .../scripts.base.frameworks.input.basic/out | 14 +++++ .../scripts/base/frameworks/input/basic.bro | 52 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.basic/out create mode 100644 testing/btest/scripts/base/frameworks/input/basic.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out new file mode 100644 index 0000000000..ebac1866b6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out @@ -0,0 +1,14 @@ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro new file mode 100644 index 0000000000..5e0c7be12e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -0,0 +1,52 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::force_update(A::LOG); + print servers; +} From f0e5303330d9842a7454001276ef5857c252bba9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:09:00 -0800 Subject: [PATCH 032/149] make want_record field for tablefilter work... --- src/InputMgr.cc | 2 +- .../out | 3 ++ .../out | 3 ++ .../frameworks/input/onecolumn-norecord.bro | 38 +++++++++++++++++++ .../frameworks/input/onecolumn-record.bro | 38 +++++++++++++++++++ 5 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out create mode 100644 testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro create mode 100644 testing/btest/scripts/base/frameworks/input/onecolumn-record.bro diff --git a/src/InputMgr.cc b/src/InputMgr.cc index b51cd6fd27..c4180f4f8d 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -515,7 +515,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const int position = i->filters[id].num_idx_fields; if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); + valval = LogValToVal(vals[position], i->filters[id].rtype->FieldType(0)); } else { RecordVal * r = new RecordVal(i->filters[id].rtype); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out new file mode 100644 index 0000000000..bbce48f4f6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out @@ -0,0 +1,3 @@ +{ +[-42] = T +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out new file mode 100644 index 0000000000..3f9af35c59 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out @@ -0,0 +1,3 @@ +{ +[-42] = [b=T] +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro new file mode 100644 index 0000000000..74fd477e28 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +#types bool int +T -42 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::force_update(A::LOG); + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro new file mode 100644 index 0000000000..3cc7090462 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +#types bool int +T -42 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::force_update(A::LOG); + print servers; +} From 18591b53d422e5b97dc8595a5171e50a64feec78 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:20:52 -0800 Subject: [PATCH 033/149] rename filter to tablefilter in preparation of event filters... --- scripts/base/frameworks/input/main.bro | 38 +++++++++---------- src/InputMgr.cc | 6 +-- src/InputMgr.h | 4 +- src/input.bif | 10 ++--- testing/btest/btest.cfg | 2 +- .../scripts/base/frameworks/input/basic.bro | 2 +- .../frameworks/input/onecolumn-norecord.bro | 2 +- .../frameworks/input/onecolumn-record.bro | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 9d83d73ec6..4560421ecc 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -9,7 +9,7 @@ export { reader: Reader &default=default_reader; }; - type Filter: record { + type TableFilter: record { ## descriptive name. for later removal name: string; @@ -29,14 +29,14 @@ export { # ev_description: any &optional; }; - const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. + #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; - global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; - global remove_filter: function(id: Log::ID, name: string) : bool; - global get_filter: function(id: ID, name: string) : Filter; + global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; + global remove_tablefilter: function(id: Log::ID, name: string) : bool; + #global get_filter: function(id: ID, name: string) : Filter; } @@ -45,7 +45,7 @@ export { module Input; -global filters: table[ID, string] of Filter; +#global filters: table[ID, string] of Filter; function create_stream(id: Log::ID, description: Input::StreamDescription) : bool { @@ -62,22 +62,22 @@ function force_update(id: Log::ID) : bool return __force_update(id); } -function add_filter(id: Log::ID, filter: Input::Filter) : bool +function add_tablefilter(id: Log::ID, filter: Input::TableFilter) : bool { - filters[id, filter$name] = filter; - return __add_filter(id, filter); +# filters[id, filter$name] = filter; + return __add_tablefilter(id, filter); } -function remove_filter(id: Log::ID, name: string) : bool +function remove_tablefilter(id: Log::ID, name: string) : bool { - delete filters[id, name]; - return __remove_filter(id, name); +# delete filters[id, name]; + return __remove_tablefilter(id, name); } -function get_filter(id: ID, name: string) : Filter - { - if ( [id, name] in filters ) - return filters[id, name]; - - return no_filter; - } +#function get_filter(id: ID, name: string) : Filter +# { +# if ( [id, name] in filters ) +# return filters[id, name]; +# +# return no_filter; +# } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index c4180f4f8d..8fda5d506f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -237,7 +237,7 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) } -bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { +bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { reporter->Error("Stream not found"); @@ -245,7 +245,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { } RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::TableFilter, 0) ) { reporter->Error("filter argument not of right type"); return false; @@ -426,7 +426,7 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } -bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { +bool InputMgr::RemoveTableFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { reporter->Error("Reader not found"); diff --git a/src/InputMgr.h b/src/InputMgr.h index 5d531cd6fc..4280ba1d81 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -24,8 +24,8 @@ public: bool ForceUpdate(const EnumVal* id); bool RemoveStream(const EnumVal* id); - bool AddFilter(EnumVal *id, RecordVal* filter); - bool RemoveFilter(EnumVal* id, const string &name); + bool AddTableFilter(EnumVal *id, RecordVal* filter); + bool RemoveTableFilter(EnumVal* id, const string &name); protected: diff --git a/src/input.bif b/src/input.bif index ef069316ab..1300f91bea 100644 --- a/src/input.bif +++ b/src/input.bif @@ -8,7 +8,7 @@ module Input; %%} type StreamDescription: record; -type Filter: record; +type TableFilter: record; function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ @@ -28,15 +28,15 @@ function Input::__force_update%(id: Log::ID%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool +function Input::__add_tablefilter%(id: Log::ID, filter: Input::TableFilter%) : bool %{ - bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_filter%(id: Log::ID, name: string%) : bool +function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool %{ - bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); + bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); %} diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 7d8283587c..739d0b2ad4 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -3,7 +3,7 @@ TestDirs = doc bifs language core scripts istate coverage TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp -IgnoreFiles = *.tmp *.swp #* *.trace +IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 5e0c7be12e..139888fa7c 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -46,7 +46,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 74fd477e28..12dbdd42aa 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -32,7 +32,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 3cc7090462..4eef12d752 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -32,7 +32,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); Input::force_update(A::LOG); print servers; } From 92b3723b0947739fb4aff9a3d7a87662f65011c7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:36:03 -0800 Subject: [PATCH 034/149] add very basic predicate test. --- .../out | 7 ++ .../base/frameworks/input/predicate.bro | 66 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicate/out create mode 100644 testing/btest/scripts/base/frameworks/input/predicate.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out new file mode 100644 index 0000000000..d805f804d8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out @@ -0,0 +1,7 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro new file mode 100644 index 0000000000..e82ded6fd0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -0,0 +1,66 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F, + $pred(typ: Input::Event, left: idx, right: bool) = { return right; } + ]); + Input::force_update(A::LOG); + if ( 1 in servers ) { + print "VALID"; + } + if ( 2 in servers ) { + print "VALID"; + } + if ( !(3 in servers) ) { + print "VALID"; + } + if ( !(4 in servers) ) { + print "VALID"; + } + if ( !(5 in servers) ) { + print "VALID"; + } + if ( !(6 in servers) ) { + print "VALID"; + } + if ( 7 in servers ) { + print "VALID"; + } +} From 77a517f2b5e90d94c9de144605c24247c620e4af Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:45:27 -0800 Subject: [PATCH 035/149] camel-casing for types --- testing/btest/scripts/base/frameworks/input/basic.bro | 8 ++++---- .../base/frameworks/input/onecolumn-norecord.bro | 8 ++++---- .../scripts/base/frameworks/input/onecolumn-record.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/predicate.bro | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 139888fa7c..3ad45eac69 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -18,11 +18,11 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; e: Log::ID; c: count; @@ -40,13 +40,13 @@ type val: record { ve: vector of int; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 12dbdd42aa..134ceb49e6 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -18,21 +18,21 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 4eef12d752..c07c9c826c 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -18,21 +18,21 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index e82ded6fd0..769536d2a6 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -24,22 +24,22 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F, - $pred(typ: Input::Event, left: idx, right: bool) = { return right; } + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); Input::force_update(A::LOG); if ( 1 in servers ) { From 53af0544ccc9a60254228ab702ec39639b5b577d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 19:01:07 -0800 Subject: [PATCH 036/149] re-enable table events --- scripts/base/frameworks/input/main.bro | 2 +- src/InputMgr.cc | 120 +++++++++++------- src/InputMgr.h | 2 +- .../scripts.base.frameworks.input.event/out | 21 +++ .../scripts/base/frameworks/input/event.bro | 48 +++++++ 5 files changed, 147 insertions(+), 46 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.event/out create mode 100644 testing/btest/scripts/base/frameworks/input/event.bro diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4560421ecc..69b4d41ebb 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -18,7 +18,7 @@ export { val: any; destination: any; want_record: bool &default=T; - table_ev: any &optional; # event containing idx, val as values. + ev: any &optional; # event containing idx, val as values. ## decision function, that decides if an insertion, update or removal should really be executed. ## or events should be thought diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 8fda5d506f..de9ef158b7 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -214,7 +214,6 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault @@ -259,6 +258,50 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Func* event = event_val ? event_val->AsFunc() : 0; + + if ( event ) { + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) { + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() != 3 ) + { + reporter->Error("Table event must take 3 arguments"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("table events first attribute must be of type Input::Event"); + return false; + } + + if ( ! same_type((*args)[1], idx) ) + { + reporter->Error("table events index attributes do not match"); + return false; + } + + if ( want_record->InternalInt() == 1 && ! same_type((*args)[2], val) ) + { + reporter->Error("table events value attributes do not match"); + return false; + } else if ( want_record->InternalInt() == 0 && !same_type((*args)[2], val->FieldType(0) ) ) { + reporter->Error("table events value attribute does not match"); + return false; + } + + } + vector fieldsV; // vector, because we don't know the length beforehands bool status = !UnrollRecordType(&fieldsV, idx, ""); @@ -273,24 +316,22 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { return false; } - Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); LogField** fields = new LogField*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { fields[i] = fieldsV[i]; } - // FIXME: remove those funky 0-tests again as the idea was changed. Filter filter; filter.name = name->AsString()->CheckString(); filter.id = id->Ref()->AsEnumVal(); filter.pred = pred ? pred->AsFunc() : 0; filter.num_idx_fields = idxfields; filter.num_val_fields = valfields; - filter.tab = dst ? dst->Ref()->AsTableVal() : 0; - filter.rtype = val ? val->Ref()->AsRecordType() : 0; - filter.itype = idx ? idx->Ref()->AsRecordType() : 0; - // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted + filter.tab = dst->Ref()->AsTableVal(); + filter.rtype = val->Ref()->AsRecordType(); + filter.itype = idx->Ref()->AsRecordType(); + filter.event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter.currDict = new PDict(InputHash); filter.lastDict = new PDict(InputHash); filter.want_record = ( want_record->InternalInt() == 1 ); @@ -601,11 +642,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const i->filters[id].currDict->Insert(idxhash, ih); - // send events now that we are kind of finished. - - /* FIXME: fix me. - std::list::iterator filter_iterator = i->events.begin(); - while ( filter_iterator != i->events.end() ) { + if ( i->filters[id].event ) { EnumVal* ev; Ref(idxval); @@ -613,16 +650,13 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); - SendEvent(*filter_iterator, ev, idxval, oldval); + SendEvent(i->filters[id].event, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(*filter_iterator, ev, idxval, valval); + SendEvent(i->filters[id].event, ev, idxval, valval); } - - - ++filter_iterator; - } */ + } } @@ -643,12 +677,17 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { //while ( ( ih = i->lastDict->NextEntry(c) ) ) { while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { - if ( i->filters[id].pred ) { - ListVal *idx = i->filters[id].tab->RecoverIndex(ih->idxkey); - assert(idx != 0); - Val *val = i->filters[id].tab->Lookup(idx); - assert(val != 0); + ListVal * idx; + Val *val; + if ( i->filters[id].pred || i->filters[id].event ) { + idx = i->filters[id].tab->RecoverIndex(ih->idxkey); + assert(idx != 0); + val = i->filters[id].tab->Lookup(idx); + assert(val != 0); + } + + if ( i->filters[id].pred ) { bool doBreak = false; // ask predicate, if we want to expire this element... @@ -673,21 +712,13 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { } - // - - { - /* FIXME: events - std::list::iterator it = i->filters[id].events.begin(); - while ( it != i->filters[id].events.end() ) { - Ref(idx); - Ref(val); - EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(*it, ev, idx, val); - ++it; - } - */ - } + } + if ( i->filters[id].event ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(i->filters[id].event, ev, idx, val); } i->filters[id].tab->Delete(ih->idxkey); @@ -792,20 +823,21 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } */ -void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) +void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) { - EventHandler* handler = event_registry->Lookup(name.c_str()); - if ( handler == 0 ) { - reporter->Error("Event %s not found", name.c_str()); - return; - } + //EventHandler* handler = event_registry->Lookup(name.c_str()); + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", name.c_str()); + // return; + //} val_list* vl = new val_list; vl->append(event); vl->append(left); vl->append(right); - mgr.Dispatch(new Event(handler, vl)); + //mgr.Dispatch(new Event(handler, vl)); + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } diff --git a/src/InputMgr.h b/src/InputMgr.h index 4280ba1d81..d04b7c9a2c 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -49,7 +49,7 @@ private: bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); - void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); + void SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); int GetLogValLength(const LogVal* val); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out new file mode 100644 index 0000000000..e32a2aea00 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out @@ -0,0 +1,21 @@ +Input::EVENT_NEW +1 +T +Input::EVENT_NEW +2 +T +Input::EVENT_NEW +3 +F +Input::EVENT_NEW +4 +F +Input::EVENT_NEW +5 +F +Input::EVENT_NEW +6 +F +Input::EVENT_NEW +7 +T diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro new file mode 100644 index 0000000000..36e8171689 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -0,0 +1,48 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +event line(tpe: Input::Event, left: Idx, right: bool) { + print tpe; + print left; + print right; +} + +event bro_init() +{ + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::force_update(A::LOG); +} From 3035eb2b219946144fe67097781226d77f34176e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 19:30:16 -0800 Subject: [PATCH 037/149] fix a little bug that prevented several simultaneous filters from working. --- src/InputMgr.cc | 8 +- .../out | 15 +++ .../base/frameworks/input/twofilters.bro | 95 +++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out create mode 100644 testing/btest/scripts/base/frameworks/input/twofilters.bro diff --git a/src/InputMgr.cc b/src/InputMgr.cc index de9ef158b7..2ec3c649be 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -341,8 +341,12 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { assert(filter.want_record); } - i->filters[id->InternalInt()] = filter; - i->reader->AddFilter( id->InternalInt(), fieldsV.size(), fields ); + int filterid = 0; + if ( i->filters.size() > 0 ) { + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map. new id = old id + 1. + } + i->filters[filterid] = filter; + i->reader->AddFilter( filterid, fieldsV.size(), fields ); return true; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out new file mode 100644 index 0000000000..5b1ee5e983 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out @@ -0,0 +1,15 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID +MARK +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro new file mode 100644 index 0000000000..575665e6e5 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -0,0 +1,95 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination1: table[int] of Val = table(); +global destination2: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } + ]); + Input::add_tablefilter(A::LOG, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); + + Input::force_update(A::LOG); + if ( 1 in destination1 ) { + print "VALID"; + } + if ( 2 in destination1 ) { + print "VALID"; + } + if ( !(3 in destination1) ) { + print "VALID"; + } + if ( !(4 in destination1) ) { + print "VALID"; + } + if ( !(5 in destination1) ) { + print "VALID"; + } + if ( !(6 in destination1) ) { + print "VALID"; + } + if ( 7 in destination1 ) { + print "VALID"; + } + + print "MARK"; + + if ( 2 in destination2 ) { + print "VALID"; + } + if ( 2 in destination2 ) { + print "VALID"; + } + if ( 3 in destination2 ) { + print "VALID"; + } + if ( 4 in destination2 ) { + print "VALID"; + } + if ( 5 in destination2 ) { + print "VALID"; + } + if ( 6 in destination2 ) { + print "VALID"; + } + if ( 7 in destination2 ) { + print "VALID"; + } + + +} From f82bf3f35fb5f1e298366f9af6d9ec0372ac3d58 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 22 Nov 2011 11:09:06 -0800 Subject: [PATCH 038/149] re-enable direct event sending from input readers --- src/InputMgr.cc | 18 +++++++++++++----- src/InputMgr.h | 3 ++- src/InputReader.cc | 7 +++---- src/InputReader.h | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 2ec3c649be..e6d739d26f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -810,22 +810,30 @@ void InputMgr::Error(InputReader* reader, const char* msg) reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -/* Does not work atm, because LogValToVal needs BroType -void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +bool InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); if ( handler == 0 ) { reporter->Error("Event %s not found", name.c_str()); - return; + return false; + } + + RecordType *type = handler->FType()->Args(); + int num_event_vals = type->NumFields(); + if ( num_vals != num_event_vals ) { + reporter->Error("Wrong number of values for event %s", name.c_str()); + return false; } val_list* vl = new val_list; for ( int i = 0; i < num_vals; i++) { - vl->append(LogValToVal(vals[i])); + vl->append(LogValToVal(vals[i], type->FieldType(i))); } mgr.Dispatch(new Event(handler, vl)); -} */ + + return true; +} void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) { diff --git a/src/InputMgr.h b/src/InputMgr.h index d04b7c9a2c..ba1ddafc92 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -59,7 +59,8 @@ private: Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReader.cc b/src/InputReader.cc index 1c65985fd6..3f296dc0aa 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -71,11 +71,10 @@ bool InputReader::Update() return DoUpdate(); } -/* -void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +bool InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { - input_mgr->SendEvent(name, num_vals, vals); -} */ + return input_mgr->SendEvent(name, num_vals, vals); +} // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) diff --git a/src/InputReader.h b/src/InputReader.h index 6e3d689750..34d549308e 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -48,7 +48,7 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table void Put(int id, const LogVal* const *val); From 3c40f00a539e5d3d2945ffd1df71e19e93c7133d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 22 Nov 2011 11:39:27 -0800 Subject: [PATCH 039/149] make filters pointers (for inheritance) --- scripts/base/frameworks/input/main.bro | 29 +- src/InputMgr.cc | 519 +++++++++++++----- src/InputMgr.h | 21 +- src/input.bif | 13 + .../out | 21 + .../scripts/base/frameworks/input/event.bro | 16 +- .../base/frameworks/input/tableevent.bro | 48 ++ 7 files changed, 507 insertions(+), 160 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out create mode 100644 testing/btest/scripts/base/frameworks/input/tableevent.bro diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 69b4d41ebb..a036eeeebd 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -23,10 +23,19 @@ export { ## decision function, that decides if an insertion, update or removal should really be executed. ## or events should be thought pred: function(typ: Input::Event, left: any, right: any): bool &optional; + }; - ## for "normalized" events - # ev: any &optional; - # ev_description: any &optional; + type EventFilter: record { + ## descriptive name. for later removal + name: string; + + # the event + ev: any; + # record describing the fields + fields: any; + + # does the event want the field unrolled (default) or as a simple record value? + want_record: bool &default=F; }; #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. @@ -36,6 +45,8 @@ export { global force_update: function(id: Log::ID) : bool; global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; global remove_tablefilter: function(id: Log::ID, name: string) : bool; + global add_eventfilter: function(id: Log::ID, filter: Input::EventFilter) : bool; + global remove_eventfilter: function(id: Log::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; } @@ -74,6 +85,18 @@ function remove_tablefilter(id: Log::ID, name: string) : bool return __remove_tablefilter(id, name); } +function add_eventfilter(id: Log::ID, filter: Input::EventFilter) : bool + { +# filters[id, filter$name] = filter; + return __add_eventfilter(id, filter); + } + +function remove_eventfilter(id: Log::ID, name: string) : bool + { +# delete filters[id, name]; + return __remove_eventfilter(id, name); + } + #function get_filter(id: ID, name: string) : Filter # { # if ( [id, name] in filters ) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e6d739d26f..0df11ea359 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -28,7 +28,13 @@ public: EnumVal* id; string name; - //int filter_type; // to distinguish between event and table filters + FilterType filter_type; // to distinguish between event and table filters + + virtual ~Filter(); +}; + +class InputMgr::TableFilter: public InputMgr::Filter { +public: unsigned int num_idx_fields; unsigned int num_val_fields; @@ -45,61 +51,42 @@ public: Func* pred; EventHandlerPtr event; - RecordType* event_type; - // ~Filter(); - // Filter(); - // Filter(const InputMgr::Filter& filter); - - void DoCleanup(); + TableFilter(); + ~TableFilter(); }; -/* -InputMgr::Filter::Filter() { +class InputMgr::EventFilter: public InputMgr::Filter { +public: + EventHandlerPtr event; + + RecordType* fields; + unsigned int num_fields; + + bool want_record; + EventFilter(); +}; + +InputMgr::TableFilter::TableFilter() { + filter_type = TABLE_FILTER; + tab = 0; itype = 0; rtype = 0; - event_type = 0; } -InputMgr::Filter::Filter(const InputMgr::Filter& f) { - id = f.id; - id->Ref(); +InputMgr::EventFilter::EventFilter() { + filter_type = EVENT_FILTER; +} - tab = f.tab; - if ( tab ) - tab->Ref(); - - itype = f.itype; - if ( itype ) - itype->Ref(); - - rtype = f.rtype; - if ( rtype ) - Ref(rtype); - - event_type = f.event_type; - if ( event_type ) - Ref(event_type); - - name = f.name; - num_idx_fields = f.num_idx_fields; - num_val_fields = f.num_val_fields; - want_record = f.want_record; - - -} */ - -void InputMgr::Filter::DoCleanup() { +InputMgr::Filter::~Filter() { Unref(id); - if ( tab ) - Unref(tab); - if ( itype ) - Unref(itype); - if ( rtype ) - Unref(rtype); - if ( event_type) - Unref(event_type); +} + +InputMgr::TableFilter::~TableFilter() { + Unref(tab); + Unref(itype); + Unref(rtype); delete currDict; delete lastDict; @@ -111,7 +98,7 @@ struct InputMgr::ReaderInfo { InputReader* reader; //list events; // events we fire when "something" happens - map filters; // filters that can prevent our actions + map filters; // filters that can prevent our actions bool HasFilter(int id); @@ -119,7 +106,11 @@ struct InputMgr::ReaderInfo { }; InputMgr::ReaderInfo::~ReaderInfo() { - // all the contents of filters should delete themselves automatically... + map::iterator it = filters.begin(); + + while ( it != filters.end() ) { + delete (*it).second; + } Unref(type); Unref(id); @@ -128,7 +119,7 @@ InputMgr::ReaderInfo::~ReaderInfo() { } bool InputMgr::ReaderInfo::HasFilter(int id) { - map::iterator it = filters.find(id); + map::iterator it = filters.find(id); if ( it == filters.end() ) { return false; } @@ -236,6 +227,114 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) } +bool InputMgr::AddEventFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Stream not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::EventFilter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + RecordType *fields = fval->Lookup(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Func* event = event_val->AsFunc(); + + { + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) { + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() < 2 ) { + reporter->Error("event takes not enough arguments"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("events first attribute must be of type Input::Event"); + return false; + } + + if ( want_record->InternalInt() == 0 ) { + if ( args->length() != fields->NumFields() + 1 ) { + reporter->Error("events has wrong number of arguments"); + return false; + } + + for ( int i = 0; i < fields->NumFields(); i++ ) { + if ( !same_type((*args)[i+1], fields->FieldType(i) ) ) { + reporter->Error("Incompatible type for event"); + return false; + } + } + + } else if ( want_record->InternalInt() == 1 ) { + if ( args->length() != 2 ) { + reporter->Error("events has wrong number of arguments"); + return false; + } + + if ( !same_type((*args)[1], fields ) ) { + reporter->Error("Incompatible type for event"); + return false; + } + + } else { + assert(false); + } + + } + + + vector fieldsV; // vector, because UnrollRecordType needs it + + bool status = !UnrollRecordType(&fieldsV, fields, ""); + + if ( status ) { + reporter->Error("Problem unrolling"); + return false; + } + + + LogField** logf = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + logf[i] = fieldsV[i]; + } + + EventFilter* filter = new EventFilter(); + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); + filter->num_fields = fieldsV.size(); + filter->fields = fields->Ref()->AsRecordType(); + filter->event = event_registry->Lookup(event->GetID()->Name()); + filter->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + + int filterid = 0; + if ( i->filters.size() > 0 ) { + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> + } + i->filters[filterid] = filter; + i->reader->AddFilter( filterid, fieldsV.size(), logf ); + + return true; +} + bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -299,6 +398,7 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { reporter->Error("table events value attribute does not match"); return false; } + assert(want_record->InternalInt() == 1 || want_record->InternalInt() == 0); } @@ -322,28 +422,28 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } - Filter filter; - filter.name = name->AsString()->CheckString(); - filter.id = id->Ref()->AsEnumVal(); - filter.pred = pred ? pred->AsFunc() : 0; - filter.num_idx_fields = idxfields; - filter.num_val_fields = valfields; - filter.tab = dst->Ref()->AsTableVal(); - filter.rtype = val->Ref()->AsRecordType(); - filter.itype = idx->Ref()->AsRecordType(); - filter.event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; - filter.currDict = new PDict(InputHash); - filter.lastDict = new PDict(InputHash); - filter.want_record = ( want_record->InternalInt() == 1 ); + TableFilter* filter = new TableFilter(); + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); + filter->pred = pred ? pred->AsFunc() : 0; + filter->num_idx_fields = idxfields; + filter->num_val_fields = valfields; + filter->tab = dst->Ref()->AsTableVal(); + filter->rtype = val->Ref()->AsRecordType(); + filter->itype = idx->Ref()->AsRecordType(); + filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + filter->currDict = new PDict(InputHash); + filter->lastDict = new PDict(InputHash); + filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { - assert(filter.want_record); + assert(filter->want_record); } int filterid = 0; if ( i->filters.size() > 0 ) { - filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map. new id = old id + 1. + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> } i->filters[filterid] = filter; i->reader->AddFilter( filterid, fieldsV.size(), fields ); @@ -478,13 +578,39 @@ bool InputMgr::RemoveTableFilter(EnumVal* id, const string &name) { return false; } - map::iterator it = i->filters.find(id->InternalInt()); + map::iterator it = i->filters.find(id->InternalInt()); if ( it == i->filters.end() ) { return false; } - i->filters[id->InternalInt()].DoCleanup(); + if ( i->filters[id->InternalInt()]->filter_type != TABLE_FILTER ) { + // wrong type; + return false; + } + delete (*it).second; + i->filters.erase(it); + return true; +} + +bool InputMgr::RemoveEventFilter(EnumVal* id, const string &name) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Reader not found"); + return false; + } + + map::iterator it = i->filters.find(id->InternalInt()); + if ( it == i->filters.end() ) { + return false; + } + + if ( i->filters[id->InternalInt()]->filter_type != EVENT_FILTER ) { + // wrong type; + return false; + } + + delete (*it).second; i->filters.erase(it); return true; } @@ -524,30 +650,53 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const return; } + if ( !i->HasFilter(id) ) { + reporter->InternalError("Unknown filter"); + return; + } + + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + SendEntryTable(reader, id, vals); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + } else { + assert(false); + } + +} + +void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + bool updated = false; + assert(i); assert(i->HasFilter(id)); + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + //reporter->Error("Hashing %d index fields", i->num_idx_fields); - HashKey* idxhash = HashLogVals(i->filters[id].num_idx_fields, vals); + HashKey* idxhash = HashLogVals(filter->num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(i->filters[id].num_val_fields, vals+i->filters[id].num_idx_fields); + HashKey* valhash = HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = i->filters[id].lastDict->Lookup(idxhash); + InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before if ( h->valhash->Hash() == valhash->Hash() ) { // ok, double. - i->filters[id].lastDict->Remove(idxhash); - i->filters[id].currDict->Insert(idxhash, h); + filter->lastDict->Remove(idxhash); + filter->currDict->Insert(idxhash, h); return; } else { // updated - i->filters[id].lastDict->Remove(idxhash); + filter->lastDict->Remove(idxhash); delete(h); updated = true; @@ -555,30 +704,25 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const } - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; - int position = i->filters[id].num_idx_fields; - if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[position], i->filters[id].rtype->FieldType(0)); + int position = filter->num_idx_fields; + if ( filter->num_val_fields == 1 && !filter->want_record ) { + valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { - RecordVal * r = new RecordVal(i->filters[id].rtype); + RecordVal * r = new RecordVal(filter->rtype); - for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { + for ( int j = 0; j < filter->rtype->NumFields(); j++) { Val* val = 0; - if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); + if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); + val = LogValToVal(vals[position], filter->rtype->FieldType(j)); position++; } - /* if ( val == 0 ) { - reporter->InternalError("conversion error"); - return; - } */ - r->Assign(j,val); } @@ -589,12 +733,12 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const Val* oldval = 0; if ( updated == true ) { // in that case, we need the old value to send the event (if we send an event). - oldval = i->filters[id].tab->Lookup(idxval); + oldval = filter->tab->Lookup(idxval); } // call filter first to determine if we really add / change the entry - if ( i->filters[id].pred ) { + if ( filter->pred ) { EnumVal* ev; Ref(idxval); Ref(valval); @@ -609,18 +753,18 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const vl.append(ev); vl.append(idxval); vl.append(valval); - Val* v = i->filters[id].pred->Call(&vl); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... - delete(i->filters[id].currDict->RemoveEntry(idxhash)); + delete(filter->currDict->RemoveEntry(idxhash)); return; } else { // keep old one - i->filters[id].currDict->Insert(idxhash, h); + filter->currDict->Insert(idxhash, h); return; } } @@ -629,24 +773,23 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const //i->tab->Assign(idxval, valval); - HashKey* k = i->filters[id].tab->ComputeHash(idxval); + HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); return; } - reporter->Error("assigning"); - i->filters[id].tab->Assign(idxval, k, valval); + filter->tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); - k = i->filters[id].tab->ComputeHash(idxval); + k = filter->tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; //i->tab->Delete(k); - i->filters[id].currDict->Insert(idxhash, ih); + filter->currDict->Insert(idxhash, ih); - if ( i->filters[id].event ) { + if ( filter->event ) { EnumVal* ev; Ref(idxval); @@ -654,11 +797,11 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); - SendEvent(i->filters[id].event, ev, idxval, oldval); + SendEvent(filter->event, 3, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(i->filters[id].event, ev, idxval, valval); + SendEvent(filter->event, 3, ev, idxval, valval); } } } @@ -673,25 +816,33 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { assert(i->HasFilter(id)); + if ( i->filters[id]->filter_type == EVENT_FILTER ) { + // nothing to do.. + return; + } + + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + // lastdict contains all deleted entries and should be empty apart from that - IterCookie *c = i->filters[id].lastDict->InitForIteration(); - i->filters[id].lastDict->MakeRobustCookie(c); + IterCookie *c = filter->lastDict->InitForIteration(); + filter->lastDict->MakeRobustCookie(c); InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { + while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) { ListVal * idx; Val *val; - if ( i->filters[id].pred || i->filters[id].event ) { - idx = i->filters[id].tab->RecoverIndex(ih->idxkey); + if ( filter->pred || filter->event ) { + idx = filter->tab->RecoverIndex(ih->idxkey); assert(idx != 0); - val = i->filters[id].tab->Lookup(idx); + val = filter->tab->Lookup(idx); assert(val != 0); } - if ( i->filters[id].pred ) { + if ( filter->pred ) { bool doBreak = false; // ask predicate, if we want to expire this element... @@ -704,37 +855,37 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { vl.append(ev); vl.append(idx); vl.append(val); - Val* v = i->filters[id].pred->Call(&vl); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... - i->filters[id].currDict->Insert(lastDictIdxKey, i->filters[id].lastDict->RemoveEntry(lastDictIdxKey)); + filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); continue; } } - if ( i->filters[id].event ) { + if ( filter->event ) { Ref(idx); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(i->filters[id].event, ev, idx, val); + SendEvent(filter->event, 3, ev, idx, val); } - i->filters[id].tab->Delete(ih->idxkey); - i->filters[id].lastDict->Remove(lastDictIdxKey); // deletex in next line + filter->tab->Delete(ih->idxkey); + filter->lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->filters[id].lastDict->Clear(); // should be empty... but... well... who knows... - delete(i->filters[id].lastDict); + filter->lastDict->Clear(); // should be empty->->-> but->->-> well->->-> who knows->->-> + delete(filter->lastDict); - i->filters[id].lastDict = i->filters[id].currDict; - i->filters[id].currDict = new PDict(InputHash); + filter->lastDict = filter->currDict; + filter->currDict = new PDict(InputHash); } void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { @@ -744,24 +895,86 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) return; } + if ( !i->HasFilter(id) ) { + reporter->InternalError("Unknown filter"); + return; + } + + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + PutTable(reader, id, vals); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + } else { + assert(false); + } + +} + +void InputMgr::SendEventFilterEvent(const InputReader* reader, EnumVal* type, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + + bool updated = false; + + assert(i); assert(i->HasFilter(id)); - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + assert(i->filters[id]->filter_type == EVENT_FILTER); + EventFilter* filter = (EventFilter*) i->filters[id]; + + Val *val; + list out_vals; + // no tracking, send everything with a new event... + //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); + out_vals.push_back(type); + + int position = 0; + if ( filter->want_record ) { + RecordVal * r = LogValToRecordVal(vals, filter->fields, &position); + out_vals.push_back(r); + + } else { + for ( int j = 0; j < filter->fields->NumFields(); j++) { + Val* val = 0; + if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->fields->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], filter->fields->FieldType(j)); + position++; + } + out_vals.push_back(val); + } + } + + SendEvent(filter->event, out_vals); + +} + +void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + + assert(i); + assert(i->HasFilter(id)); + + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; - int position = i->filters[id].num_idx_fields; - if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); + int position = filter->num_idx_fields; + if ( filter->num_val_fields == 1 && !filter->want_record ) { + valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->filters[id].rtype); + RecordVal * r = new RecordVal(filter->rtype); - for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { + for ( int j = 0; j < filter->rtype->NumFields(); j++) { Val* val = 0; - if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); + if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); + val = LogValToVal(vals[position], filter->rtype->FieldType(j)); position++; } @@ -776,7 +989,7 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) valval = r; } - i->filters[id].tab->Assign(idxval, valval); + filter->tab->Assign(idxval, valval); } void InputMgr::Clear(const InputReader* reader, int id) { @@ -788,7 +1001,10 @@ void InputMgr::Clear(const InputReader* reader, int id) { assert(i->HasFilter(id)); - i->filters[id].tab->RemoveAll(); + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + + filter->tab->RemoveAll(); } bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *vals) { @@ -800,9 +1016,18 @@ bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *va assert(i->HasFilter(id)); - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); - - return ( i->filters[id].tab->Delete(idxval) != 0 ); + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + TableFilter* filter = (TableFilter*) i->filters[id]; + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); + return( filter->tab->Delete(idxval) != 0 ); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + return true; + } else { + assert(false); + return false; + } } void InputMgr::Error(InputReader* reader, const char* msg) @@ -835,25 +1060,35 @@ bool InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c return true; } -void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) +void InputMgr::SendEvent(EventHandlerPtr ev, const int numvals, ...) { - //EventHandler* handler = event_registry->Lookup(name.c_str()); - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", name.c_str()); - // return; - //} - val_list* vl = new val_list; - vl->append(event); - vl->append(left); - vl->append(right); + + va_list lP; + va_start(lP, numvals); + for ( int i = 0; i < numvals; i++ ) + { + vl->append( va_arg(lP, Val*) ); + } + va_end(lP); + + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); +} + +void InputMgr::SendEvent(EventHandlerPtr ev, list events) +{ + val_list* vl = new val_list; + + for ( list::iterator i = events.begin(); i != events.end(); i++ ) { + vl->append( *i ); + } - //mgr.Dispatch(new Event(handler, vl)); mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } -Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { + +RecordVal* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { reporter->InternalError("Need position"); return 0; diff --git a/src/InputMgr.h b/src/InputMgr.h index ba1ddafc92..cebed231e4 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -27,6 +27,8 @@ public: bool AddTableFilter(EnumVal *id, RecordVal* filter); bool RemoveTableFilter(EnumVal* id, const string &name); + bool AddEventFilter(EnumVal *id, RecordVal* filter); + bool RemoveEventFilter(EnumVal* id, const string &name); protected: friend class InputReader; @@ -46,10 +48,17 @@ protected: private: struct ReaderInfo; + void SendEntryTable(const InputReader* reader, int id, const LogVal* const *vals); + void PutTable(const InputReader* reader, int id, const LogVal* const *vals); + void SendEventFilterEvent(const InputReader* reader, EnumVal* type, int id, const LogVal* const *vals); + bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); - void SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right); + + void SendEvent(EventHandlerPtr ev, const int numvals, ...); + void SendEvent(EventHandlerPtr ev, list events); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); int GetLogValLength(const LogVal* val); @@ -57,9 +66,8 @@ private: Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); - Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); @@ -68,7 +76,12 @@ private: string Hash(const string &input); - struct Filter; + class Filter; + class TableFilter; + class EventFilter; + + enum FilterType { TABLE_FILTER, EVENT_FILTER }; + }; diff --git a/src/input.bif b/src/input.bif index 1300f91bea..b1d57b1df6 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,6 +9,7 @@ module Input; type StreamDescription: record; type TableFilter: record; +type EventFilter: record; function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ @@ -40,6 +41,18 @@ function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL); %} +function Input::__add_eventfilter%(id: Log::ID, filter: Input::EventFilter%) : bool + %{ + bool res = input_mgr->AddEventFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_eventfilter%(id: Log::ID, name: string%) : bool + %{ + bool res = input_mgr->RemoveEventFilter(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL); + %} + # Options for Ascii Reader module InputAscii; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out new file mode 100644 index 0000000000..e32a2aea00 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -0,0 +1,21 @@ +Input::EVENT_NEW +1 +T +Input::EVENT_NEW +2 +T +Input::EVENT_NEW +3 +F +Input::EVENT_NEW +4 +F +Input::EVENT_NEW +5 +F +Input::EVENT_NEW +6 +F +Input::EVENT_NEW +7 +T diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index 36e8171689..d9be733a1b 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -16,7 +16,6 @@ 7 T @TEST-END-FILE -redef InputAscii::empty_field = "EMPTY"; module A; @@ -24,25 +23,20 @@ export { redef enum Log::ID += { LOG }; } -type Idx: record { - i: int; -}; - type Val: record { + i: int; b: bool; }; -global destination: table[int] of Val = table(); - -event line(tpe: Input::Event, left: Idx, right: bool) { +event line(tpe: Input::Event, i: int, b: bool) { print tpe; - print left; - print right; + print i; + print b; } event bro_init() { Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::add_eventfilter(A::LOG, [$name="input", $fields=Val, $ev=line]); Input::force_update(A::LOG); } diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro new file mode 100644 index 0000000000..36e8171689 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -0,0 +1,48 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +event line(tpe: Input::Event, left: Idx, right: bool) { + print tpe; + print left; + print right; +} + +event bro_init() +{ + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::force_update(A::LOG); +} From be1b3ce5e1f239861013517b8d8c5a5fafbdf0bd Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 28 Nov 2011 13:29:02 -0600 Subject: [PATCH 040/149] Add note about independent component releases to Broxygen index. --- doc/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/index.rst b/doc/index.rst index ba3df81e7d..0a6b225431 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -46,6 +46,12 @@ Script Reference Other Bro Components -------------------- +The following are snapshots of documentation for components that come +with this version of Bro (|version|). Since they can also be used +independently, see the `download page +`_ for documentation of any +current, independent component releases. + .. toctree:: :maxdepth: 1 From 4975584e01af095cf6a6d10a4c09eb88a3f198e0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 28 Nov 2011 13:28:44 -0800 Subject: [PATCH 041/149] change Log enum to Input enum. --- scripts/base/frameworks/input/main.bro | 10 +++++----- src/input.bif | 10 +++++----- src/types.bif | 1 - testing/btest/scripts/base/frameworks/input/basic.bro | 8 ++++---- testing/btest/scripts/base/frameworks/input/event.bro | 8 ++++---- .../base/frameworks/input/onecolumn-norecord.bro | 8 ++++---- .../scripts/base/frameworks/input/onecolumn-record.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/predicate.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/twofilters.bro | 10 +++++----- 9 files changed, 35 insertions(+), 36 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index a036eeeebd..1a9e8b885c 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -58,28 +58,28 @@ module Input; #global filters: table[ID, string] of Filter; -function create_stream(id: Log::ID, description: Input::StreamDescription) : bool +function create_stream(id: Input::ID, description: Input::StreamDescription) : bool { return __create_stream(id, description); } -function remove_stream(id: Log::ID) : bool +function remove_stream(id: Input::ID) : bool { return __remove_stream(id); } -function force_update(id: Log::ID) : bool +function force_update(id: Input::ID) : bool { return __force_update(id); } -function add_tablefilter(id: Log::ID, filter: Input::TableFilter) : bool +function add_tablefilter(id: Input::ID, filter: Input::TableFilter) : bool { # filters[id, filter$name] = filter; return __add_tablefilter(id, filter); } -function remove_tablefilter(id: Log::ID, name: string) : bool +function remove_tablefilter(id: Input::ID, name: string) : bool { # delete filters[id, name]; return __remove_tablefilter(id, name); diff --git a/src/input.bif b/src/input.bif index b1d57b1df6..a7d561c060 100644 --- a/src/input.bif +++ b/src/input.bif @@ -11,31 +11,31 @@ type StreamDescription: record; type TableFilter: record; type EventFilter: record; -function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool +function Input::__create_stream%(id: Input::ID, description: Input::StreamDescription%) : bool %{ InputReader *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__remove_stream%(id: Log::ID%) : bool +function Input::__remove_stream%(id: Input::ID%) : bool %{ bool res = input_mgr->RemoveStream(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__force_update%(id: Log::ID%) : bool +function Input::__force_update%(id: Input::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_tablefilter%(id: Log::ID, filter: Input::TableFilter%) : bool +function Input::__add_tablefilter%(id: Input::ID, filter: Input::TableFilter%) : bool %{ bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool +function Input::__remove_tablefilter%(id: Input::ID, name: string%) : bool %{ bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); diff --git a/src/types.bif b/src/types.bif index f90a954224..15fab1a7b1 100644 --- a/src/types.bif +++ b/src/types.bif @@ -180,7 +180,6 @@ enum Event %{ EVENT_REMOVED, %} - enum ID %{ Unknown, %} diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 3ad45eac69..10cc7376a8 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -45,8 +45,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index d9be733a1b..a07f0934a0 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -20,7 +20,7 @@ module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Val: record { @@ -36,7 +36,7 @@ event line(tpe: Input::Event, i: int, b: bool) { event bro_init() { - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_eventfilter(A::LOG, [$name="input", $fields=Val, $ev=line]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); + Input::force_update(A::INPUT); } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 134ceb49e6..88838cc8d6 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -31,8 +31,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index c07c9c826c..fc4d862cd3 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -31,8 +31,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 769536d2a6..5e6bae7b62 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -21,7 +21,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -37,11 +37,11 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); - Input::force_update(A::LOG); + Input::force_update(A::INPUT); if ( 1 in servers ) { print "VALID"; } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro index 575665e6e5..5af664e0e9 100644 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -21,7 +21,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -38,13 +38,13 @@ global destination2: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); - Input::add_tablefilter(A::LOG, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); + Input::add_tablefilter(A::INPUT, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); - Input::force_update(A::LOG); + Input::force_update(A::INPUT); if ( 1 in destination1 ) { print "VALID"; } From 1abb1424b8f1e050f24ffb9a9a7829fd8e10c3f0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 28 Nov 2011 14:15:04 -0800 Subject: [PATCH 042/149] begin documenting... --- doc/index.rst | 1 + doc/input.rst | 190 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 doc/input.rst diff --git a/doc/index.rst b/doc/index.rst index 0a6b225431..975ee7def4 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -20,6 +20,7 @@ Frameworks notice logging + input cluster signatures diff --git a/doc/input.rst b/doc/input.rst new file mode 100644 index 0000000000..78e96fe06e --- /dev/null +++ b/doc/input.rst @@ -0,0 +1,190 @@ +===================== +Loading Data into Bro +===================== + +.. rst-class:: opening + + Bro comes with a flexible input interface that allows to read + previously stored data. Data is either read into bro tables or + sent to scripts using events. + This document describes how the input framework can be used. + +.. contents:: + +Terminology +=========== + +Bro's input framework is built around three main abstracts, that are +very similar to the abstracts used in the logging framework: + + Input Streams + An input stream corresponds to a single input source + (usually a textfile). It defined the information necessary + to find the source (e.g. the filename) + + Filters + Each input stream has a set of filters attached to it, that + determine exaclty what kind of information is read. + There are two different kind of streams, event streams and table + streams. + By default, event streams generate an event for each line read + from the input source. + Table streams on the other hand read the input source in a bro + table for easy later access. + + Readers + A reader defines the input format for the specific input stream. + At the moment, Bro comes with only one type of reader, which can + read the tab seperated ASCII logfiles that were generated by the + logging framework. + + +Basics +====== + +For examples, please look at the unit tests in +``testing/btest/scripts/base/frameworks/input/``. + +A very basic example to open an input stream is: + +.. code:: bro + + module Foo; + + export { + # Create an ID for our new stream + redef enum Input::ID += { INPUT }; + } + + event bro_init() { + Input::create_stream(FOO::INPUT, [$source="input.log"]); + } + +The fields that can be set when creating a stream are: + + ``source`` + A mandatory string identifying the source of the data. + For the ASCII reader this is the filename. + + ``reader`` + The reader used for this stream. Default is ``READER_ASCII``. + + +Filters +======= + +Each filter defines the data fields that it wants to receive from the respective +input file. Depending on the type of filter, events or a table are created from +the data in the source file. + +Event Filters +------------- + +Event filters are filters that generate an event for each line in of the input source. + +For example, a simple filter retrieving the fields ``i`` and ``b`` from an inputSource +could be defined as follows: + +.. code:: bro + + type Val: record { + i: int; + b: bool; + }; + + event line(tpe: Input::Event, i: int, b: bool) { + # work with event data + } + + event bro_init { + # Input stream definition, etc + ... + + Input::add_eventfilter(Foo::INPUT, [$name="input", $fields=Val, $ev=line]); + + # read the file after all filters have been set + Input::force_update(Foo::INPUT); + } + +The fields that can be set for an event filter are: + + ``name`` + A mandatory name for the filter that can later be used + to manipulate it further. + + ``fields`` + Name of a record type containing the fields, which should be retrieved from + the input stream. + + ``ev`` + The event which is fired, after a line has been read from the input source. + The first argument that is passed to the event is an Input::Event structure, + followed by the data, either inside of a record (if ``want_record is set``) or as + individual fields. + The Input::Event structure can contain information, if the received line is ``NEW``, has + been ``CHANGED`` or ``DELETED``. Singe the ascii reader cannot track this information + for event filters, the value is always ``NEW`` at the moment. + + ``want_record`` + Boolean value, that defines if the event wants to receive the fields inside of + a single record value, or individually (default). + +Table Filters +------------- + +Table filters are the second, more complex type of filter. + +Table filters store the information they read from an input source in a bro table. For example, +when reading a file that contains ip addresses and connection attemt information one could use +an approach similar to this: + +.. code:: bro + + type Idx: record { + a: addr; + }; + + type Val: record { + tries: count; + }; + + global conn_attempts: table[addr] of count = table(); + + event bro_init { + # Input stream definitions, etc. + ... + + Input::add_tablefilter(Foo::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=conn_attempts]); + + # read the file after all filters have been set + Input::force_update(Foo::INPUT); + } + +The table conn_attempts will then contain the information about connection attemps. + +The possible fields that can be set for an table filter are: + + ``name`` + A mandatory name for the filter that can later be used + to manipulate it further. + + ``idx`` + Record type that defines the index of the table + + ``val`` + Record type that defines the values of the table + + ``want_record`` + Defines if the values of the table should be stored as a record (default), + or as a simple value. Has to be set if Val contains more than one element. + + ``destination`` + The destination table + + ``ev`` + Optional event that is raised, when values are added to, changed in or deleted from the table. + Events are passed an Input::Event description as the first argument, the index record as the second argument + and the values as the third argument. + + ``pred`` + Optional predicate, that can prevent entries from being added to the table and events from being sent. From 2a6387129ceb68941beccb866822958c661ab2c4 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 29 Nov 2011 11:25:11 -0800 Subject: [PATCH 043/149] documentation --- doc/scripts/DocSourcesList.cmake | 3 + scripts/base/frameworks/input/main.bro | 96 ++++++++++++++++++++------ 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 9d99effc02..bd41d301c0 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal) rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro) @@ -31,6 +32,8 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro) rest_target(${psd} base/frameworks/communication/main.bro) rest_target(${psd} base/frameworks/control/main.bro) rest_target(${psd} base/frameworks/dpd/main.bro) +rest_target(${psd} base/frameworks/input/main.bro) +rest_target(${psd} base/frameworks/input/readers/ascii.bro) rest_target(${psd} base/frameworks/intel/main.bro) rest_target(${psd} base/frameworks/logging/main.bro) rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 1a9e8b885c..66b13743b8 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -1,52 +1,106 @@ +##! The input framework provides a way to read previously stored data either +##! as an event stream or into a bro table. module Input; export { + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; + ## Stream decription type used for the `create_stream` method type StreamDescription: record { + ## String that allows the reader to find the source. + ## For `READER_ASCII`, this is the filename. source: string; + + ## Reader to use for this steam reader: Reader &default=default_reader; }; + ## TableFilter description type used for the `add_tablefilter` method. type TableFilter: record { - ## descriptive name. for later removal + ## Descriptive name. Used to remove a filter at a later time name: string; - ## for tables - idx: any; - val: any; + ## Table which will contain the data read by the input framework destination: any; + ## Record that defines the values used as the index of the table + idx: any; + ## Record that defines the values used as the values of the table + val: any; + ## Defines if the value of the table is a record (default), or a single value. + ## Val can only contain one element when this is set to false. want_record: bool &default=T; + + ## The event that is raised each time a value is added to, changed in or removed from the table. + ## The event will receive an Input::Event enum as the first argument, the idx record as the second argument + ## and the value (record) as the third argument. ev: any &optional; # event containing idx, val as values. - ## decision function, that decides if an insertion, update or removal should really be executed. - ## or events should be thought + ## Predicate function, that can decide if an insertion, update or removal should really be executed. + ## Parameters are the same as for the event. If true is returned, the update is performed. If false + ## is returned, it is skipped pred: function(typ: Input::Event, left: any, right: any): bool &optional; }; + ## EventFilter description type used for the `add_eventfilter` method. type EventFilter: record { - ## descriptive name. for later removal + ## Descriptive name. Used to remove a filter at a later time name: string; - # the event - ev: any; - # record describing the fields + ## Record describing the fields to be retrieved from the source input. fields: any; - - # does the event want the field unrolled (default) or as a simple record value? + ## If want_record if false (default), the event receives each value in fields as a seperate argument. + ## If it is set to true, the event receives all fields in a signle record value. want_record: bool &default=F; + + ## The event that is rised each time a new line is received from the reader. + ## The event will receive an Input::Event enum as the first element, and the fields as the following arguments. + ev: any; + }; #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; - global remove_stream: function(id: Log::ID) : bool; - global force_update: function(id: Log::ID) : bool; - global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; - global remove_tablefilter: function(id: Log::ID, name: string) : bool; - global add_eventfilter: function(id: Log::ID, filter: Input::EventFilter) : bool; - global remove_eventfilter: function(id: Log::ID, name: string) : bool; + ## Create a new input stream from a given source. Returns true on success. + ## + ## id: `Input::ID` enum value identifying this stream + ## description: `StreamDescription` record describing the source. + global create_stream: function(id: Input::ID, description: Input::StreamDescription) : bool; + + ## Remove a current input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream to be removed + global remove_stream: function(id: Input::ID) : bool; + + ## Forces the current input to be checked for changes. + ## + ## id: `Input::ID` enum value identifying the stream + global force_update: function(id: Input::ID) : bool; + + ## Adds a table filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## filter: the `TableFilter` record describing the filter. + global add_tablefilter: function(id: Input::ID, filter: Input::TableFilter) : bool; + + ## Removes a named table filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## name: the name of the filter to be removed. + global remove_tablefilter: function(id: Input::ID, name: string) : bool; + + ## Adds an event filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## filter: the `EventFilter` record describing the filter. + global add_eventfilter: function(id: Input::ID, filter: Input::EventFilter) : bool; + + ## Removes a named event filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## name: the name of the filter to be removed. + global remove_eventfilter: function(id: Input::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; } @@ -85,13 +139,13 @@ function remove_tablefilter(id: Input::ID, name: string) : bool return __remove_tablefilter(id, name); } -function add_eventfilter(id: Log::ID, filter: Input::EventFilter) : bool +function add_eventfilter(id: Input::ID, filter: Input::EventFilter) : bool { # filters[id, filter$name] = filter; return __add_eventfilter(id, filter); } -function remove_eventfilter(id: Log::ID, name: string) : bool +function remove_eventfilter(id: Input::ID, name: string) : bool { # delete filters[id, name]; return __remove_eventfilter(id, name); From a68e6b9fa49211a7c51c45f83ec0c610f03a956d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 29 Nov 2011 14:32:53 -0800 Subject: [PATCH 044/149] allow sets to be read from files, convenience function for reading a file once, bug in destructor that could lead to a segfault. --- scripts/base/frameworks/input/main.bro | 34 +++++++++++- src/InputMgr.cc | 54 ++++++++++++++----- .../scripts/base/frameworks/input/basic.bro | 2 + 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 66b13743b8..c76eba80b9 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,6 +4,8 @@ module Input; export { + redef enum Input::ID += { TABLE_READ }; + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; @@ -27,7 +29,8 @@ export { ## Record that defines the values used as the index of the table idx: any; ## Record that defines the values used as the values of the table - val: any; + ## If val is undefined, destination has to be a set. + val: any &optional; ## Defines if the value of the table is a record (default), or a single value. ## Val can only contain one element when this is set to false. want_record: bool &default=T; @@ -102,6 +105,14 @@ export { ## name: the name of the filter to be removed. global remove_eventfilter: function(id: Input::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; + + ## Convenience function for reading a specific input source exactly once using + ## exactly one tablefilter + ## + ## id: `Input::ID` enum value identifying the stream + ## description: `StreamDescription` record describing the source. + ## filter: the `TableFilter` record describing the filter. + global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; } @@ -151,6 +162,27 @@ function remove_eventfilter(id: Input::ID, name: string) : bool return __remove_eventfilter(id, name); } +function read_table(description: Input::StreamDescription, filter: Input::TableFilter) : bool { + local ok: bool = T; + # since we create and delete it ourselves this should be ok... at least for singlethreaded operation + local id: Input::ID = Input::TABLE_READ; + + ok = create_stream(id, description); + if ( ok ) { + ok = add_tablefilter(id, filter); + } + if ( ok ) { + ok = force_update(id); + } + if ( ok ) { + ok = remove_stream(id); + } else { + remove_stream(id); + } + + return ok; +} + #function get_filter(id: ID, name: string) : Filter # { # if ( [id, name] in filters ) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0df11ea359..7ec5974199 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -86,7 +86,8 @@ InputMgr::Filter::~Filter() { InputMgr::TableFilter::~TableFilter() { Unref(tab); Unref(itype); - Unref(rtype); + if ( rtype ) // can be 0 for sets + Unref(rtype); delete currDict; delete lastDict; @@ -110,6 +111,7 @@ InputMgr::ReaderInfo::~ReaderInfo() { while ( it != filters.end() ) { delete (*it).second; + ++it; } Unref(type); @@ -354,7 +356,10 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { Val* pred = fval->Lookup(rtype->FieldOffset("pred")); RecordType *idx = fval->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); - RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = 0; + if ( fval->Lookup(rtype->FieldOffset("val")) != 0 ) { + val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + } TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); @@ -408,9 +413,14 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { int idxfields = fieldsV.size(); - status = status || !UnrollRecordType(&fieldsV, val, ""); + if ( val ) // if we are not a set + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + if ( !val ) + assert(valfields == 0); + if ( status ) { reporter->Error("Problem unrolling"); return false; @@ -429,7 +439,7 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { filter->num_idx_fields = idxfields; filter->num_val_fields = valfields; filter->tab = dst->Ref()->AsTableVal(); - filter->rtype = val->Ref()->AsRecordType(); + filter->rtype = val ? val->Ref()->AsRecordType() : 0; filter->itype = idx->Ref()->AsRecordType(); filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter->currDict = new PDict(InputHash); @@ -681,7 +691,10 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c HashKey* idxhash = HashLogVals(filter->num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); + HashKey* valhash = 0; + if ( filter->num_val_fields > 0 ) + HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); + //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); @@ -689,12 +702,13 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before - if ( h->valhash->Hash() == valhash->Hash() ) { - // ok, double. + if ( filter->num_val_fields == 0 || h->valhash->Hash() == valhash->Hash() ) { + // ok, exact duplicate filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); return; } else { + assert( filter->num_val_fields > 0 ); // updated filter->lastDict->Remove(idxhash); delete(h); @@ -708,7 +722,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Val* valval; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 1 && !filter->want_record ) { + if ( filter->num_val_fields == 0 ) { + valval = 0; + } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { RecordVal * r = new RecordVal(filter->rtype); @@ -732,8 +748,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Val* oldval = 0; if ( updated == true ) { - // in that case, we need the old value to send the event (if we send an event). - oldval = filter->tab->Lookup(idxval); + assert(filter->num_val_fields > 0); + // in that case, we need the old value to send the event (if we send an event). + oldval = filter->tab->Lookup(idxval); } @@ -749,10 +766,12 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); } - val_list vl(3); + val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. vl.append(ev); vl.append(idxval); - vl.append(valval); + if ( filter->num_val_fields > 0 ) + vl.append(valval); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); @@ -794,6 +813,7 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Ref(idxval); if ( updated ) { // in case of update send back the old value. + assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); @@ -801,7 +821,11 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(filter->event, 3, ev, idxval, valval); + if ( filter->num_val_fields == 0 ) { + SendEvent(filter->event, 3, ev, idxval); + } else { + SendEvent(filter->event, 3, ev, idxval, valval); + } } } } @@ -963,7 +987,9 @@ void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const * Val* valval; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 1 && !filter->want_record ) { + if ( filter->num_val_fields == 0 ) { + valval = 0; + } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { RecordVal * r = new RecordVal(filter->rtype); diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 10cc7376a8..d1b6659eb6 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -49,4 +49,6 @@ event bro_init() Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); print servers; + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); } From 78b24da7e4f84de7f499cc9365a30e7c966a7d14 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 15:02:03 -0800 Subject: [PATCH 045/149] start support for annotation for log field types. commit before rolling part of it back... --- src/Attr.h | 1 + src/InputMgr.cc | 10 ++++++++++ src/InputReaderAscii.h | 3 +++ src/LogMgr.cc | 8 +++++--- src/LogMgr.h | 6 ++++-- src/parse.y | 5 ++++- src/scan.l | 1 + 7 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/Attr.h b/src/Attr.h index 6c835dc61c..471acfe4ba 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -35,6 +35,7 @@ typedef enum { ATTR_GROUP, ATTR_LOG, ATTR_ERROR_HANDLER, + ATTR_TYPE_COLUMN, // for input framework ATTR_TRACKED, // hidden attribute, tracked by NotifierRegistry #define NUM_ATTRS (int(ATTR_TRACKED) + 1) } attr_tag; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 7ec5974199..0cfb59ee90 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -561,6 +561,16 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); } else if ( field->type == TYPE_VECTOR ) { field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); + } else if ( field->type == TYPE_PORT && + rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) { + // we have an annotation for the second column + + Val* c = rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN)->AttrExpr()->Eval(0); + + assert(c); + assert(c->Type()->Tag() == TYPE_STRING); + + field->secondary_name = c->AsStringVal()->AsString()->CheckString(); } fields->push_back(field); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 01169a3cfc..c174248454 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,8 +12,11 @@ struct FieldMapping { string name; TypeTag type; + // internal type for sets and vectors TypeTag subtype; int position; + // for ports: pos of the second field + int secondary_position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 7fc8d4ef86..dd4f9c8e5a 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -83,7 +83,8 @@ bool LogField::Read(SerializationFormat* fmt) int t; int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); + bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && + fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); type = (TypeTag) t; subtype = (TypeTag) it; @@ -92,7 +93,8 @@ bool LogField::Read(SerializationFormat* fmt) bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); + return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && + fmt->Write((int)subtype, "subtype")); } LogVal::~LogVal() @@ -151,7 +153,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) if ( ! t->IsSet() ) return false; - return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); } case TYPE_VECTOR: diff --git a/src/LogMgr.h b/src/LogMgr.h index b8530d29ab..4ccdeb793c 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -14,13 +14,15 @@ class SerializationFormat; // Description of a log field. struct LogField { string name; + // needed by input framework. port fields have two names (one for the port, one for the type) - this specifies the secondary name. + string secondary_name; TypeTag type; - // needed by input framework. otherwise it cannot determine the inner type of a set or vector. + // needed by input framework. otherwise it cannot determine the inner type of a set. TypeTag subtype; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type), subtype(other.subtype) { } + : name(other.name), secondary_name(other.secondary_name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); diff --git a/src/parse.y b/src/parse.y index 495931aae0..988a19714b 100644 --- a/src/parse.y +++ b/src/parse.y @@ -2,7 +2,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 88 +%expect 91 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -24,6 +24,7 @@ %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED %token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE %token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER +%token TOK_ATTR_TYPE_COLUMN %token TOK_DEBUG @@ -1313,6 +1314,8 @@ attr: { $$ = new Attr(ATTR_PRIORITY, $3); } | TOK_ATTR_GROUP '=' expr { $$ = new Attr(ATTR_GROUP, $3); } + | TOK_ATTR_TYPE_COLUMN '=' expr + { $$ = new Attr(ATTR_TYPE_COLUMN, $3); } | TOK_ATTR_LOG { $$ = new Attr(ATTR_LOG); } | TOK_ATTR_ERROR_HANDLER diff --git a/src/scan.l b/src/scan.l index 7ebd7894e1..6a85e89780 100644 --- a/src/scan.l +++ b/src/scan.l @@ -308,6 +308,7 @@ when return TOK_WHEN; &optional return TOK_ATTR_OPTIONAL; &persistent return TOK_ATTR_PERSISTENT; &priority return TOK_ATTR_PRIORITY; +&type_column return TOK_ATTR_TYPE_COLUMN; &read_expire return TOK_ATTR_EXPIRE_READ; &redef return TOK_ATTR_REDEF; &rotate_interval return TOK_ATTR_ROTATE_INTERVAL; From aecbbdd966c896b475591a3415578adc1629273e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 16:18:54 -0800 Subject: [PATCH 046/149] make logging framework send the protocol to the writer. for use in future writers, that have a special type for port, which includes the protocol. --- src/LogMgr.cc | 32 +++++++++++++++++++++++++++++--- src/LogMgr.h | 4 ++++ src/LogWriterAscii.cc | 5 ++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index dd4f9c8e5a..ed32e4e40b 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -118,6 +118,10 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } + +// if ( type == TYPE_PORT && present ) +// delete val.port_val.proto; + } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -190,9 +194,12 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Read(&val.uint_val, "uint"); + case TYPE_PORT: + val.port_val.proto = new string; + return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -305,9 +312,11 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Write(val.uint_val, "uint"); + case TYPE_PORT: + return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -1066,6 +1075,22 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } +string LogMgr::TransportProtoToString(TransportProto p) { + switch ( p ) { + case TRANSPORT_UNKNOWN: + return "unknown"; + case TRANSPORT_TCP: + return "tcp"; + case TRANSPORT_UDP: + return "udp"; + case TRANSPORT_ICMP: + return "icmp"; + } + + assert(false); + return ""; +} + LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1097,7 +1122,8 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; case TYPE_PORT: - lval->val.uint_val = val->AsPortVal()->Port(); + lval->val.port_val.port = val->AsPortVal()->Port(); + lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 4ccdeb793c..d9d61236e0 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -38,10 +38,12 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; + struct port_t { bro_uint_t port; string* proto; }; union _val { bro_int_t int_val; bro_uint_t uint_val; + port_t port_val; uint32 addr_val[NUM_ADDR_WORDS]; subnet_type subnet_val; double double_val; @@ -136,6 +138,8 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); + string TransportProtoToString(TransportProto p); + vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9b1fda3b62..c449c1a788 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -169,10 +169,13 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: desc->Add(val->val.uint_val); break; + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + case TYPE_SUBNET: desc->Add(dotted_addr(val->val.subnet_val.net)); desc->Add("/"); From 4a690484ecaf5ea086090d2b6ef855cc6c913cad Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:42:37 -0800 Subject: [PATCH 047/149] make port annotation work and ascii input reader way more rebust with better error messages. --- src/Attr.cc | 22 ++- src/InputMgr.cc | 44 ++++- src/InputMgr.h | 2 + src/InputReaderAscii.cc | 150 +++++++++--------- src/InputReaderAscii.h | 4 +- src/LogMgr.cc | 1 - .../scripts.base.frameworks.input.port/out | 6 + .../scripts/base/frameworks/input/port.bro | 39 +++++ 8 files changed, 187 insertions(+), 81 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.port/out create mode 100644 testing/btest/scripts/base/frameworks/input/port.bro diff --git a/src/Attr.cc b/src/Attr.cc index a5a350f452..1d610f7cb4 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -17,7 +17,7 @@ const char* attr_name(attr_tag t) "&persistent", "&synchronized", "&postprocessor", "&encrypt", "&match", "&disable_print_hook", "&raw_output", "&mergeable", "&priority", - "&group", "&log", "&error_handler", "(&tracked)", + "&group", "&log", "&error_handler", "&type_column", "(&tracked)", }; return attr_names[int(t)]; @@ -417,6 +417,26 @@ void Attributes::CheckAttr(Attr* a) Error("&log applied to a type that cannot be logged"); break; + case ATTR_TYPE_COLUMN: + { + if ( type->Tag() != TYPE_PORT ) + { + Error("type_column tag only applicable to ports"); + break; + } + + BroType* atype = a->AttrExpr()->Type(); + + if ( atype->Tag() != TYPE_STRING ) { + Error("type column needs to have a string argument"); + break; + } + + + break; + } + + default: BadTag("Attributes::CheckAttr", attr_name(a->Tag())); } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0cfb59ee90..612461bae8 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -1167,9 +1167,14 @@ int InputMgr::GetLogValLength(const LogVal* val) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: length += sizeof(val->val.uint_val); break; + + case TYPE_PORT: + length += sizeof(val->val.port_val.port); + if ( val->val.port_val.proto != 0 ) + length += val->val.port_val.proto->size(); + break; case TYPE_DOUBLE: case TYPE_TIME: @@ -1228,12 +1233,24 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: //*(data+startpos) = val->val.uint_val; memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); return sizeof(val->val.uint_val); break; + case TYPE_PORT: { + int length = 0; + memcpy(data+startpos, (const void*) &(val->val.port_val.port), sizeof(val->val.port_val.port)); + length += sizeof(val->val.port_val.port); + if ( val->val.port_val.proto != 0 ) { + memcpy(data+startpos, val->val.port_val.proto->c_str(), val->val.port_val.proto->length()); + length += val->val.port_val.proto->size(); + } + return length; + break; + } + + case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: @@ -1320,6 +1337,24 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } +TransportProto InputMgr::StringToProto(const string &proto) { + if ( proto == "unknown" ) { + return TRANSPORT_UNKNOWN; + } else if ( proto == "tcp" ) { + return TRANSPORT_TCP; + } else if ( proto == "udp" ) { + return TRANSPORT_UDP; + } else if ( proto == "icmp" ) { + return TRANSPORT_ICMP; + } + + //assert(false); + + reporter->Error("Tried to parse invalid/unknown protocol: %s", proto.c_str()); + + return TRANSPORT_UNKNOWN; +} + Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { @@ -1357,7 +1392,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { } case TYPE_PORT: - return new PortVal(val->val.uint_val); + if ( val->val.port_val.proto == 0 ) + return new PortVal(val->val.port_val.port); + else + return new PortVal(val->val.port_val.port, StringToProto(*val->val.port_val.proto)); break; case TYPE_ADDR: diff --git a/src/InputMgr.h b/src/InputMgr.h index cebed231e4..ba6e208fd1 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -68,6 +68,8 @@ private: Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + TransportProto StringToProto(const string &proto); + ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 84feb74e61..501022d58e 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -10,25 +10,27 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int : name(arg_name), type(arg_type) { position = arg_position; + secondary_position = -1; } FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) : name(arg_name), type(arg_type), subtype(arg_subtype) { position = arg_position; + secondary_position = -1; } FieldMapping::FieldMapping(const FieldMapping& arg) : name(arg.name), type(arg.type), subtype(arg.subtype) { position = arg.position; + secondary_position = arg.secondary_position; } FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } - InputReaderAscii::InputReaderAscii() { file = 0; @@ -122,45 +124,45 @@ bool InputReaderAscii::ReadHeader() { return false; } + map fields; + + // construcr list of field names. + istringstream splitstream(line); + int pos=0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0])) + break; + + fields[s] = pos; + pos++; + } + + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - // split on tabs... - istringstream splitstream(line); - unsigned int currTab = 0; - int wantFields = 0; - while ( splitstream ) { - string s; - if ( !getline(splitstream, s, separator[0])) - break; - // current found heading in s... compare if we want it - for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - const LogField* field = (*it).second.fields[i]; - if ( field->name == s ) { - // cool, found field. note position - FieldMapping f(field->name, field->type, field->subtype, i); - (*it).second.columnMap.push_back(f); - wantFields++; - break; // done with searching + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + const LogField* field = (*it).second.fields[i]; + + map::iterator fit = fields.find(field->name); + if ( fit == fields.end() ) { + Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); + return false; + } + + + FieldMapping f(field->name, field->type, field->subtype, fields[field->name]); + if ( field->secondary_name != "" ) { + map::iterator fit2 = fields.find(field->secondary_name); + if ( fit2 == fields.end() ) { + Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); + return false; } + f.secondary_position = fields[field->secondary_name]; } - - // look if we did push something... - if ( (*it).second.columnMap.size() == currTab ) { - // no, we didn't. note that... - FieldMapping empty; - (*it).second.columnMap.push_back(empty); - } - - // done - currTab++; - } - - if ( wantFields != (int) (*it).second.num_fields ) { - // we did not find all fields? - // :( - Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d. Filternum: %d", wantFields, (*it).second.num_fields, (*it).first)); - return false; + (*it).second.columnMap.push_back(f); } + } // well, that seems to have worked... @@ -220,10 +222,14 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: val->val.uint_val = atoi(s.c_str()); break; + case TYPE_PORT: + val->val.port_val.port = atoi(s.c_str()); + val->val.port_val.proto = 0; + break; + case TYPE_SUBNET: { int pos = s.find("/"); string width = s.substr(pos+1); @@ -346,59 +352,55 @@ bool InputReaderAscii::DoUpdate() { string line; while ( GetLine(line ) ) { + // split on tabs + istringstream splitstream(line); + + map stringfields; + int pos = 0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0]) ) + break; + + stringfields[pos] = s; + pos++; + } + + pos--; // for easy comparisons of max element. for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - // split on tabs - - istringstream splitstream(line); - LogVal** fields = new LogVal*[(*it).second.num_fields]; - //string string_fields[num_fields]; - unsigned int currTab = 0; - unsigned int currField = 0; - while ( splitstream ) { + int fpos = 0; + for ( vector::iterator fit = (*it).second.columnMap.begin(); + fit != (*it).second.columnMap.end(); + fit++ ){ - string s; - if ( !getline(splitstream, s, separator[0]) ) - break; - - - if ( currTab >= (*it).second.columnMap.size() ) { - Error("Tabs in heading do not match tabs in data?"); - //disabled = true; + if ( (*fit).position > pos || (*fit).secondary_position > pos ) { + Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); return false; } - FieldMapping currMapping = (*it).second.columnMap[currTab]; - currTab++; - - if ( currMapping.IsEmpty() ) { - // well, that was easy - continue; - } - - if ( currField >= (*it).second.num_fields ) { - Error("internal error - fieldnum greater as possible"); - return false; - } - - LogVal* val = EntryToVal(s, currMapping); + LogVal* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { return false; } - fields[currMapping.position] = val; - //string_fields[currMapping.position] = s; + + if ( (*fit).secondary_position != -1 ) { + // we have a port definition :) + assert(val->type == TYPE_PORT ); + // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - currField++; - } - - if ( currField != (*it).second.num_fields ) { - Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); - return false; + val->val.port_val.proto = new string(stringfields[(*fit).secondary_position]); + } + + fields[fpos] = val; + + fpos++; } + assert ( (unsigned int) fpos == (*it).second.num_fields ); SendEntry((*it).first, fields); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c174248454..2670d785d5 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -21,10 +21,10 @@ struct FieldMapping { FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); FieldMapping(const FieldMapping& arg); - FieldMapping() { position = -1; } + FieldMapping() { position = -1; secondary_position = -1; } FieldMapping subType(); - bool IsEmpty() { return position == -1; } + //bool IsEmpty() { return position == -1; } }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index ed32e4e40b..307a2e24e8 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -1088,7 +1088,6 @@ string LogMgr::TransportProtoToString(TransportProto p) { } assert(false); - return ""; } LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port/out b/testing/btest/Baseline/scripts.base.frameworks.input.port/out new file mode 100644 index 0000000000..6f2bd3271b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port/out @@ -0,0 +1,6 @@ +Trying to find field: t +{ +[1.2.3.4] = [p=80/tcp], +[1.2.4.6] = [p=30/unknown], +[1.2.3.5] = [p=52/udp] +} diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro new file mode 100644 index 0000000000..6f98c363c7 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -0,0 +1,39 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#fields i p t +1.2.3.4 80 tcp +1.2.3.5 52 udp +1.2.4.6 30 unknown +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Idx: record { + i: addr; +}; + +type Val: record { + p: port &type_column="t"; +}; + +global servers: table[addr] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); + print servers; + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); +} From 9f32f68a13f290f01747f02b05dbe4b8b63d2790 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:50:36 -0800 Subject: [PATCH 048/149] make test more robust. --- .../Baseline/scripts.base.frameworks.input.port/out | 9 +++------ testing/btest/scripts/base/frameworks/input/port.bro | 10 ++++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port/out b/testing/btest/Baseline/scripts.base.frameworks.input.port/out index 6f2bd3271b..858551aa2f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.port/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port/out @@ -1,6 +1,3 @@ -Trying to find field: t -{ -[1.2.3.4] = [p=80/tcp], -[1.2.4.6] = [p=30/unknown], -[1.2.3.5] = [p=52/udp] -} +[p=80/tcp] +[p=52/udp] +[p=30/unknown] diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 6f98c363c7..c14892ae36 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -6,7 +6,7 @@ #fields i p t 1.2.3.4 80 tcp 1.2.3.5 52 udp -1.2.4.6 30 unknown +1.2.3.6 30 unknown @TEST-END-FILE redef InputAscii::empty_field = "EMPTY"; @@ -31,9 +31,11 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); - print servers; - Input::remove_tablefilter(A::INPUT, "ssh"); + print servers[1.2.3.4]; + print servers[1.2.3.5]; + print servers[1.2.3.6]; + Input::remove_tablefilter(A::INPUT, "input"); Input::remove_stream(A::INPUT); } From eb64eeedcd27c543becf330d7da48db9aa541ea8 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:56:26 -0800 Subject: [PATCH 049/149] memleak fix. --- src/LogMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 307a2e24e8..dd5bc4b66a 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -119,8 +119,8 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } -// if ( type == TYPE_PORT && present ) -// delete val.port_val.proto; + if ( type == TYPE_PORT && present ) + delete val.port_val.proto; } From e0b7dc04512a228b332e2b9aaddd39300646d5cc Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 8 Dec 2011 14:12:59 -0800 Subject: [PATCH 050/149] fix compile warnings --- src/InputMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 00a1a26311..1fe3a82abe 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -866,8 +866,8 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { //while ( ( ih = i->lastDict->NextEntry(c) ) ) { while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) { - ListVal * idx; - Val *val; + ListVal * idx = 0; + Val *val = 0; if ( filter->pred || filter->event ) { idx = filter->tab->RecoverIndex(ih->idxkey); From a14ec02d3b75788966db7dccb1e8a533e627300a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 19 Dec 2011 12:43:25 -0800 Subject: [PATCH 051/149] change empty field defenition like in logging framework --- scripts/base/frameworks/input/readers/ascii.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro index 9f630975a2..14c04757f7 100644 --- a/scripts/base/frameworks/input/readers/ascii.bro +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -12,7 +12,7 @@ export { const set_separator = "," &redef; ## String to use for empty fields. - const empty_field = "-" &redef; + const empty_field = "(empty)" &redef; ## String to use for an unset &optional field. const unset_field = "-" &redef; From 70a2cf67324c607014b067af13fc9c6887e571e3 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 19 Dec 2011 12:43:51 -0800 Subject: [PATCH 052/149] update baseline to include input framework --- .../coverage.bare-load-baseline/canonified_loaded_scripts.log | 4 ++++ .../canonified_loaded_scripts.log | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 8fab67304e..5a0de92a6e 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -18,4 +18,8 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/__load__.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro scripts/base/frameworks/logging/./writers/ascii.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/./main.bro + build/src/base/input.bif.bro + scripts/base/frameworks/input/./readers/ascii.bro scripts/policy/misc/loaded-scripts.bro diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 3f77797df8..e727928579 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -18,6 +18,10 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/__load__.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro scripts/base/frameworks/logging/./writers/ascii.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/./main.bro + build/src/base/input.bif.bro + scripts/base/frameworks/input/./readers/ascii.bro scripts/base/init-default.bro scripts/base/utils/site.bro scripts/base/utils/./patterns.bro From eb53a3d1c8bb9b7a590fc4f5ba70a5ea5017b6ee Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 12 Jan 2012 11:51:12 -0800 Subject: [PATCH 053/149] make input framework compile with brov6 --- src/InputReaderAscii.cc | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 6da161a2bf..257cb4cf71 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -253,19 +253,37 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // NOTE: dotted_to_addr BREAKS THREAD SAFETY! it uses reporter. // Solve this some other time.... +#ifdef BROv6 + if ( s.find(':') != s.npos ) { + uint32* addr = dotted_to_addr6(s.c_str()); + copy_addr(val->val.subnet_val.net, addr); + delete addr; + } else { + val->val.subnet_val.net[0] = val->val.subnet_val.net[1] = val->val.subnet_val.net[2] = 0; + val->val.subnet_val.net[3] = dotted_to_addr(s.c_str()); + } +#else val->val.subnet_val.net = dotted_to_addr(s.c_str()); +#endif break; } case TYPE_ADDR: { // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. // Solve this some other time.... - addr_type t = dotted_to_addr(s.c_str()); #ifdef BROv6 - copy_addr(t, val->val.addr_val); + if ( s.find(':') != s.npos ) { + uint32* addr = dotted_to_addr6(s.c_str()); + copy_addr(val->val.addr_val, addr); + delete addr; + } else { + val->val.addr_val[0] = val->val.addr_val[1] = val->val.addr_val[2] = 0; + val->val.addr_val[3] = dotted_to_addr(s.c_str()); + } #else + uint32 t = dotted_to_addr(s.c_str()); copy_addr(&t, val->val.addr_val); #endif break; From ac1708f8436126d1be5b522461dc88c8e1fa6158 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 20 Jan 2012 12:33:48 -0800 Subject: [PATCH 054/149] fix handling of predicates - now the second argument that is sent to the predicate really is a recordVal and not a ListVal. --- src/InputMgr.cc | 52 ++++++++++++++++++++++++++++++------------------- src/InputMgr.h | 1 + 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 1fe3a82abe..0e01bd6333 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -737,22 +737,7 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { - RecordVal * r = new RecordVal(filter->rtype); - - for ( int j = 0; j < filter->rtype->NumFields(); j++) { - - Val* val = 0; - if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); - } else { - val = LogValToVal(vals[position], filter->rtype->FieldType(j)); - position++; - } - - r->Assign(j,val); - - } - valval = r; + valval = LogValToRecordVal(vals, filter->rtype, &position); } @@ -767,7 +752,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c // call filter first to determine if we really add / change the entry if ( filter->pred ) { EnumVal* ev; - Ref(idxval); + //Ref(idxval); + int startpos = 0; + Val* predidx = LogValToRecordVal(vals, filter->itype, &startpos); Ref(valval); if ( updated ) { @@ -778,7 +765,7 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. vl.append(ev); - vl.append(idxval); + vl.append(predidx); if ( filter->num_val_fields > 0 ) vl.append(valval); @@ -882,12 +869,14 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { // ask predicate, if we want to expire this element... EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - Ref(idx); + //Ref(idx); + int startpos = 0; + Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); Ref(val); val_list vl(3); vl.append(ev); - vl.append(idx); + vl.append(predidx); vl.append(val); Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); @@ -1123,6 +1112,29 @@ void InputMgr::SendEvent(EventHandlerPtr ev, list events) } +RecordVal* InputMgr::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) { + RecordVal* rec = new RecordVal(request_type->AsRecordType()); + + int maxpos = list->Length(); + + for ( int i = 0; i < request_type->NumFields(); i++ ) { + assert ( (*position) <= maxpos ); + + Val* fieldVal = 0; + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + fieldVal = ListValToRecordVal(list, request_type->FieldType(i)->AsRecordType(), position); + } else { + fieldVal = list->Index(*position); + (*position)++; + } + + rec->Assign(i, fieldVal); + } + + return rec; +} + + RecordVal* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { diff --git a/src/InputMgr.h b/src/InputMgr.h index 21af018a47..5623c15338 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -67,6 +67,7 @@ private: Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); From f24c50b49a88b22db612d098489eeac042122a76 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 20 Jan 2012 12:42:23 -0800 Subject: [PATCH 055/149] remove unnecessary stuff from function. --- src/InputMgr.cc | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0e01bd6333..89530febe6 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -955,7 +955,6 @@ void InputMgr::SendEventFilterEvent(const InputReader* reader, EnumVal* type, in if ( filter->want_record ) { RecordVal * r = LogValToRecordVal(vals, filter->fields, &position); out_vals.push_back(r); - } else { for ( int j = 0; j < filter->fields->NumFields(); j++) { Val* val = 0; @@ -991,27 +990,7 @@ void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const * } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { - RecordVal * r = new RecordVal(filter->rtype); - - for ( int j = 0; j < filter->rtype->NumFields(); j++) { - - Val* val = 0; - if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); - } else { - val = LogValToVal(vals[position], filter->rtype->FieldType(j)); - position++; - } - - if ( val == 0 ) { - reporter->InternalError("conversion error"); - return; - } - - r->Assign(j,val); - - } - valval = r; + valval = LogValToRecordVal(vals, filter->rtype, &position); } filter->tab->Assign(idxval, valval); From 833e7244000c1a1d13b67c9087ada12686bf4b98 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 6 Feb 2012 16:14:39 -0800 Subject: [PATCH 056/149] way less compile errors. --- src/input/Manager.cc | 191 +++++++++++++++------------- src/input/Manager.h | 6 +- src/input/ReaderBackend.cc | 95 +++++++++----- src/input/ReaderBackend.h | 23 ++-- src/input/ReaderFrontend.h | 54 ++++++++ src/input/readers/Ascii.h | 29 +++-- src/logging/Manager.cc | 2 +- src/threading/SerializationTypes.cc | 6 +- src/threading/SerializationTypes.h | 2 + 9 files changed, 257 insertions(+), 151 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 24bc464daf..a97286162d 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -3,18 +3,20 @@ #include #include "Manager.h" +#include "ReaderFrontend.h" +#include "ReaderBackend.h" +#include "readers/Ascii.h" + #include "Event.h" #include "EventHandler.h" #include "NetVar.h" #include "Net.h" -#include "InputReader.h" - -#include "InputReaderAscii.h" - #include "CompHash.h" +#include "../threading/SerializationTypes.h" + using namespace input; using threading::Value; using threading::Field; @@ -99,7 +101,7 @@ Manager::TableFilter::~TableFilter() { struct Manager::ReaderInfo { EnumVal* id; EnumVal* type; - InputReader* reader; + ReaderFrontend* reader; //list events; // events we fire when "something" happens map filters; // filters that can prevent our actions @@ -132,38 +134,27 @@ bool Manager::ReaderInfo::HasFilter(int id) { } -struct InputReaderDefinition { +struct ReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages bool (*init)(); // optional one-time inifializing function - InputReader* (*factory)(); // factory function for creating instances + ReaderBackend* (*factory)(ReaderFrontend* frontend); // factory function for creating instances }; -InputReaderDefinition input_readers[] = { - { BifEnum::Input::READER_ASCII, "Ascii", 0, InputReaderAscii::Instantiate }, +ReaderDefinition input_readers[] = { + { BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate }, // End marker - { BifEnum::Input::READER_DEFAULT, "None", 0, (InputReader* (*)())0 } + { BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 } }; Manager::Manager() { } -// create a new input reader object to be used at whomevers leisure lateron. -InputReader* Manager::CreateStream(EnumVal* id, RecordVal* description) -{ - InputReaderDefinition* ir = input_readers; - - RecordType* rtype = description->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) - { - reporter->Error("Streamdescription argument not of right type"); - return 0; - } +ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) { + ReaderDefinition* ir = input_readers; - EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); - while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { @@ -171,7 +162,7 @@ InputReader* Manager::CreateStream(EnumVal* id, RecordVal* description) return 0; } - if ( ir->type != reader->AsEnum() ) { + if ( ir->type != type ) { // no, didn't find the right one... ++ir; continue; @@ -201,9 +192,30 @@ InputReader* Manager::CreateStream(EnumVal* id, RecordVal* description) // all done. break. break; } - assert(ir->factory); - InputReader* reader_obj = (*ir->factory)(); + + ReaderBackend* backend = (*ir->factory)(frontend); + assert(backend); + + frontend->ty_name = ir->name; + return backend; +} + +// create a new input reader object to be used at whomevers leisure lateron. +ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) +{ + ReaderDefinition* ir = input_readers; + + RecordType* rtype = description->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) + { + reporter->Error("Streamdescription argument not of right type"); + return 0; + } + + EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); + + ReaderFrontend* reader_obj = new ReaderFrontend(id->AsEnum()); assert(reader_obj); // get the source... @@ -217,16 +229,16 @@ InputReader* Manager::CreateStream(EnumVal* id, RecordVal* description) readers.push_back(info); - int success = reader_obj->Init(source); - if ( success == false ) { + reader_obj->Init(source); + /* if ( success == false ) { assert( RemoveStream(id) ); return 0; - } - success = reader_obj->Update(); - if ( success == false ) { + } */ + reader_obj->Update(); + /* if ( success == false ) { assert ( RemoveStream(id) ); return 0; - } + } */ return reader_obj; @@ -306,7 +318,7 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { } - vector fieldsV; // vector, because UnrollRecordType needs it + vector fieldsV; // vector, because UnrollRecordType needs it bool status = !UnrollRecordType(&fieldsV, fields, ""); @@ -316,7 +328,7 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { } - LogField** logf = new LogField*[fieldsV.size()]; + Field** logf = new Field*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { logf[i] = fieldsV[i]; } @@ -410,7 +422,7 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { } - vector fieldsV; // vector, because we don't know the length beforehands + vector fieldsV; // vector, because we don't know the length beforehands bool status = !UnrollRecordType(&fieldsV, idx, ""); @@ -430,7 +442,7 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { } - LogField** fields = new LogField*[fieldsV.size()]; + Field** fields = new Field*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { fields[i] = fieldsV[i]; } @@ -538,12 +550,12 @@ bool Manager::RemoveStream(const EnumVal* id) { return true; } -bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { +bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { if ( !IsCompatibleType(rec->FieldType(i)) ) { - reporter->Error("Incompatible type \"%s\" in table definition for InputReader", type_name(rec->FieldType(i)->Tag())); + reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); return false; } @@ -557,7 +569,7 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, } } else { - LogField* field = new LogField(); + Field* field = new Field(); field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); if ( field->type == TYPE_TABLE ) { @@ -591,7 +603,9 @@ bool Manager::ForceUpdate(const EnumVal* id) return false; } - return i->reader->Update(); + i->reader->Update(); + + return true; // update is async :( } bool Manager::RemoveTableFilter(EnumVal* id, const string &name) { @@ -638,21 +652,21 @@ bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { return true; } -Val* Manager::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { +Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) { Val* idxval; int position = 0; if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { - idxval = LogValToVal(vals[0], type->FieldType(0)); + idxval = ValueToVal(vals[0], type->FieldType(0)); position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); for ( int j = 0 ; j < type->NumFields(); j++ ) { if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { - l->Append(LogValToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); + l->Append(ValueToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); } else { - l->Append(LogValToVal(vals[position], type->FieldType(j))); + l->Append(ValueToVal(vals[position], type->FieldType(j))); position++; } } @@ -666,7 +680,7 @@ Val* Manager::LogValToIndexVal(int num_fields, const RecordType *type, const Log } -void Manager::SendEntry(const InputReader* reader, int id, const LogVal* const *vals) { +void Manager::SendEntry(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -689,7 +703,7 @@ void Manager::SendEntry(const InputReader* reader, int id, const LogVal* const * } -void Manager::SendEntryTable(const InputReader* reader, int id, const LogVal* const *vals) { +void Manager::SendEntryTable(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); bool updated = false; @@ -701,12 +715,12 @@ void Manager::SendEntryTable(const InputReader* reader, int id, const LogVal* co TableFilter* filter = (TableFilter*) i->filters[id]; //reporter->Error("Hashing %d index fields", i->num_idx_fields); - HashKey* idxhash = HashLogVals(filter->num_idx_fields, vals); + HashKey* idxhash = HashValues(filter->num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = 0; if ( filter->num_val_fields > 0 ) - HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); + HashValues(filter->num_val_fields, vals+filter->num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); @@ -731,16 +745,16 @@ void Manager::SendEntryTable(const InputReader* reader, int id, const LogVal* co } - Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); + Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; int position = filter->num_idx_fields; if ( filter->num_val_fields == 0 ) { valval = 0; } else if ( filter->num_val_fields == 1 && !filter->want_record ) { - valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); + valval = ValueToVal(vals[position], filter->rtype->FieldType(0)); } else { - valval = LogValToRecordVal(vals, filter->rtype, &position); + valval = ValueToRecordVal(vals, filter->rtype, &position); } @@ -757,7 +771,7 @@ void Manager::SendEntryTable(const InputReader* reader, int id, const LogVal* co EnumVal* ev; //Ref(idxval); int startpos = 0; - Val* predidx = LogValToRecordVal(vals, filter->itype, &startpos); + Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); Ref(valval); if ( updated ) { @@ -831,7 +845,7 @@ void Manager::SendEntryTable(const InputReader* reader, int id, const LogVal* co } -void Manager::EndCurrentSend(const InputReader* reader, int id) { +void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -914,7 +928,7 @@ void Manager::EndCurrentSend(const InputReader* reader, int id) { filter->currDict = new PDict(InputHash); } -void Manager::Put(const InputReader* reader, int id, const LogVal* const *vals) { +void Manager::Put(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -937,7 +951,7 @@ void Manager::Put(const InputReader* reader, int id, const LogVal* const *vals) } -void Manager::SendEventFilterEvent(const InputReader* reader, EnumVal* type, int id, const LogVal* const *vals) { +void Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); bool updated = false; @@ -956,15 +970,15 @@ void Manager::SendEventFilterEvent(const InputReader* reader, EnumVal* type, int int position = 0; if ( filter->want_record ) { - RecordVal * r = LogValToRecordVal(vals, filter->fields, &position); + RecordVal * r = ValueToRecordVal(vals, filter->fields, &position); out_vals.push_back(r); } else { for ( int j = 0; j < filter->fields->NumFields(); j++) { Val* val = 0; if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, filter->fields->FieldType(j)->AsRecordType(), &position); + val = ValueToRecordVal(vals, filter->fields->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], filter->fields->FieldType(j)); + val = ValueToVal(vals[position], filter->fields->FieldType(j)); position++; } out_vals.push_back(val); @@ -975,7 +989,7 @@ void Manager::SendEventFilterEvent(const InputReader* reader, EnumVal* type, int } -void Manager::PutTable(const InputReader* reader, int id, const LogVal* const *vals) { +void Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); assert(i); @@ -984,22 +998,22 @@ void Manager::PutTable(const InputReader* reader, int id, const LogVal* const *v assert(i->filters[id]->filter_type == TABLE_FILTER); TableFilter* filter = (TableFilter*) i->filters[id]; - Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); + Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; int position = filter->num_idx_fields; if ( filter->num_val_fields == 0 ) { valval = 0; } else if ( filter->num_val_fields == 1 && !filter->want_record ) { - valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); + valval = ValueToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { - valval = LogValToRecordVal(vals, filter->rtype, &position); + valval = ValueToRecordVal(vals, filter->rtype, &position); } filter->tab->Assign(idxval, valval); } -void Manager::Clear(const InputReader* reader, int id) { +void Manager::Clear(const ReaderFrontend* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -1014,7 +1028,7 @@ void Manager::Clear(const InputReader* reader, int id) { filter->tab->RemoveAll(); } -bool Manager::Delete(const InputReader* reader, int id, const LogVal* const *vals) { +bool Manager::Delete(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -1025,7 +1039,7 @@ bool Manager::Delete(const InputReader* reader, int id, const LogVal* const *val if ( i->filters[id]->filter_type == TABLE_FILTER ) { TableFilter* filter = (TableFilter*) i->filters[id]; - Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); + Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); return( filter->tab->Delete(idxval) != 0 ); } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); @@ -1037,12 +1051,12 @@ bool Manager::Delete(const InputReader* reader, int id, const LogVal* const *val } } -void Manager::Error(InputReader* reader, const char* msg) +void Manager::Error(ReaderFrontend* reader, const char* msg) { reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -bool Manager::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +bool Manager::SendEvent(const string& name, const int num_vals, const Value* const *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); if ( handler == 0 ) { @@ -1059,7 +1073,7 @@ bool Manager::SendEvent(const string& name, const int num_vals, const LogVal* co val_list* vl = new val_list; for ( int i = 0; i < num_vals; i++) { - vl->append(LogValToVal(vals[i], type->FieldType(i))); + vl->append(ValueToVal(vals[i], type->FieldType(i))); } mgr.Dispatch(new Event(handler, vl)); @@ -1118,7 +1132,7 @@ RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, -RecordVal* Manager::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { +RecordVal* Manager::ValueToRecordVal(const Value* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { reporter->InternalError("Need position"); return 0; @@ -1136,9 +1150,9 @@ RecordVal* Manager::LogValToRecordVal(const LogVal* const *vals, RecordType *req Val* fieldVal = 0; if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { - fieldVal = LogValToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); + fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); } else { - fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)); + fieldVal = ValueToVal(vals[*position], request_type->FieldType(i)); (*position)++; } @@ -1150,7 +1164,7 @@ RecordVal* Manager::LogValToRecordVal(const LogVal* const *vals, RecordType *req } -int Manager::GetLogValLength(const LogVal* val) { +int Manager::GetValueLength(const Value* val) { int length = 0; switch (val->type) { @@ -1193,7 +1207,7 @@ int Manager::GetLogValLength(const LogVal* val) { case TYPE_TABLE: { for ( int i = 0; i < val->val.set_val.size; i++ ) { - length += GetLogValLength(val->val.set_val.vals[i]); + length += GetValueLength(val->val.set_val.vals[i]); } break; } @@ -1201,20 +1215,20 @@ int Manager::GetLogValLength(const LogVal* val) { case TYPE_VECTOR: { int j = val->val.vector_val.size; for ( int i = 0; i < j; i++ ) { - length += GetLogValLength(val->val.vector_val.vals[i]); + length += GetValueLength(val->val.vector_val.vals[i]); } break; } default: - reporter->InternalError("unsupported type %d for GetLogValLength", val->type); + reporter->InternalError("unsupported type %d for GetValueLength", val->type); } return length; } -int Manager::CopyLogVal(char *data, const int startpos, const LogVal* val) { +int Manager::CopyValue(char *data, const int startpos, const Value* val) { switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: @@ -1276,7 +1290,7 @@ int Manager::CopyLogVal(char *data, const int startpos, const LogVal* val) { case TYPE_TABLE: { int length = 0; for ( int i = 0; i < val->val.set_val.size; i++ ) { - length += CopyLogVal(data, startpos+length, val->val.set_val.vals[i]); + length += CopyValue(data, startpos+length, val->val.set_val.vals[i]); } return length; break; @@ -1286,14 +1300,14 @@ int Manager::CopyLogVal(char *data, const int startpos, const LogVal* val) { int length = 0; int j = val->val.vector_val.size; for ( int i = 0; i < j; i++ ) { - length += CopyLogVal(data, startpos+length, val->val.vector_val.vals[i]); + length += CopyValue(data, startpos+length, val->val.vector_val.vals[i]); } return length; break; } default: - reporter->InternalError("unsupported type %d for CopyLogVal", val->type); + reporter->InternalError("unsupported type %d for CopyValue", val->type); return 0; } @@ -1302,12 +1316,12 @@ int Manager::CopyLogVal(char *data, const int startpos, const LogVal* val) { } -HashKey* Manager::HashLogVals(const int num_elements, const LogVal* const *vals) { +HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { int length = 0; for ( int i = 0; i < num_elements; i++ ) { - const LogVal* val = vals[i]; - length += GetLogValLength(val); + const Value* val = vals[i]; + length += GetValueLength(val); } //reporter->Error("Length: %d", length); @@ -1318,8 +1332,8 @@ HashKey* Manager::HashLogVals(const int num_elements, const LogVal* const *vals) reporter->InternalError("Could not malloc?"); } for ( int i = 0; i < num_elements; i++ ) { - const LogVal* val = vals[i]; - position += CopyLogVal(data, position, val); + const Value* val = vals[i]; + position += CopyValue(data, position, val); } assert(position == length); @@ -1328,7 +1342,7 @@ HashKey* Manager::HashLogVals(const int num_elements, const LogVal* const *vals) } -Val* Manager::LogValToVal(const LogVal* val, BroType* request_type) { +Val* Manager::ValueToVal(const Value* val, BroType* request_type) { if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); @@ -1384,7 +1398,7 @@ Val* Manager::LogValToVal(const LogVal* val, BroType* request_type) { SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); for ( int i = 0; i < val->val.set_val.size; i++ ) { - t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + t->Assign(ValueToVal( val->val.set_val.vals[i], type ), 0); } return t; break; @@ -1396,7 +1410,7 @@ Val* Manager::LogValToVal(const LogVal* val, BroType* request_type) { VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); for ( int i = 0; i < val->val.vector_val.size; i++ ) { - v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); + v->Assign(i, ValueToVal( val->val.set_val.vals[i], type ), 0); } return v; @@ -1425,7 +1439,7 @@ Val* Manager::LogValToVal(const LogVal* val, BroType* request_type) { return NULL; } -Manager::ReaderInfo* Manager::FindReader(const InputReader* reader) +Manager::ReaderInfo* Manager::FindReader(const ReaderFrontend* reader) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -1460,4 +1474,3 @@ string Manager::Hash(const string &input) { return out; } - diff --git a/src/input/Manager.h b/src/input/Manager.h index fe37efa08b..507af6468f 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -14,6 +14,7 @@ namespace input { class ReaderFrontend; +class ReaderBackend; class Manager { public: @@ -30,7 +31,8 @@ public: bool RemoveEventFilter(EnumVal* id, const string &name); protected: - + friend class ReaderFrontend; + // Reports an error for the given reader. void Error(ReaderFrontend* reader, const char* msg); @@ -42,6 +44,8 @@ protected: // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) void SendEntry(const ReaderFrontend* reader, int id, const threading::Value* const *vals); void EndCurrentSend(const ReaderFrontend* reader, int id); + + ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); private: struct ReaderInfo; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index b2bcedb2ad..8c996db4a1 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -1,13 +1,44 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "InputReader.h" +#include "ReaderBackend.h" +#include "ReaderFrontend.h" using threading::Value; using threading::Field; -namespace logging { +namespace input { -InputReader::InputReader(ReaderFrontend *arg_frontend) :MsgThread() +class ErrorMessage : public threading::OutputMessage { +public: + ErrorMessage(ReaderFrontend* reader, string message) + : threading::OutputMessage("Error", reader), + message(message) {} + + virtual bool Process() { + input_mgr->Error(object, message.c_str()); + return true; + } + +private: + string message; +} + +class PutMessage : public threading::OutputMessage { +public: + PutMessage(ReaderFrontend* reader, int id, const Value* const *val) + : threading::OutputMessage("Error", reader), + id(id), val(val) {} + + virtual bool Process() { + return input_mgr->Put(object, id, val); + } + +private: + int id; + Value* val; +} + +ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() { buf = 0; buf_len = 1024; @@ -18,38 +49,47 @@ InputReader::InputReader(ReaderFrontend *arg_frontend) :MsgThread() SetName(frontend->Name()); } -InputReader::~InputReader() +ReaderBackend::~ReaderBackend() { } -void InputReader::Error(const char *msg) +void ReaderBackend::Error(const string &msg) { - input_mgr->Error(this, msg); + SendOut(new ErrorMessage(frontend, msg); } -void InputReader::Error(const string &msg) +void ReaderBackend::Put(int id, const Value* const *val) { - input_mgr->Error(this, msg.c_str()); + SendOut(new PutMessage(frontend, id, val); } -void InputReader::Put(int id, const LogVal* const *val) +void ReaderBackend::Delete(int id, const Value* const *val) { - input_mgr->Put(this, id, val); + SendOut(new DeleteMessage(frontend, id, val); } -void InputReader::Clear(int id) +void ReaderBackend::Clear(int id) { - input_mgr->Clear(this, id); + SendOut(new ClearMessage(frontend, id); } -void InputReader::Delete(int id, const LogVal* const *val) +bool ReaderBackend::SendEvent(const string& name, const int num_vals, const Value* const *vals) { - input_mgr->Delete(this, id, val); + SendOut(new SendEventMessage(frontend, name, num_vals, vals); +} + +void ReaderBackend::EndCurrentSend(int id) +{ + SendOut(new EndCurrentSendMessage(frontent, id); } +void ReaderBackend::SendEntry(int id, const Value* const *vals) +{ + SendOut(new SendEntryMessage(frontend, id, vals); +} -bool InputReader::Init(string arg_source) +bool ReaderBackend::Init(string arg_source) { source = arg_source; @@ -58,35 +98,31 @@ bool InputReader::Init(string arg_source) return !disabled; } -bool InputReader::AddFilter(int id, int arg_num_fields, - const LogField* const * arg_fields) +bool ReaderBackend::AddFilter(int id, int arg_num_fields, + const Field* const * arg_fields) { return DoAddFilter(id, arg_num_fields, arg_fields); } -bool InputReader::RemoveFilter(int id) +bool ReaderBackend::RemoveFilter(int id) { return DoRemoveFilter(id); } -void InputReader::Finish() +void ReaderBackend::Finish() { DoFinish(); disabled = true; } -bool InputReader::Update() +bool ReaderBackend::Update() { return DoUpdate(); } -bool InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) -{ - return input_mgr->SendEvent(name, num_vals, vals); -} // stolen from logwriter -const char* InputReader::Fmt(const char* format, ...) +const char* ReaderBackend::Fmt(const char* format, ...) { if ( ! buf ) buf = (char*) malloc(buf_len); @@ -111,14 +147,5 @@ const char* InputReader::Fmt(const char* format, ...) } -void InputReader::SendEntry(int id, const LogVal* const *vals) -{ - input_mgr->SendEntry(this, id, vals); -} - -void InputReader::EndCurrentSend(int id) -{ - input_mgr->EndCurrentSend(this, id); -} } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 7d2640b4fd..1fe44a09b2 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -1,26 +1,25 @@ // See the file "COPYING" in the main distribution directory for copyright. -// -// Same notes about thread safety as in LogWriter.h apply. - #ifndef INPUT_READERBACKEND_H #define INPUT_READERBACKEND_H -#include "InputMgr.h" #include "BroString.h" -#include "LogMgr.h" +#include "../threading/SerializationTypes.h" +#include "threading/MsgThread.h" namespace input { +class ReaderFrontend; + class ReaderBackend : public threading::MsgThread { public: - ReaderBackend(ReaderFrontend *frontend); + ReaderBackend(ReaderFrontend* frontend); virtual ~ReaderBackend(); bool Init(string arg_source); - bool AddFilter( int id, int arg_num_fields, const LogField* const* fields ); + bool AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); bool RemoveFilter ( int id ); @@ -32,7 +31,7 @@ protected: // Methods that have to be overwritten by the individual readers virtual bool DoInit(string arg_sources) = 0; - virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) = 0; + virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ) = 0; virtual bool DoRemoveFilter( int id ) = 0; @@ -51,15 +50,15 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + bool SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table - void Put(int id, const LogVal* const *val); - void Delete(int id, const LogVal* const *val); + void Put(int id, const threading::Value* const *val); + void Delete(int id, const threading::Value* const *val); void Clear(int id); // Table-functions (tracking mode): Only changed lines are propagated. - void SendEntry(int id, const LogVal* const *vals); + void SendEntry(int id, const threading::Value* const *vals); void EndCurrentSend(int id); diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index e69de29bb2..984ba30794 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -0,0 +1,54 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERFRONTEND_H +#define INPUT_READERFRONTEND_H + +#include "Manager.h" + +#include "threading/MsgThread.h" + +namespace input { + +class ReaderBackend; + +class ReaderFrontend { +public: + ReaderFrontend(bro_int_t type); + + virtual ~ReaderFrontend(); + + void Init(string arg_source); + + void Update(); + + void AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + + void Finish(); + + /** + * Returns a descriptive name for the reader, including the type of + * the backend and the source used. + * + * This method is safe to call from any thread. + */ + string Name() const; + + +protected: + friend class Manager; + + const string Source() const { return source; } + + string ty_name; // Name of the backend type. Set by the manager. + +private: + string source; + +}; + +} + + +#endif /* INPUT_READERFRONTEND_H */ + + diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 1747f983e4..a3bf5c21a6 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -1,13 +1,15 @@ // See the file "COPYING" in the main distribution directory for copyright. -#ifndef INPUTREADERASCII_H -#define INPUTREADERASCII_H +#ifndef INPUT_READERS_ASCII_H +#define INPUT_READERS_ASCII_H -#include "InputReader.h" -#include #include #include +#include "../ReaderBackend.h" + +namespace input { namespace reader { + // Description for input field mapping struct FieldMapping { string name; @@ -28,18 +30,18 @@ struct FieldMapping { }; -class InputReaderAscii : public InputReader { +class Ascii : public ReaderBackend { public: - InputReaderAscii(); - ~InputReaderAscii(); + Ascii(ReaderFrontend* frontend); + ~Ascii(); - static InputReader* Instantiate() { return new InputReaderAscii; } + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Ascii(frontend); } protected: virtual bool DoInit(string path); - virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ); + virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); virtual bool DoRemoveFilter ( int id ); @@ -52,7 +54,7 @@ private: struct Filter { unsigned int num_fields; - const LogField* const * fields; // raw mapping + const threading::Field* const * fields; // raw mapping // map columns in the file to columns to send back to the manager vector columnMap; @@ -64,7 +66,7 @@ private: TransportProto StringToProto(const string &proto); bool ReadHeader(); - LogVal* EntryToVal(string s, FieldMapping type); + threading::Value* EntryToVal(string s, FieldMapping type); bool GetLine(string& str); @@ -85,4 +87,7 @@ private: }; -#endif /* INPUTREADERASCII_H */ +} +} + +#endif /* INPUT_READERS_ASCII_H */ diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index add10b3f10..65a55dee02 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -14,7 +14,7 @@ #include "writers/Ascii.h" #include "writers/None.h" -#include "threading/SerializationTypes.h" +#include "../threading/SerializationTypes.h" using namespace logging; using threading::Value; diff --git a/src/threading/SerializationTypes.cc b/src/threading/SerializationTypes.cc index f74de6ce57..dc5a1a14f9 100644 --- a/src/threading/SerializationTypes.cc +++ b/src/threading/SerializationTypes.cc @@ -12,7 +12,8 @@ bool Field::Read(SerializationFormat* fmt) int t; int st; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); + bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && + fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); type = (TypeTag) t; subtype = (TypeTag) st; @@ -21,7 +22,8 @@ bool Field::Read(SerializationFormat* fmt) bool Field::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); + return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && + fmt->Write((int)subtype, "subtype")); } Value::~Value() diff --git a/src/threading/SerializationTypes.h b/src/threading/SerializationTypes.h index 11ceda929c..ffcf774842 100644 --- a/src/threading/SerializationTypes.h +++ b/src/threading/SerializationTypes.h @@ -13,6 +13,8 @@ namespace threading { */ struct Field { string name; //! Name of the field. + // needed by input framework. port fields have two names (one for the port, one for the type) - this specifies the secondary name. + string secondary_name; TypeTag type; //! Type of the field. TypeTag subtype; //! Inner type for sets. From 8385d5bb2dedab25da82b67213a6051bd52e1746 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 6 Feb 2012 17:37:02 -0800 Subject: [PATCH 057/149] it compiles :) But that's all, not tested, don't expect it to do anything but crash. --- src/input/Manager.cc | 4 +- src/input/Manager.h | 17 +++-- src/input/ReaderBackend.cc | 114 ++++++++++++++++++++++++++++----- src/input/ReaderBackend.h | 4 +- src/input/ReaderFrontend.cc | 121 +++++++++++++++++++++++++++++++----- src/input/ReaderFrontend.h | 17 ++--- src/input/readers/Ascii.cc | 48 +++++++------- 7 files changed, 258 insertions(+), 67 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index a97286162d..189a034b0f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -680,7 +680,7 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu } -void Manager::SendEntry(const ReaderFrontend* reader, int id, const Value* const *vals) { +void Manager::SendEntry(const ReaderFrontend* reader, const int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -703,7 +703,7 @@ void Manager::SendEntry(const ReaderFrontend* reader, int id, const Value* const } -void Manager::SendEntryTable(const ReaderFrontend* reader, int id, const Value* const *vals) { +void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); bool updated = false; diff --git a/src/input/Manager.h b/src/input/Manager.h index 507af6468f..a0b98294ca 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -14,7 +14,7 @@ namespace input { class ReaderFrontend; -class ReaderBackend; +class ReaderBackend; class Manager { public: @@ -32,6 +32,13 @@ public: protected: friend class ReaderFrontend; + friend class ErrorMessage; + friend class PutMessage; + friend class DeleteMessage; + friend class ClearMessage; + friend class SendEventMessage; + friend class SendEntryMessage; + friend class EndCurrentSendMessage; // Reports an error for the given reader. void Error(ReaderFrontend* reader, const char* msg); @@ -42,11 +49,14 @@ protected: bool Delete(const ReaderFrontend* reader, int id, const threading::Value* const *vals); // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) - void SendEntry(const ReaderFrontend* reader, int id, const threading::Value* const *vals); - void EndCurrentSend(const ReaderFrontend* reader, int id); + void SendEntry(const ReaderFrontend* reader, const int id, const threading::Value* const *vals); + void EndCurrentSend(const ReaderFrontend* reader, const int id); + + bool SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); + private: struct ReaderInfo; @@ -60,7 +70,6 @@ private: void SendEvent(EventHandlerPtr ev, const int numvals, ...); void SendEvent(EventHandlerPtr ev, list events); - bool SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); HashKey* HashValues(const int num_elements, const threading::Value* const *vals); int GetValueLength(const threading::Value* val); diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 8c996db4a1..72c8f95d8e 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -2,6 +2,7 @@ #include "ReaderBackend.h" #include "ReaderFrontend.h" +#include "Manager.h" using threading::Value; using threading::Field; @@ -15,28 +16,106 @@ public: message(message) {} virtual bool Process() { - input_mgr->Error(object, message.c_str()); + input_mgr->Error(Object(), message.c_str()); return true; } private: string message; -} +}; class PutMessage : public threading::OutputMessage { public: PutMessage(ReaderFrontend* reader, int id, const Value* const *val) - : threading::OutputMessage("Error", reader), + : threading::OutputMessage("Put", reader), id(id), val(val) {} virtual bool Process() { - return input_mgr->Put(object, id, val); + input_mgr->Put(Object(), id, val); + return true; } private: int id; - Value* val; -} + const Value* const *val; +}; + +class DeleteMessage : public threading::OutputMessage { +public: + DeleteMessage(ReaderFrontend* reader, int id, const Value* const *val) + : threading::OutputMessage("Delete", reader), + id(id), val(val) {} + + virtual bool Process() { + return input_mgr->Delete(Object(), id, val); + } + +private: + int id; + const Value* const *val; +}; + +class ClearMessage : public threading::OutputMessage { +public: + ClearMessage(ReaderFrontend* reader, int id) + : threading::OutputMessage("Clear", reader), + id(id) {} + + virtual bool Process() { + input_mgr->Clear(Object(), id); + return true; + } + +private: + int id; +}; + +class SendEventMessage : public threading::OutputMessage { +public: + SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, const Value* const *val) + : threading::OutputMessage("SendEvent", reader), + name(name), num_vals(num_vals), val(val) {} + + virtual bool Process() { + return input_mgr->SendEvent(name, num_vals, val); + } + +private: + const string name; + const int num_vals; + const Value* const *val; +}; + +class SendEntryMessage : public threading::OutputMessage { +public: + SendEntryMessage(ReaderFrontend* reader, const int id, const Value* const *val) + : threading::OutputMessage("SendEntry", reader), + id(id), val(val) {} + + virtual bool Process() { + input_mgr->SendEntry(Object(), id, val); + return true; + } + +private: + const int id; + const Value* const *val; +}; + +class EndCurrentSendMessage : public threading::OutputMessage { +public: + EndCurrentSendMessage(ReaderFrontend* reader, int id) + : threading::OutputMessage("SendEntry", reader), + id(id) {} + + virtual bool Process() { + input_mgr->EndCurrentSend(Object(), id); + return true; + } + +private: + int id; +}; ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() { @@ -56,37 +135,44 @@ ReaderBackend::~ReaderBackend() void ReaderBackend::Error(const string &msg) { - SendOut(new ErrorMessage(frontend, msg); + SendOut(new ErrorMessage(frontend, msg)); } +/* +void ReaderBackend::Error(const char *msg) +{ + SendOut(new ErrorMessage(frontend, string(msg))); +} */ + + void ReaderBackend::Put(int id, const Value* const *val) { - SendOut(new PutMessage(frontend, id, val); + SendOut(new PutMessage(frontend, id, val)); } void ReaderBackend::Delete(int id, const Value* const *val) { - SendOut(new DeleteMessage(frontend, id, val); + SendOut(new DeleteMessage(frontend, id, val)); } void ReaderBackend::Clear(int id) { - SendOut(new ClearMessage(frontend, id); + SendOut(new ClearMessage(frontend, id)); } -bool ReaderBackend::SendEvent(const string& name, const int num_vals, const Value* const *vals) +void ReaderBackend::SendEvent(const string& name, const int num_vals, const Value* const *vals) { - SendOut(new SendEventMessage(frontend, name, num_vals, vals); + SendOut(new SendEventMessage(frontend, name, num_vals, vals)); } void ReaderBackend::EndCurrentSend(int id) { - SendOut(new EndCurrentSendMessage(frontent, id); + SendOut(new EndCurrentSendMessage(frontend, id)); } void ReaderBackend::SendEntry(int id, const Value* const *vals) { - SendOut(new SendEntryMessage(frontend, id, vals); + SendOut(new SendEntryMessage(frontend, id, vals)); } bool ReaderBackend::Init(string arg_source) diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 1fe44a09b2..a37daaf4b6 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -42,7 +42,7 @@ protected: // Reports an error to the user. void Error(const string &msg); - void Error(const char *msg); + //void Error(const char *msg); // The following methods return the information as passed to Init(). const string Source() const { return source; } @@ -50,7 +50,7 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - bool SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); + void SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table void Put(int id, const threading::Value* const *val); diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index 44638d90b3..a7f9a4d2f6 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -1,28 +1,117 @@ // See the file "COPYING" in the main distribution directory for copyright. -#ifndef INPUT_READERFRONTEND_H -#define INPUT_READERFRONTEND_H - #include "Manager.h" - +#include "ReaderFrontend.h" +#include "ReaderBackend.h" #include "threading/MsgThread.h" -namespace logging { - -class ReaderBackend; - -class ReaderFrontend { - - ReaderFrontend(bro_int_t type); - - virtual ~ReaderFrontend(); +namespace input { -protected: - friend class Manager; +class InitMessage : public threading::InputMessage +{ +public: + InitMessage(ReaderBackend* backend, const string source) + : threading::InputMessage("Init", backend), + source(source) { } + + virtual bool Process() { return Object()->Init(source); } + +private: + const string source; }; +class UpdateMessage : public threading::InputMessage +{ +public: + UpdateMessage(ReaderBackend* backend) + : threading::InputMessage("Update", backend) + { } + + virtual bool Process() { return Object()->Update(); } +}; + +class FinishMessage : public threading::InputMessage +{ +public: + FinishMessage(ReaderBackend* backend) + : threading::InputMessage("Finish", backend) + { } + + virtual bool Process() { Object()->Finish(); return true; } +}; + +class AddFilterMessage : public threading::InputMessage +{ +public: + AddFilterMessage(ReaderBackend* backend, const int id, const int num_fields, const threading::Field* const* fields) + : threading::InputMessage("AddFilter", backend), + id(id), num_fields(num_fields), fields(fields) { } + + virtual bool Process() { return Object()->AddFilter(id, num_fields, fields); } + +private: + const int id; + const int num_fields; + const threading::Field* const* fields; +}; + +ReaderFrontend::ReaderFrontend(bro_int_t type) { + disabled = initialized = false; + ty_name = ""; + backend = input_mgr->CreateBackend(this, type); + + assert(backend); + backend->Start(); +} + +ReaderFrontend::~ReaderFrontend() { +} + +void ReaderFrontend::Init(string arg_source) { + if ( disabled ) + return; + + if ( initialized ) + reporter->InternalError("writer initialize twice"); + + source = arg_source; + initialized = true; + + backend->SendIn(new InitMessage(backend, arg_source)); +} + +void ReaderFrontend::Update() { + if ( disabled ) + return; + + backend->SendIn(new UpdateMessage(backend)); +} + +void ReaderFrontend::Finish() { + if ( disabled ) + return; + + backend->SendIn(new FinishMessage(backend)); +} + +void ReaderFrontend::AddFilter(const int id, const int arg_num_fields, const threading::Field* const* fields) { + if ( disabled ) + return; + + backend->SendIn(new AddFilterMessage(backend, id, arg_num_fields, fields)); +} + +string ReaderFrontend::Name() const + { + if ( source.size() ) + return ty_name; + + return ty_name + "/" + source; + } + + + } -#endif /* INPUT_READERFRONTEND_H */ diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index 984ba30794..876082d9a6 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -3,13 +3,12 @@ #ifndef INPUT_READERFRONTEND_H #define INPUT_READERFRONTEND_H -#include "Manager.h" +#include "../threading/MsgThread.h" +#include "../threading/SerializationTypes.h" -#include "threading/MsgThread.h" +namespace input { -namespace input { - -class ReaderBackend; +class Manager; class ReaderFrontend { public: @@ -21,7 +20,7 @@ public: void Update(); - void AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + void AddFilter( const int id, const int arg_num_fields, const threading::Field* const* fields ); void Finish(); @@ -32,17 +31,19 @@ public: * This method is safe to call from any thread. */ string Name() const; - protected: friend class Manager; - const string Source() const { return source; } + const string Source() const { return source; }; string ty_name; // Name of the backend type. Set by the manager. private: + ReaderBackend* backend; // The backend we have instanatiated. string source; + bool disabled; // True if disabled. + bool initialized; // True if initialized. }; diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 257cb4cf71..e798f69a36 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -1,11 +1,17 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "InputReaderAscii.h" -#include "DebugLogger.h" +#include "Ascii.h" #include "NetVar.h" +#include #include +#include "../../threading/SerializationTypes.h" + +using namespace input::reader; +using threading::Value; +using threading::Field; + FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) : name(arg_name), type(arg_type) { @@ -31,7 +37,7 @@ FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } -InputReaderAscii::InputReaderAscii() +Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) { file = 0; @@ -53,13 +59,13 @@ InputReaderAscii::InputReaderAscii() } -InputReaderAscii::~InputReaderAscii() +Ascii::~Ascii() { DoFinish(); } -void InputReaderAscii::DoFinish() +void Ascii::DoFinish() { filters.empty(); if ( file != 0 ) { @@ -69,7 +75,7 @@ void InputReaderAscii::DoFinish() } } -bool InputReaderAscii::DoInit(string path) +bool Ascii::DoInit(string path) { fname = path; @@ -82,7 +88,7 @@ bool InputReaderAscii::DoInit(string path) return true; } -bool InputReaderAscii::DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) { +bool Ascii::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { if ( HasFilter(id) ) { return false; // no, we don't want to add this a second time } @@ -96,7 +102,7 @@ bool InputReaderAscii::DoAddFilter( int id, int arg_num_fields, const LogField* return true; } -bool InputReaderAscii::DoRemoveFilter ( int id ) { +bool Ascii::DoRemoveFilter ( int id ) { if (!HasFilter(id) ) { return false; } @@ -107,7 +113,7 @@ bool InputReaderAscii::DoRemoveFilter ( int id ) { } -bool InputReaderAscii::HasFilter(int id) { +bool Ascii::HasFilter(int id) { map::iterator it = filters.find(id); if ( it == filters.end() ) { return false; @@ -116,7 +122,7 @@ bool InputReaderAscii::HasFilter(int id) { } -bool InputReaderAscii::ReadHeader() { +bool Ascii::ReadHeader() { // try to read the header line... string line; if ( !GetLine(line) ) { @@ -142,7 +148,7 @@ bool InputReaderAscii::ReadHeader() { for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - const LogField* field = (*it).second.fields[i]; + const Field* field = (*it).second.fields[i]; map::iterator fit = fields.find(field->name); if ( fit == fields.end() ) { @@ -169,7 +175,7 @@ bool InputReaderAscii::ReadHeader() { return true; } -bool InputReaderAscii::GetLine(string& str) { +bool Ascii::GetLine(string& str) { while ( getline(*file, str) ) { if ( str[0] != '#' ) { return true; @@ -184,7 +190,7 @@ bool InputReaderAscii::GetLine(string& str) { return false; } -TransportProto InputReaderAscii::StringToProto(const string &proto) { +TransportProto Ascii::StringToProto(const string &proto) { if ( proto == "unknown" ) { return TRANSPORT_UNKNOWN; } else if ( proto == "tcp" ) { @@ -202,12 +208,12 @@ TransportProto InputReaderAscii::StringToProto(const string &proto) { return TRANSPORT_UNKNOWN; } -LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { +Value* Ascii::EntryToVal(string s, FieldMapping field) { - LogVal* val = new LogVal(field.type, true); + Value* val = new Value(field.type, true); if ( s.compare(unset_field) == 0 ) { // field is not set... - return new LogVal(field.type, false); + return new Value(field.type, false); } switch ( field.type ) { @@ -306,7 +312,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { if ( s.compare(empty_field) == 0 ) length = 0; - LogVal** lvals = new LogVal* [length]; + Value** lvals = new Value* [length]; if ( field.type == TYPE_TABLE ) { val->val.set_val.vals = lvals; @@ -333,7 +339,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; } - LogVal* newval = EntryToVal(element, field.subType()); + Value* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { Error("Error while reading set"); return 0; @@ -365,7 +371,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { } // read the entire file and send appropriate thingies back to InputMgr -bool InputReaderAscii::DoUpdate() { +bool Ascii::DoUpdate() { // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) if ( file && file->is_open() ) { @@ -405,7 +411,7 @@ bool InputReaderAscii::DoUpdate() { for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - LogVal** fields = new LogVal*[(*it).second.num_fields]; + Value** fields = new Value*[(*it).second.num_fields]; int fpos = 0; for ( vector::iterator fit = (*it).second.columnMap.begin(); @@ -417,7 +423,7 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal* val = EntryToVal(stringfields[(*fit).position], *fit); + Value* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { return false; } From 88233efb2c8b6700dd2ec0a1f6addc65a282b622 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 13 Feb 2012 22:29:55 -0800 Subject: [PATCH 058/149] It works. Even including all unit tests. But: there are still a few places where I am sure that there are race conditions & memory leaks & I do not really like the current interface & I have to add a few more messages between the front and backend. But - it works :) --- scripts/base/frameworks/input/main.bro | 2 + src/input/Manager.cc | 78 ++++++++++++++----- src/input/Manager.h | 14 ++-- src/input/ReaderBackend.cc | 28 +++---- src/input/ReaderBackend.h | 8 +- src/input/ReaderFrontend.cc | 1 + src/input/readers/Ascii.cc | 4 +- src/main.cc | 1 + src/threading/Manager.cc | 2 +- src/threading/MsgThread.cc | 2 +- .../scripts/base/frameworks/input/basic.bro | 5 +- .../frameworks/input/onecolumn-norecord.bro | 4 + .../frameworks/input/onecolumn-record.bro | 4 + .../scripts/base/frameworks/input/port.bro | 7 ++ .../base/frameworks/input/predicate.bro | 3 + .../base/frameworks/input/twofilters.bro | 13 +++- 16 files changed, 127 insertions(+), 49 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index c76eba80b9..cac1aca54a 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -114,6 +114,8 @@ export { ## filter: the `TableFilter` record describing the filter. global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; + global update_finished: event(id: Input::ID); + } @load base/input.bif diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 189a034b0f..79d42fe71f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -215,7 +215,7 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); - ReaderFrontend* reader_obj = new ReaderFrontend(id->AsEnum()); + ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); assert(reader_obj); // get the source... @@ -680,7 +680,7 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu } -void Manager::SendEntry(const ReaderFrontend* reader, const int id, const Value* const *vals) { +void Manager::SendEntry(const ReaderFrontend* reader, const int id, Value* *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -692,18 +692,25 @@ void Manager::SendEntry(const ReaderFrontend* reader, const int id, const Value* return; } + int readFields; if ( i->filters[id]->filter_type == TABLE_FILTER ) { - SendEntryTable(reader, id, vals); + readFields = SendEntryTable(reader, id, vals); } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - SendEventFilterEvent(reader, type, id, vals); + readFields = SendEventFilterEvent(reader, type, id, vals); } else { assert(false); } + for ( int i = 0; i < readFields; i++ ) { + delete vals[i]; + } + delete [] vals; + + } -void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { +int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); bool updated = false; @@ -733,7 +740,7 @@ void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const V // ok, exact duplicate filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); - return; + return filter->num_val_fields + filter->num_idx_fields; } else { assert( filter->num_val_fields > 0 ); // updated @@ -794,11 +801,11 @@ void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const V if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... delete(filter->currDict->RemoveEntry(idxhash)); - return; + return filter->num_val_fields + filter->num_idx_fields; } else { // keep old one filter->currDict->Insert(idxhash, h); - return; + return filter->num_val_fields + filter->num_idx_fields; } } @@ -809,7 +816,7 @@ void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const V HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); - return; + return filter->num_val_fields + filter->num_idx_fields; } filter->tab->Assign(idxval, k, valval); @@ -842,6 +849,9 @@ void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const V } } } + + + return filter->num_val_fields + filter->num_idx_fields; } @@ -926,9 +936,21 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { filter->lastDict = filter->currDict; filter->currDict = new PDict(InputHash); + + // Send event that the current update is indeed finished. + + + EventHandler* handler = event_registry->Lookup("Input::update_finished"); + if ( handler == 0 ) { + reporter->InternalError("Input::update_finished not found!"); + } + + + Ref(i->id); + SendEvent(handler, 1, i->id); } -void Manager::Put(const ReaderFrontend* reader, int id, const Value* const *vals) { +void Manager::Put(const ReaderFrontend* reader, int id, Value* *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -951,7 +973,7 @@ void Manager::Put(const ReaderFrontend* reader, int id, const Value* const *vals } -void Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { +int Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); bool updated = false; @@ -985,11 +1007,13 @@ void Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, } } - SendEvent(filter->event, out_vals); + SendEvent(filter->event, out_vals); + + return filter->fields->NumFields(); } -void Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const *vals) { +int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); assert(i); @@ -1011,6 +1035,8 @@ void Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const } filter->tab->Assign(idxval, valval); + + return filter->num_idx_fields + filter->num_val_fields; } void Manager::Clear(const ReaderFrontend* reader, int id) { @@ -1028,7 +1054,7 @@ void Manager::Clear(const ReaderFrontend* reader, int id) { filter->tab->RemoveAll(); } -bool Manager::Delete(const ReaderFrontend* reader, int id, const Value* const *vals) { +bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -1037,18 +1063,29 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, const Value* const *v assert(i->HasFilter(id)); + bool success = false; + int readVals = 0; + if ( i->filters[id]->filter_type == TABLE_FILTER ) { TableFilter* filter = (TableFilter*) i->filters[id]; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); - return( filter->tab->Delete(idxval) != 0 ); + readVals = filter->num_idx_fields + filter->num_val_fields; + success = ( filter->tab->Delete(idxval) != 0 ); } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEventFilterEvent(reader, type, id, vals); - return true; + readVals = SendEventFilterEvent(reader, type, id, vals); + success = true; } else { assert(false); return false; } + + for ( int i = 0; i < readVals; i++ ) { + delete vals[i]; + } + delete [] vals; + + return success; } void Manager::Error(ReaderFrontend* reader, const char* msg) @@ -1056,7 +1093,7 @@ void Manager::Error(ReaderFrontend* reader, const char* msg) reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -bool Manager::SendEvent(const string& name, const int num_vals, const Value* const *vals) +bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); if ( handler == 0 ) { @@ -1078,6 +1115,11 @@ bool Manager::SendEvent(const string& name, const int num_vals, const Value* con mgr.Dispatch(new Event(handler, vl)); + for ( int i = 0; i < num_vals; i++ ) { + delete vals[i]; + } + delete [] vals; + return true; } diff --git a/src/input/Manager.h b/src/input/Manager.h index a0b98294ca..45c07895f2 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -44,15 +44,15 @@ protected: void Error(ReaderFrontend* reader, const char* msg); // for readers to write to input stream in direct mode (reporting new/deleted values directly) - void Put(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + void Put(const ReaderFrontend* reader, int id, threading::Value* *vals); void Clear(const ReaderFrontend* reader, int id); - bool Delete(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + bool Delete(const ReaderFrontend* reader, int id, threading::Value* *vals); // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) - void SendEntry(const ReaderFrontend* reader, const int id, const threading::Value* const *vals); + void SendEntry(const ReaderFrontend* reader, const int id, threading::Value* *vals); void EndCurrentSend(const ReaderFrontend* reader, const int id); - bool SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); + bool SendEvent(const string& name, const int num_vals, threading::Value* *vals); ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); @@ -60,9 +60,9 @@ protected: private: struct ReaderInfo; - void SendEntryTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); - void PutTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); - void SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const threading::Value* const *vals); + int SendEntryTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + int PutTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + int SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const threading::Value* const *vals); bool IsCompatibleType(BroType* t, bool atomic_only=false); diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 72c8f95d8e..f9992f5f0e 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -26,7 +26,7 @@ private: class PutMessage : public threading::OutputMessage { public: - PutMessage(ReaderFrontend* reader, int id, const Value* const *val) + PutMessage(ReaderFrontend* reader, int id, Value* *val) : threading::OutputMessage("Put", reader), id(id), val(val) {} @@ -37,12 +37,12 @@ public: private: int id; - const Value* const *val; + Value* *val; }; class DeleteMessage : public threading::OutputMessage { public: - DeleteMessage(ReaderFrontend* reader, int id, const Value* const *val) + DeleteMessage(ReaderFrontend* reader, int id, Value* *val) : threading::OutputMessage("Delete", reader), id(id), val(val) {} @@ -52,7 +52,7 @@ public: private: int id; - const Value* const *val; + Value* *val; }; class ClearMessage : public threading::OutputMessage { @@ -72,7 +72,7 @@ private: class SendEventMessage : public threading::OutputMessage { public: - SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, const Value* const *val) + SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, Value* *val) : threading::OutputMessage("SendEvent", reader), name(name), num_vals(num_vals), val(val) {} @@ -83,14 +83,14 @@ public: private: const string name; const int num_vals; - const Value* const *val; + Value* *val; }; class SendEntryMessage : public threading::OutputMessage { public: - SendEntryMessage(ReaderFrontend* reader, const int id, const Value* const *val) + SendEntryMessage(ReaderFrontend* reader, const int id, Value* *val) : threading::OutputMessage("SendEntry", reader), - id(id), val(val) {} + id(id), val(val) { } virtual bool Process() { input_mgr->SendEntry(Object(), id, val); @@ -99,13 +99,13 @@ public: private: const int id; - const Value* const *val; + Value* *val; }; class EndCurrentSendMessage : public threading::OutputMessage { public: EndCurrentSendMessage(ReaderFrontend* reader, int id) - : threading::OutputMessage("SendEntry", reader), + : threading::OutputMessage("EndCurrentSend", reader), id(id) {} virtual bool Process() { @@ -145,12 +145,12 @@ void ReaderBackend::Error(const char *msg) } */ -void ReaderBackend::Put(int id, const Value* const *val) +void ReaderBackend::Put(int id, Value* *val) { SendOut(new PutMessage(frontend, id, val)); } -void ReaderBackend::Delete(int id, const Value* const *val) +void ReaderBackend::Delete(int id, Value* *val) { SendOut(new DeleteMessage(frontend, id, val)); } @@ -160,7 +160,7 @@ void ReaderBackend::Clear(int id) SendOut(new ClearMessage(frontend, id)); } -void ReaderBackend::SendEvent(const string& name, const int num_vals, const Value* const *vals) +void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) { SendOut(new SendEventMessage(frontend, name, num_vals, vals)); } @@ -170,7 +170,7 @@ void ReaderBackend::EndCurrentSend(int id) SendOut(new EndCurrentSendMessage(frontend, id)); } -void ReaderBackend::SendEntry(int id, const Value* const *vals) +void ReaderBackend::SendEntry(int id, Value* *vals) { SendOut(new SendEntryMessage(frontend, id, vals)); } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index a37daaf4b6..c12d187545 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -50,15 +50,15 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - void SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); + void SendEvent(const string& name, const int num_vals, threading::Value* *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table - void Put(int id, const threading::Value* const *val); - void Delete(int id, const threading::Value* const *val); + void Put(int id, threading::Value* *val); + void Delete(int id, threading::Value* *val); void Clear(int id); // Table-functions (tracking mode): Only changed lines are propagated. - void SendEntry(int id, const threading::Value* const *vals); + void SendEntry(int id, threading::Value* *vals); void EndCurrentSend(int id); diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index a7f9a4d2f6..0dac33d5e8 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -56,6 +56,7 @@ private: const threading::Field* const* fields; }; + ReaderFrontend::ReaderFrontend(bro_int_t type) { disabled = initialized = false; ty_name = ""; diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index e798f69a36..095d74bf11 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -445,10 +445,12 @@ bool Ascii::DoUpdate() { SendEntry((*it).first, fields); - for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + /* Do not do this, ownership changes to other thread + * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { delete fields[i]; } delete [] fields; + */ } } diff --git a/src/main.cc b/src/main.cc index 9df06aa0a0..8205b6de0b 100644 --- a/src/main.cc +++ b/src/main.cc @@ -295,6 +295,7 @@ void terminate_bro() log_mgr->Terminate(); thread_mgr->Terminate(); + mgr.Drain(); delete timer_mgr; delete dns_mgr; diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d008d2e5e8..7b571e753c 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -111,7 +111,7 @@ void Manager::Process() else { - string s = msg->Name() + " failed, terminating thread"; + string s = msg->Name() + " failed, terminating thread " + t->Name() + " (in ThreadManager)"; reporter->Error("%s", s.c_str()); t->Stop(); } diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index b7782b9a05..5f77a1c9f8 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -267,7 +267,7 @@ void MsgThread::Run() if ( ! result ) { - string s = msg->Name() + " failed, terminating thread"; + string s = msg->Name() + " failed, terminating thread (MsgThread)"; Error(s.c_str()); Stop(); break; diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index d1b6659eb6..3b75220625 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -48,7 +48,10 @@ event bro_init() Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); - print servers; Input::remove_tablefilter(A::INPUT, "ssh"); Input::remove_stream(A::INPUT); } + +event Input::update_finished(id: Input::ID) { + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 88838cc8d6..712a877960 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -34,5 +34,9 @@ event bro_init() Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); Input::force_update(A::INPUT); +} + +event Input::update_finished(id: Input::ID) { print servers; } + diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index fc4d862cd3..7b62ddcddd 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -34,5 +34,9 @@ event bro_init() Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); +} + +event Input::update_finished(id: Input::ID) { print servers; } + diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index c14892ae36..65d73c54f7 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -39,3 +39,10 @@ event bro_init() Input::remove_tablefilter(A::INPUT, "input"); Input::remove_stream(A::INPUT); } + +event Input::update_finished(id: Input::ID) { + print servers[1.2.3.4]; + print servers[1.2.3.5]; + print servers[1.2.3.6]; +} + diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 5e6bae7b62..bc1ab89bb2 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -42,6 +42,9 @@ event bro_init() $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); Input::force_update(A::INPUT); +} + +event Input::update_finished(id: Input::ID) { if ( 1 in servers ) { print "VALID"; } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro index 5af664e0e9..d5bff0c5bb 100644 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -35,6 +35,8 @@ type Val: record { global destination1: table[int] of Val = table(); global destination2: table[int] of Val = table(); +global done: bool = F; + event bro_init() { # first read in the old stuff into the table... @@ -45,6 +47,15 @@ event bro_init() Input::add_tablefilter(A::INPUT, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); Input::force_update(A::INPUT); +} + +event Input::update_finished(id: Input::ID) { + if ( done == T ) { + return; + } + + done = T; + if ( 1 in destination1 ) { print "VALID"; } @@ -90,6 +101,4 @@ event bro_init() if ( 7 in destination2 ) { print "VALID"; } - - } From a850cc59922118a8dd62e5c9c9e9c6abe22bd942 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 15 Feb 2012 15:14:04 -0800 Subject: [PATCH 059/149] make filter removal and stream closure asynchronous. --- src/input/Manager.cc | 64 ++++++++++++++++++--- src/input/Manager.h | 5 +- src/input/ReaderBackend.cc | 110 +++++++++++++++++------------------- src/input/ReaderBackend.h | 13 ++--- src/input/ReaderFrontend.cc | 29 ++++++++-- src/input/ReaderFrontend.h | 57 +++++++++++++++++++ 6 files changed, 200 insertions(+), 78 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 79d42fe71f..6655ae5e82 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -534,7 +534,6 @@ bool Manager::RemoveStream(const EnumVal* id) { if ( (*s)->id == id ) { i = (*s); - readers.erase(s); // remove from vector break; } } @@ -545,11 +544,29 @@ bool Manager::RemoveStream(const EnumVal* id) { i->reader->Finish(); - delete(i); - return true; } +bool Manager::RemoveStreamContinuation(const ReaderFrontend* reader) { + ReaderInfo *i = 0; + + + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->reader && (*s)->reader == reader ) + { + i = *s; + delete(i); + readers.erase(s); + return true; + } + } + + reporter->Error("Stream not found in RemoveStreamContinuation"); + return false; + +} + bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { @@ -615,20 +632,51 @@ bool Manager::RemoveTableFilter(EnumVal* id, const string &name) { return false; } - map::iterator it = i->filters.find(id->InternalInt()); - if ( it == i->filters.end() ) { + bool found = false; + int filterId; + + for ( map::iterator it = i->filters.begin(); it != i->filters.end(); ++it ) { + if ( (*it).second->name == name ) { + found = true; + filterId = (*it).first; + + if ( (*it).second->filter_type != TABLE_FILTER ) { + reporter->Error("Trying to remove filter %s of wrong type", name.c_str()); + return false; + } + + break; + } + } + + if ( !found ) { + reporter->Error("Trying to remove nonexisting filter %s", name.c_str()); return false; } - if ( i->filters[id->InternalInt()]->filter_type != TABLE_FILTER ) { - // wrong type; + i->reader->RemoveFilter(filterId); + + return true; +} + +bool Manager::RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->Error("Reader not found"); + return false; + } + + map::iterator it = i->filters.find(filterId); + if ( it == i->filters.end() ) { + reporter->Error("Got RemoveFilterContinuation where filter nonexistant for %d", filterId); return false; } delete (*it).second; i->filters.erase(it); + return true; -} +} bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); diff --git a/src/input/Manager.h b/src/input/Manager.h index 45c07895f2..9e35dd2199 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -32,13 +32,14 @@ public: protected: friend class ReaderFrontend; - friend class ErrorMessage; friend class PutMessage; friend class DeleteMessage; friend class ClearMessage; friend class SendEventMessage; friend class SendEntryMessage; friend class EndCurrentSendMessage; + friend class FilterRemovedMessage; + friend class ReaderFinishedMessage; // Reports an error for the given reader. void Error(ReaderFrontend* reader, const char* msg); @@ -56,6 +57,8 @@ protected: ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); + bool RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId); + bool RemoveStreamContinuation(const ReaderFrontend* reader); private: struct ReaderInfo; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index f9992f5f0e..5cb4fe34f2 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -9,21 +9,6 @@ using threading::Field; namespace input { -class ErrorMessage : public threading::OutputMessage { -public: - ErrorMessage(ReaderFrontend* reader, string message) - : threading::OutputMessage("Error", reader), - message(message) {} - - virtual bool Process() { - input_mgr->Error(Object(), message.c_str()); - return true; - } - -private: - string message; -}; - class PutMessage : public threading::OutputMessage { public: PutMessage(ReaderFrontend* reader, int id, Value* *val) @@ -104,7 +89,7 @@ private: class EndCurrentSendMessage : public threading::OutputMessage { public: - EndCurrentSendMessage(ReaderFrontend* reader, int id) + EndCurrentSendMessage(ReaderFrontend* reader, const int id) : threading::OutputMessage("EndCurrentSend", reader), id(id) {} @@ -114,9 +99,46 @@ public: } private: - int id; + const int id; }; +class FilterRemovedMessage : public threading::OutputMessage { +public: + FilterRemovedMessage(ReaderFrontend* reader, const int id) + : threading::OutputMessage("FilterRemoved", reader), + id(id) {} + + virtual bool Process() { + return input_mgr->RemoveFilterContinuation(Object(), id); + } + +private: + const int id; +}; + +class ReaderFinishedMessage : public threading::OutputMessage { +public: + ReaderFinishedMessage(ReaderFrontend* reader) + : threading::OutputMessage("ReaderFinished", reader) {} + + virtual bool Process() { + return input_mgr->RemoveStreamContinuation(Object()); + } + +private: +}; + + +class DisableMessage : public threading::OutputMessage +{ +public: + DisableMessage(ReaderFrontend* writer) + : threading::OutputMessage("Disable", writer) {} + + virtual bool Process() { Object()->SetDisable(); return true; } +}; + + ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() { buf = 0; @@ -133,18 +155,6 @@ ReaderBackend::~ReaderBackend() } -void ReaderBackend::Error(const string &msg) -{ - SendOut(new ErrorMessage(frontend, msg)); -} - -/* -void ReaderBackend::Error(const char *msg) -{ - SendOut(new ErrorMessage(frontend, string(msg))); -} */ - - void ReaderBackend::Put(int id, Value* *val) { SendOut(new PutMessage(frontend, id, val)); @@ -181,6 +191,11 @@ bool ReaderBackend::Init(string arg_source) // disable if DoInit returns error. disabled = !DoInit(arg_source); + + if ( disabled ) { + DisableFrontend(); + } + return !disabled; } @@ -192,13 +207,17 @@ bool ReaderBackend::AddFilter(int id, int arg_num_fields, bool ReaderBackend::RemoveFilter(int id) { - return DoRemoveFilter(id); + bool success = DoRemoveFilter(id); + SendOut(new FilterRemovedMessage(frontend, id)); + return success; // yes, I know, noone reads this. } void ReaderBackend::Finish() { DoFinish(); disabled = true; + DisableFrontend(); + SendOut(new ReaderFinishedMessage(frontend)); } bool ReaderBackend::Update() @@ -206,32 +225,9 @@ bool ReaderBackend::Update() return DoUpdate(); } - -// stolen from logwriter -const char* ReaderBackend::Fmt(const char* format, ...) - { - if ( ! buf ) - buf = (char*) malloc(buf_len); - - va_list al; - va_start(al, format); - int n = safe_vsnprintf(buf, buf_len, format, al); - va_end(al); - - if ( (unsigned int) n >= buf_len ) - { // Not enough room, grow the buffer. - buf_len = n + 32; - buf = (char*) realloc(buf, buf_len); - - // Is it portable to restart? - va_start(al, format); - n = safe_vsnprintf(buf, buf_len, format, al); - va_end(al); - } - - return buf; - } - - +void ReaderBackend::DisableFrontend() +{ + SendOut(new DisableMessage(frontend)); +} } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index c12d187545..de4a056c22 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -26,6 +26,12 @@ public: void Finish(); bool Update(); + + /** + * Disables the frontend that has instantiated this backend. Once + * disabled,the frontend will not send any further message over. + */ + void DisableFrontend(); protected: // Methods that have to be overwritten by the individual readers @@ -40,16 +46,9 @@ protected: // update file contents to logmgr virtual bool DoUpdate() = 0; - // Reports an error to the user. - void Error(const string &msg); - //void Error(const char *msg); - // The following methods return the information as passed to Init(). const string Source() const { return source; } - // A thread-safe version of fmt(). (stolen from logwriter) - const char* Fmt(const char* format, ...); - void SendEvent(const string& name, const int num_vals, threading::Value* *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index 0dac33d5e8..0fdf90d9ad 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -5,8 +5,9 @@ #include "ReaderBackend.h" #include "threading/MsgThread.h" -namespace input { +// FIXME: cleanup of disabled inputreaders is missing. we need this, because stuff can e.g. fail in init and might never be removed afterwards. +namespace input { class InitMessage : public threading::InputMessage { @@ -56,6 +57,19 @@ private: const threading::Field* const* fields; }; +class RemoveFilterMessage : public threading::InputMessage +{ +public: + RemoveFilterMessage(ReaderBackend* backend, const int id) + : threading::InputMessage("RemoveFilter", backend), + id(id) { } + + virtual bool Process() { return Object()->RemoveFilter(id); } + +private: + const int id; +}; + ReaderFrontend::ReaderFrontend(bro_int_t type) { disabled = initialized = false; @@ -103,15 +117,20 @@ void ReaderFrontend::AddFilter(const int id, const int arg_num_fields, const thr backend->SendIn(new AddFilterMessage(backend, id, arg_num_fields, fields)); } +void ReaderFrontend::RemoveFilter(const int id) { + if ( disabled ) + return; + + backend->SendIn(new RemoveFilterMessage(backend, id)); +} + string ReaderFrontend::Name() const - { +{ if ( source.size() ) return ty_name; return ty_name + "/" + source; - } - - +} } diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index 876082d9a6..97433c8af6 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -10,20 +10,77 @@ namespace input { class Manager; +/** + * Bridge class between the input::Manager and backend input threads. The + * Manager instantiates one \a ReaderFrontend for each open input stream. + * Each frontend in turns instantiates a ReaderBackend-derived class + * internally that's specific to the particular input format. That backend + * spawns a new thread, and it receives messages from the frontend that + * correspond to method called by the manager. + */ class ReaderFrontend { public: + /** + * Constructor. + * + * type: The backend writer type, with the value corresponding to the + * script-level \c Input::Reader enum (e.g., \a READER_ASCII). The + * frontend will internally instantiate a ReaderBackend of the + * corresponding type. + * + * Frontends must only be instantiated by the main thread. + */ ReaderFrontend(bro_int_t type); + /** + * Destructor. + * + * Frontends must only be destroyed by the main thread. + */ virtual ~ReaderFrontend(); + /** + * Initializes the reader. + * + * This method generates a message to the backend reader and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See ReaderBackend::Init() for arguments. + * This method must only be called from the main thread. + */ void Init(string arg_source); void Update(); + /* * The method takes + * ownership of \a fields. */ + void AddFilter( const int id, const int arg_num_fields, const threading::Field* const* fields ); + void RemoveFilter ( const int id ); + void Finish(); + /** + * Disables the reader frontend. From now on, all method calls that + * would normally send message over to the backend, turn into no-ops. + * Note though that it does not stop the backend itself, use Finsh() + * to do that as well (this method is primarily for use as callback + * when the backend wants to disable the frontend). + * + * Disabled frontend will eventually be discarded by the + * input::Manager. + * + * This method must only be called from the main thread. + */ + void SetDisable() { disabled = true; } + + /** + * Returns true if the reader frontend has been disabled with SetDisable(). + */ + bool Disabled() { return disabled; } + /** * Returns a descriptive name for the reader, including the type of * the backend and the source used. From 84883348ecc11643b6e96cecdbcdefce2bef45ba Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 16 Feb 2012 11:27:10 -0800 Subject: [PATCH 060/149] interface documentation. to a big part stolen from the logging framework --- scripts/base/frameworks/input/main.bro | 8 +- src/input/Manager.cc | 33 +++-- src/input/Manager.h | 122 ++++++++++++++++- src/input/ReaderBackend.h | 182 +++++++++++++++++++++++-- src/input/ReaderFrontend.h | 31 ++++- src/types.bif | 6 + 6 files changed, 350 insertions(+), 32 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index cac1aca54a..7e581070e6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,11 +4,14 @@ module Input; export { + redef enum Input::ID += { TABLE_READ }; - + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; + const default_mode = MANUAL &redef; + ## Stream decription type used for the `create_stream` method type StreamDescription: record { ## String that allows the reader to find the source. @@ -17,6 +20,9 @@ export { ## Reader to use for this steam reader: Reader &default=default_reader; + + ## Read mode to use for this stream + mode: Mode &default=default_mode; }; ## TableFilter description type used for the `add_tablefilter` method. diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 6655ae5e82..d4e5cdaee9 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -685,18 +685,28 @@ bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { return false; } - map::iterator it = i->filters.find(id->InternalInt()); - if ( it == i->filters.end() ) { + bool found = false; + int filterId; + for ( map::iterator it = i->filters.begin(); it != i->filters.end(); ++it ) { + if ( (*it).second->name == name ) { + found = true; + filterId = (*it).first; + + if ( (*it).second->filter_type != EVENT_FILTER ) { + reporter->Error("Trying to remove filter %s of wrong type", name.c_str()); + return false; + } + + break; + } + } + + if ( !found ) { + reporter->Error("Trying to remove nonexisting filter %s", name.c_str()); return false; } - if ( i->filters[id->InternalInt()]->filter_type != EVENT_FILTER ) { - // wrong type; - return false; - } - - delete (*it).second; - i->filters.erase(it); + i->reader->RemoveFilter(filterId); return true; } @@ -1136,11 +1146,6 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { return success; } -void Manager::Error(ReaderFrontend* reader, const char* msg) -{ - reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); -} - bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); diff --git a/src/input/Manager.h b/src/input/Manager.h index 9e35dd2199..be84ee416d 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -1,4 +1,6 @@ // See the file "COPYING" in the main distribution directory for copyright. +// +// Class for managing input streams and filters #ifndef INPUT_MANAGER_H #define INPUT_MANAGER_H @@ -16,18 +18,100 @@ namespace input { class ReaderFrontend; class ReaderBackend; +/** + * Singleton class for managing input streams. + */ class Manager { public: + /** + * Constructor. + */ Manager(); + + /** + * Destructor. + */ + ~Manager(); + /** + * Creates a new input stream. + * + * @param id The enum value corresponding the input stream. + * + * @param description A record of script type \c Input:StreamDescription. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ ReaderFrontend* CreateStream(EnumVal* id, RecordVal* description); + + /** + * Force update on a input stream. + * Forces a re-read of the whole input source. + * Usually used, when an input stream is opened in managed mode. + * Otherwise, this can be used to trigger a input source check before a heartbeat message arrives. + * May be ignored by the reader. + * + * @param id The enum value corresponding the input stream. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool ForceUpdate(const EnumVal* id); + + /** + * Deletes an existing input stream + * + * @param id The enum value corresponding the input stream. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool RemoveStream(const EnumVal* id); + /** + * Add a filter to an input source, which will write the data from the data source into + * a Bro table. + * + * @param id The enum value corresponding the input stream. + * + * @param description A record of script type \c Input:TableFilter. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool AddTableFilter(EnumVal *id, RecordVal* filter); + + /** + * Removes a tablefilter from the log stream + * + * @param id The enum value corresponding the input stream. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool RemoveTableFilter(EnumVal* id, const string &name); + /** + * Add a filter to an input source, which sends events for read input data. + * + * @param id The enum value corresponding the input stream. + * + * @param description A record of script type \c Input:EventFilter. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool AddEventFilter(EnumVal *id, RecordVal* filter); + + /** + * Removes a eventfilter from the log stream + * + * @param id The enum value corresponding the input stream. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool RemoveEventFilter(EnumVal* id, const string &name); protected: @@ -41,46 +125,76 @@ protected: friend class FilterRemovedMessage; friend class ReaderFinishedMessage; - // Reports an error for the given reader. - void Error(ReaderFrontend* reader, const char* msg); - - // for readers to write to input stream in direct mode (reporting new/deleted values directly) + // For readers to write to input stream in direct mode (reporting new/deleted values directly) + // Functions take ownership of threading::Value fields void Put(const ReaderFrontend* reader, int id, threading::Value* *vals); void Clear(const ReaderFrontend* reader, int id); bool Delete(const ReaderFrontend* reader, int id, threading::Value* *vals); // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) + // Functions take ownership of threading::Value fields void SendEntry(const ReaderFrontend* reader, const int id, threading::Value* *vals); void EndCurrentSend(const ReaderFrontend* reader, const int id); + // Allows readers to directly send Bro events. + // The num_vals and vals must be the same the named event expects. + // Takes ownership of threading::Value fields bool SendEvent(const string& name, const int num_vals, threading::Value* *vals); + // Instantiates a new ReaderBackend of the given type (note that + // doing so creates a new thread!). ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); + // Functions are called from the ReaderBackend to notify the manager, that a filter has been removed + // or a stream has been closed. + // Used to prevent race conditions where data for a specific filter is still in the queue when the + // RemoveFilter directive is executed by the main thread. + // This makes sure all data that has ben queued for a filter is still received. bool RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId); bool RemoveStreamContinuation(const ReaderFrontend* reader); private: struct ReaderInfo; + // SendEntry implementation for Tablefilter int SendEntryTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + + // Put implementation for Tablefilter int PutTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + + // SendEntry and Put implementation for Eventfilter int SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const threading::Value* const *vals); + // Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types + // from the log framework bool IsCompatibleType(BroType* t, bool atomic_only=false); + // Check if a record is made up of compatible types and return a list of all fields that are in the record in order. + // Recursively unrolls records bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + // Send events void SendEvent(EventHandlerPtr ev, const int numvals, ...); void SendEvent(EventHandlerPtr ev, list events); + // get a hashkey for a set of threading::Values HashKey* HashValues(const int num_elements, const threading::Value* const *vals); + + // Get the memory used by a specific value int GetValueLength(const threading::Value* val); + // Copies the raw data in a specific threading::Value to position sta int CopyValue(char *data, const int startpos, const threading::Value* val); + // Convert Threading::Value to an internal Bro Type (works also with Records) Val* ValueToVal(const threading::Value* val, BroType* request_type); + + // Convert Threading::Value to an internal Bro List type Val* ValueToIndexVal(int num_fields, const RecordType* type, const threading::Value* const *vals); + + // Converts a threading::value to a record type. mostly used by ValueToVal RecordVal* ValueToRecordVal(const threading::Value* const *vals, RecordType *request_type, int* position); + + // Converts a Bro ListVal to a RecordVal given the record type RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); ReaderInfo* FindReader(const ReaderFrontend* reader); diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index de4a056c22..c6fbaac715 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -11,20 +11,90 @@ namespace input { class ReaderFrontend; +/** + * Base class for reader implementation. When the input:Manager creates a + * new input stream, it instantiates a ReaderFrontend. That then in turn + * creates a ReaderBackend of the right type. The frontend then forwards + * message over the backend as its methods are called. + * + * All of this methods must be called only from the corresponding child + * thread (the constructor is the one exception.) + */ class ReaderBackend : public threading::MsgThread { public: + /** + * Constructor. + * + * @param frontend The frontend reader that created this backend. The + * *only* purpose of this value is to be passed back via messages as + * a argument to callbacks. One must not otherwise access the + * frontend, it's running in a different thread. + * + * @param frontend pointer to the reader frontend + */ ReaderBackend(ReaderFrontend* frontend); + /** + * Destructor. + */ virtual ~ReaderBackend(); - + + /** + * One-time initialization of the reader to define the input source. + * + * @param arg_source A string left to the interpretation of the reader + * implementation; it corresponds to the value configured on the + * script-level for the input stream. + * + * @param num_fields The number of log fields for the stream. + * + * @param fields An array of size \a num_fields with the log fields. + * The methods takes ownership of the array. + * + * @return False if an error occured. + */ bool Init(string arg_source); + /** + * Add an input filter to the input stream + * + * @param id identifier of the input stream + * + * @param arg_num_fields number of fields contained in \a fields + * + * @param fields the types and names of the fields to be retrieved from the input source + * + * @return False if an error occured. + */ bool AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + + /** + * Remove an input filter to the input stream + * + * @param id identifier of the input stream + * + * @return False if an error occured. + */ bool RemoveFilter ( int id ); + /** + * Finishes reading from this input stream in a regular fashion. Must not be + * called if an error has been indicated earlier. After calling this, + * no further reading from the stream can be performed + * + * @return False if an error occured. + */ void Finish(); + /** + * Force trigger an update of the input stream. + * The action that will be taken depends on the current read mode and the individual input backend + * + * An backend can choose to ignore this. + * + * @return False if an error occured. + */ bool Update(); /** @@ -34,30 +104,126 @@ public: void DisableFrontend(); protected: - // Methods that have to be overwritten by the individual readers + // Methods that have to be overwritten by the individual readers + + /** + * Reader-specific intialization method. + * + * A reader implementation must override this method. If it returns + * false, it will be assumed that a fatal error has occured that + * prevents the reader from further operation; it will then be + * disabled and eventually deleted. When returning false, an + * implementation should also call Error() to indicate what happened. + */ virtual bool DoInit(string arg_sources) = 0; + /** + * Reader-specific method to add a filter. + * + * A reader implementation must override this method. + */ virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ) = 0; + /** + * Reader-specific method to remove a filter. + * + * A reader implementation must override this method. + */ virtual bool DoRemoveFilter( int id ) = 0; + /** + * Reader-specific method implementing input finalization at + * termination. + * + * A reader implementation must override this method but it can just + * ignore calls if an input source must not be closed. + * + * After the method is called, the writer will be deleted. If an error occurs + * during shutdown, an implementation should also call Error() to indicate what + * happened. + */ virtual void DoFinish() = 0; - // update file contents to logmgr + /** + * Reader-specific method implementing the forced update trigger + * + * A reader implementation must override this method but it can just ignore + * calls, if a forced update does not fit the input source or the current input + * reading mode + */ virtual bool DoUpdate() = 0; - // The following methods return the information as passed to Init(). + /** + * Returns the input source as passed into the constructor. + */ const string Source() const { return source; } + /** + * Method allowing a reader to send a specified bro event. + * Vals must match the values expected by the bro event. + * + * @param name name of the bro event to send + * + * @param num_vals number of entries in \a vals + * + * @param vals the values to be given to the event + */ void SendEvent(const string& name, const int num_vals, threading::Value* *vals); - // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table + // Content-sending-functions (simple mode). Including table-specific stuff that simply is not used if we have no table + /** + * Method allowing a reader to send a list of values read for a specific filter back to the manager. + * + * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised + * + * @param id the input filter id for which the values are sent + * + * @param val list of threading::Values expected by the filter + */ void Put(int id, threading::Value* *val); + + /** + * Method allowing a reader to delete a specific value from a bro table. + * + * If the receiving filter is an event, only a removed event is raised + * + * @param id the input filter id for which the values are sent + * + * @param val list of threading::Values expected by the filter + */ void Delete(int id, threading::Value* *val); + + /** + * Method allowing a reader to clear a value from a bro table. + * + * If the receiving filter is an event, this is ignored. + * + * @param id the input filter id for which the values are sent + */ void Clear(int id); - // Table-functions (tracking mode): Only changed lines are propagated. + // Content-sending-functions (tracking mode): Only changed lines are propagated. + + + /** + * Method allowing a reader to send a list of values read for a specific filter back to the manager. + * + * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised. + * + * @param id the input filter id for which the values are sent + * + * @param val list of threading::Values expected by the filter + */ void SendEntry(int id, threading::Value* *vals); + + /** + * Method telling the manager, that the current list of entries sent by SendEntry is finished. + * + * For table filters, all entries that were not updated since the last EndCurrentSend will be deleted, because they are no longer + * present in the input source + * + * @param id the input filter id for which the values are sent + */ void EndCurrentSend(int id); @@ -68,11 +234,7 @@ private: string source; - // When an error occurs, this method is called to set a flag marking the - // writer as disabled. - bool disabled; - bool Disabled() { return disabled; } // For implementing Fmt(). char* buf; diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index 97433c8af6..c29071612d 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -51,15 +51,37 @@ public: */ void Init(string arg_source); + /** + * Force an update of the current input source. Actual action depends on + * the opening mode and on the input source. + * + * This method generates a message to the backend reader and triggers + * the corresponding message there. + * This method must only be called from the main thread. + */ void Update(); - /* * The method takes - * ownership of \a fields. */ - + /** + * Add a filter to the current input source. + * + * See ReaderBackend::AddFilter for arguments. + * + * The method takes ownership of \a fields + */ void AddFilter( const int id, const int arg_num_fields, const threading::Field* const* fields ); + /** + * Removes a filter to the current input source. + */ void RemoveFilter ( const int id ); + /** + * Finalizes writing to this tream. + * + * This method generates a message to the backend reader and triggers + * the corresponding message there. + * This method must only be called from the main thread. + */ void Finish(); /** @@ -92,6 +114,9 @@ public: protected: friend class Manager; + /** + * Returns the source as passed into the constructor + */ const string Source() const { return source; }; string ty_name; // Name of the backend type. Set by the manager. diff --git a/src/types.bif b/src/types.bif index 9256fe3bd0..1529319197 100644 --- a/src/types.bif +++ b/src/types.bif @@ -185,4 +185,10 @@ enum ID %{ Unknown, %} +enum Mode %{ + MANUAL, + REREAD, + STREAM, +%} + module GLOBAL; From 91943c26559590c858f8c7e30db6ffc41a68ec59 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 16 Feb 2012 15:03:20 -0800 Subject: [PATCH 061/149] * rework script interface, add autostart stream flag that starts up a stream automatically when first filter has been added ( probably the most common use case ) * change internal reader interface again * remove some quite embarassing bugs that must have been in the interface for rather long * add different read methods to script & internal interface (like normal, streaming, etc). Not implemented in ascii reader yet. --- scripts/base/frameworks/input/main.bro | 3 + src/input/Manager.cc | 24 ++++---- src/input/ReaderBackend.cc | 32 +++++++++- src/input/ReaderBackend.h | 46 +++++++++++++-- src/input/ReaderFrontend.cc | 31 ++++++++-- src/input/ReaderFrontend.h | 13 ++++- src/input/readers/Ascii.cc | 58 +++++++++++++++++-- src/input/readers/Ascii.h | 10 +++- src/threading/Manager.cc | 2 +- src/types.bif | 6 +- .../scripts/base/frameworks/input/basic.bro | 1 - .../scripts/base/frameworks/input/event.bro | 1 - .../frameworks/input/onecolumn-norecord.bro | 1 - .../frameworks/input/onecolumn-record.bro | 1 - .../scripts/base/frameworks/input/port.bro | 1 - .../base/frameworks/input/predicate.bro | 1 - .../base/frameworks/input/tableevent.bro | 1 - .../base/frameworks/input/twofilters.bro | 2 +- 18 files changed, 191 insertions(+), 43 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 7e581070e6..445f947106 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -23,6 +23,9 @@ export { ## Read mode to use for this stream mode: Mode &default=default_mode; + + ## Automatically start the input stream after the first filter has been added + autostart: bool &default=T; }; ## TableFilter description type used for the `add_tablefilter` method. diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d4e5cdaee9..4438a07c6c 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -214,6 +214,10 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) } EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); + EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); + Val *autostart = description->LookupWithDefault(rtype->FieldOffset("autostart")); + bool do_autostart = ( autostart->InternalInt() == 1 ); + Unref(autostart); // Ref'd by LookupWithDefault ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); assert(reader_obj); @@ -229,16 +233,7 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) readers.push_back(info); - reader_obj->Init(source); - /* if ( success == false ) { - assert( RemoveStream(id) ); - return 0; - } */ - reader_obj->Update(); - /* if ( success == false ) { - assert ( RemoveStream(id) ); - return 0; - } */ + reader_obj->Init(source, mode->InternalInt(), do_autostart); return reader_obj; @@ -785,7 +780,7 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va //reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = 0; if ( filter->num_val_fields > 0 ) - HashValues(filter->num_val_fields, vals+filter->num_idx_fields); + valhash = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); @@ -794,6 +789,13 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before + + valhash->Hash(); + + h->valhash->Hash(); + + + if ( filter->num_val_fields == 0 || h->valhash->Hash() == valhash->Hash() ) { // ok, exact duplicate filter->lastDict->Remove(idxhash); diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 5cb4fe34f2..cfc74d33a8 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -185,24 +185,42 @@ void ReaderBackend::SendEntry(int id, Value* *vals) SendOut(new SendEntryMessage(frontend, id, vals)); } -bool ReaderBackend::Init(string arg_source) +bool ReaderBackend::Init(string arg_source, int mode, bool arg_autostart) { source = arg_source; + autostart = arg_autostart; + SetName("InputReader/"+source); // disable if DoInit returns error. - disabled = !DoInit(arg_source); + disabled = !DoInit(arg_source, mode); if ( disabled ) { + Error("Init failed"); DisableFrontend(); } return !disabled; } +bool ReaderBackend::StartReading() { + int success = DoStartReading(); + + if ( success == false ) { + DisableFrontend(); + } + + return success; +} + bool ReaderBackend::AddFilter(int id, int arg_num_fields, const Field* const * arg_fields) { - return DoAddFilter(id, arg_num_fields, arg_fields); + bool success = DoAddFilter(id, arg_num_fields, arg_fields); + if ( success && autostart) { + autostart = false; + return StartReading(); + } + return success; } bool ReaderBackend::RemoveFilter(int id) @@ -230,4 +248,12 @@ void ReaderBackend::DisableFrontend() SendOut(new DisableMessage(frontend)); } +bool ReaderBackend::DoHeartbeat(double network_time, double current_time) +{ + MsgThread::DoHeartbeat(network_time, current_time); + + return true; +} + + } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index c6fbaac715..e34db3e559 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -51,9 +51,24 @@ public: * @param fields An array of size \a num_fields with the log fields. * The methods takes ownership of the array. * + * @param mode the opening mode for the input source + * + * @param autostart automatically start the input source after the first filter has been added + * * @return False if an error occured. */ - bool Init(string arg_source); + bool Init(string arg_source, int mode, bool autostart); + + /** + * One-time start method of the reader. + * + * This method is called from the scripting layer, after all filters have been added. + * No data should be read before this method is called. + * + * If autostart in Init is set to true, this method is called automatically by the backend after + * the first filter has been added. + */ + bool StartReading(); /** * Add an input filter to the input stream @@ -107,7 +122,8 @@ protected: // Methods that have to be overwritten by the individual readers /** - * Reader-specific intialization method. + * Reader-specific intialization method. Note that data may only be read from the input source + * after the Start function has been called. * * A reader implementation must override this method. If it returns * false, it will be assumed that a fatal error has occured that @@ -115,7 +131,19 @@ protected: * disabled and eventually deleted. When returning false, an * implementation should also call Error() to indicate what happened. */ - virtual bool DoInit(string arg_sources) = 0; + virtual bool DoInit(string arg_sources, int mode) = 0; + + /** + * Reader-specific start method. After this function has been called, data may be read from + * the input source and be sent to the specified filters + * + * A reader implementation must override this method. + * If it returns false, it will be assumed that a fatal error has occured + * that prevents the reader from further operation; it will then be + * disabled and eventually deleted. When returning false, an implementation + * should also call Error to indicate what happened. + */ + virtual bool DoStartReading() = 0; /** * Reader-specific method to add a filter. @@ -225,7 +253,14 @@ protected: * @param id the input filter id for which the values are sent */ void EndCurrentSend(int id); - + + /** + * Triggered by regular heartbeat messages from the main thread. + * + * This method can be overridden but once must call + * ReaderBackend::DoHeartbeat(). + */ + virtual bool DoHeartbeat(double network_time, double current_time); private: // Frontend that instantiated us. This object must not be access from @@ -238,7 +273,8 @@ private: // For implementing Fmt(). char* buf; - unsigned int buf_len; + unsigned int buf_len; + bool autostart; }; } diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index 0fdf90d9ad..f7fc23bf72 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -12,14 +12,16 @@ namespace input { class InitMessage : public threading::InputMessage { public: - InitMessage(ReaderBackend* backend, const string source) + InitMessage(ReaderBackend* backend, const string source, const int mode, const bool autostart) : threading::InputMessage("Init", backend), - source(source) { } + source(source), mode(mode), autostart(autostart) { } - virtual bool Process() { return Object()->Init(source); } + virtual bool Process() { return Object()->Init(source, mode, autostart); } private: const string source; + const int mode; + const bool autostart; }; class UpdateMessage : public threading::InputMessage @@ -42,6 +44,16 @@ public: virtual bool Process() { Object()->Finish(); return true; } }; +class StartReadingMessage : public threading::InputMessage +{ +public: + StartReadingMessage(ReaderBackend* backend) + : threading::InputMessage("StartReading", backend) + { } + + virtual bool Process() { Object()->StartReading(); return true; } +}; + class AddFilterMessage : public threading::InputMessage { public: @@ -83,17 +95,17 @@ ReaderFrontend::ReaderFrontend(bro_int_t type) { ReaderFrontend::~ReaderFrontend() { } -void ReaderFrontend::Init(string arg_source) { +void ReaderFrontend::Init(string arg_source, int mode, bool autostart) { if ( disabled ) return; if ( initialized ) - reporter->InternalError("writer initialize twice"); + reporter->InternalError("reader initialize twice"); source = arg_source; initialized = true; - backend->SendIn(new InitMessage(backend, arg_source)); + backend->SendIn(new InitMessage(backend, arg_source, mode, autostart)); } void ReaderFrontend::Update() { @@ -132,6 +144,13 @@ string ReaderFrontend::Name() const return ty_name + "/" + source; } +void ReaderFrontend::StartReading() { + if ( disabled ) + return; + + backend->SendIn(new StartReadingMessage(backend)); +} + } diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index c29071612d..d67ca299c0 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -49,7 +49,18 @@ public: * See ReaderBackend::Init() for arguments. * This method must only be called from the main thread. */ - void Init(string arg_source); + void Init(string arg_source, int mode, bool autostart); + + /** + * Start the reader. + * + * This methods starts the reader, after all necessary filters have been added. + * It is not necessary to call this function, if autostart has been set. + * If autostart has been set, the reader will be initialized automatically after the first filter has been added + * + * This method must only be called from the main thread. + */ + void StartReading(); /** * Force an update of the current input source. Actual action depends on diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 095d74bf11..cd1723e5e4 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -8,10 +8,14 @@ #include "../../threading/SerializationTypes.h" +#define MANUAL 0 +#define REREAD 1 + using namespace input::reader; using threading::Value; using threading::Field; + FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) : name(arg_name), type(arg_type) { @@ -75,16 +79,41 @@ void Ascii::DoFinish() } } -bool Ascii::DoInit(string path) +bool Ascii::DoInit(string path, int arg_mode) { + started = false; fname = path; + mode = arg_mode; file = new ifstream(path.c_str()); if ( !file->is_open() ) { - Error(Fmt("cannot open %s", fname.c_str())); + Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } + if ( ( mode != MANUAL ) && (mode != REREAD) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); + return false; + } + + return true; +} + +bool Ascii::DoStartReading() { + if ( started == true ) { + Error("Started twice"); + return false; + } + + started = true; + switch ( mode ) { + case MANUAL: + DoUpdate(); + break; + default: + assert(false); + } + return true; } @@ -132,7 +161,7 @@ bool Ascii::ReadHeader() { map fields; - // construcr list of field names. + // construct list of field names. istringstream splitstream(line); int pos=0; while ( splitstream ) { @@ -146,6 +175,7 @@ bool Ascii::ReadHeader() { for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + (*it).second.columnMap.clear(); for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { const Field* field = (*it).second.fields[i]; @@ -372,7 +402,6 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { // read the entire file and send appropriate thingies back to InputMgr bool Ascii::DoUpdate() { - // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) if ( file && file->is_open() ) { file->close(); @@ -418,6 +447,7 @@ bool Ascii::DoUpdate() { fit != (*it).second.columnMap.end(); fit++ ){ + if ( (*fit).position > pos || (*fit).secondary_position > pos ) { Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); return false; @@ -455,6 +485,7 @@ bool Ascii::DoUpdate() { } + //file->clear(); // remove end of file evil bits //file->seekg(0, ios::beg); // and seek to start. @@ -463,3 +494,22 @@ bool Ascii::DoUpdate() { } return true; } + +bool Ascii::DoHeartbeat(double network_time, double current_time) +{ + ReaderBackend::DoHeartbeat(network_time, current_time); + + switch ( mode ) { + case MANUAL: + // yay, we do nothing :) + break; + case REREAD: + + + default: + assert(false); + } + + return true; +} + diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index a3bf5c21a6..766716e29d 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -39,7 +39,7 @@ public: protected: - virtual bool DoInit(string path); + virtual bool DoInit(string path, int mode); virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); @@ -48,9 +48,13 @@ protected: virtual void DoFinish(); virtual bool DoUpdate(); + + virtual bool DoStartReading(); private: + virtual bool DoHeartbeat(double network_time, double current_time); + struct Filter { unsigned int num_fields; @@ -84,6 +88,10 @@ private: string unset_field; + int mode; + + bool started; + }; diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 7b571e753c..472d10139a 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -106,7 +106,7 @@ void Manager::Process() Message* msg = t->RetrieveOut(); - if ( msg->Process() && network_time ) + if ( msg->Process() ) //&& network_time ) // FIXME: ask robin again if he needs this. makes input interface not work in bro_init. did_process = true; else diff --git a/src/types.bif b/src/types.bif index 1529319197..e2a47a7ece 100644 --- a/src/types.bif +++ b/src/types.bif @@ -186,9 +186,9 @@ enum ID %{ %} enum Mode %{ - MANUAL, - REREAD, - STREAM, + MANUAL = 0, + REREAD = 1, + STREAM = 2, %} module GLOBAL; diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 3b75220625..156898edca 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -47,7 +47,6 @@ event bro_init() # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::INPUT); Input::remove_tablefilter(A::INPUT, "ssh"); Input::remove_stream(A::INPUT); } diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index a07f0934a0..41eba1613c 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -38,5 +38,4 @@ event bro_init() { Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); - Input::force_update(A::INPUT); } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 712a877960..bcbba05a3e 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -33,7 +33,6 @@ event bro_init() # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); - Input::force_update(A::INPUT); } event Input::update_finished(id: Input::ID) { diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 7b62ddcddd..1c532ba6a9 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -33,7 +33,6 @@ event bro_init() # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::INPUT); } event Input::update_finished(id: Input::ID) { diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 65d73c54f7..801d6bac3f 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -32,7 +32,6 @@ event bro_init() # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::INPUT); print servers[1.2.3.4]; print servers[1.2.3.5]; print servers[1.2.3.6]; diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index bc1ab89bb2..009911e6a8 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -41,7 +41,6 @@ event bro_init() Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); - Input::force_update(A::INPUT); } event Input::update_finished(id: Input::ID) { diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index 36e8171689..0c86ac94b8 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -44,5 +44,4 @@ event bro_init() { Input::create_stream(A::LOG, [$source="input.log"]); Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); - Input::force_update(A::LOG); } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro index d5bff0c5bb..260f73e58f 100644 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -40,7 +40,7 @@ global done: bool = F; event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); + Input::create_stream(A::INPUT, [$source="input.log", $autostart=F]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); From d21a450f36ba621f5802f9c1c7b9f28ce2ec264d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 16 Feb 2012 15:40:07 -0800 Subject: [PATCH 062/149] add streaming reads & automatic re-reading of files to ascii reader. completely untested, but compiles & old tests still work --- src/input/Manager.cc | 7 ---- src/input/readers/Ascii.cc | 70 ++++++++++++++++++++++++++++++-------- src/input/readers/Ascii.h | 1 + 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 4438a07c6c..ea4c5643fa 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -789,13 +789,6 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before - - valhash->Hash(); - - h->valhash->Hash(); - - - if ( filter->num_val_fields == 0 || h->valhash->Hash() == valhash->Hash() ) { // ok, exact duplicate filter->lastDict->Remove(idxhash); diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index cd1723e5e4..5a3569a95f 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -10,6 +10,11 @@ #define MANUAL 0 #define REREAD 1 +#define STREAM 2 + +#include +#include +#include using namespace input::reader; using threading::Value; @@ -84,6 +89,7 @@ bool Ascii::DoInit(string path, int arg_mode) started = false; fname = path; mode = arg_mode; + mtime = 0; file = new ifstream(path.c_str()); if ( !file->is_open() ) { @@ -91,7 +97,7 @@ bool Ascii::DoInit(string path, int arg_mode) return false; } - if ( ( mode != MANUAL ) && (mode != REREAD) ) { + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; } @@ -402,23 +408,58 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { // read the entire file and send appropriate thingies back to InputMgr bool Ascii::DoUpdate() { - // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) - if ( file && file->is_open() ) { - file->close(); - } - file = new ifstream(fname.c_str()); - if ( !file->is_open() ) { - Error(Fmt("cannot open %s", fname.c_str())); - return false; + switch ( mode ) { + case REREAD: + // check if the file has changed + struct stat sb; + if ( stat(fname.c_str(), &sb) == -1 ) { + Error(Fmt("Could not get stat for %s", fname.c_str())); + return false; + } + + if ( sb.st_mtime <= mtime ) { + // no change + return true; + } + + mtime = sb.st_mtime; + // file changed. reread. + + // fallthrough + case MANUAL: + case STREAM: + + // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) + if ( file && file->is_open() ) { + if ( mode == STREAM ) { + file->clear(); // remove end of file evil bits + break; + } + file->close(); + } + file = new ifstream(fname.c_str()); + if ( !file->is_open() ) { + Error(Fmt("cannot open %s", fname.c_str())); + return false; + } + + + if ( ReadHeader() == false ) { + return false; + } + + break; + default: + assert(false); + } + + // // file->seekg(0, ios::beg); // do not forget clear. - if ( ReadHeader() == false ) { - return false; - } string line; while ( GetLine(line ) ) { @@ -504,8 +545,9 @@ bool Ascii::DoHeartbeat(double network_time, double current_time) // yay, we do nothing :) break; case REREAD: - - + case STREAM: + DoUpdate(); + break; default: assert(false); } diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 766716e29d..017e5630d4 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -91,6 +91,7 @@ private: int mode; bool started; + time_t mtime; }; From 4126b458ca002093bf964243bb5cdd8a45931544 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 20 Feb 2012 13:18:15 -0800 Subject: [PATCH 063/149] Automatic file re-refresh and streaming works. * simple testcase for file refresh (check for changes) and streaming reads * add events for simple put and delete operations * fix bugs in table filter events (type for first element was wrong) * and I think a couple of other small bugs --- .../scripts.base.frameworks.input.reread/out | 67 ++++++++++ .../scripts.base.frameworks.input.stream/out | 115 ++++++++++++++++++ .../scripts/base/frameworks/input/reread.bro | 92 ++++++++++++++ .../scripts/base/frameworks/input/stream.bro | 89 ++++++++++++++ 4 files changed, 363 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.reread/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.stream/out create mode 100644 testing/btest/scripts/base/frameworks/input/reread.bro create mode 100644 testing/btest/scripts/base/frameworks/input/stream.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out new file mode 100644 index 0000000000..4234a5056d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -0,0 +1,67 @@ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +{ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.stream/out b/testing/btest/Baseline/scripts.base.frameworks.input.stream/out new file mode 100644 index 0000000000..39b06c9092 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.stream/out @@ -0,0 +1,115 @@ +============EVENT============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============SERVERS============ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Input::EVENT_CHANGED +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============SERVERS============ +{ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +done diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro new file mode 100644 index 0000000000..5058f4a068 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -0,0 +1,92 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cp input2.log input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + # first read in the old stuff into the table... + Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::REREAD]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); +} + +event Input::update_finished(id: Input::ID) { + print outfile, servers; + + try = try + 1; + if ( try == 3 ) { + print outfile, "done"; + close(outfile); + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); + } +} diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro new file mode 100644 index 0000000000..db368074aa --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -0,0 +1,89 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input2.log +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input3.log +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(tpe: Input::Event, left: Idx, right: Val) { + print outfile, "============EVENT============"; + print outfile, tpe; + print outfile, left; + print outfile, right; + print outfile, "============SERVERS============"; + print outfile, servers; + + try = try + 1; + + if ( try == 3 ) { + print outfile, "done"; + close(outfile); + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + # first read in the old stuff into the table... + Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::STREAM]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); +} + From 4f57817b1a7af9b4084a22d00e195c46afa3c3af Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 20 Feb 2012 13:20:29 -0800 Subject: [PATCH 064/149] ...forgotten in last commit. --- src/input/Manager.cc | 163 +++++++++++++++++++++++++++++++++---- src/input/readers/Ascii.cc | 56 +++++++++---- src/input/readers/Ascii.h | 5 +- 3 files changed, 191 insertions(+), 33 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index ea4c5643fa..66dadfdb2d 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -818,13 +818,6 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } - Val* oldval = 0; - if ( updated == true ) { - assert(filter->num_val_fields > 0); - // in that case, we need the old value to send the event (if we send an event). - oldval = filter->tab->Lookup(idxval); - } - // call filter first to determine if we really add / change the entry if ( filter->pred ) { @@ -865,6 +858,13 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } + Val* oldval = 0; + if ( updated == true ) { + assert(filter->num_val_fields > 0); + // in that case, we need the old value to send the event (if we send an event). + oldval = filter->tab->Lookup(idxval); + } + //i->tab->Assign(idxval, valval); HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { @@ -884,21 +884,22 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va if ( filter->event ) { EnumVal* ev; - Ref(idxval); + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); if ( updated ) { // in case of update send back the old value. assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); - SendEvent(filter->event, 3, ev, idxval, oldval); + SendEvent(filter->event, 3, ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); if ( filter->num_val_fields == 0 ) { - SendEvent(filter->event, 3, ev, idxval); + SendEvent(filter->event, 3, ev, predidx); } else { - SendEvent(filter->event, 3, ev, idxval, valval); + SendEvent(filter->event, 3, ev, predidx, valval); } } } @@ -973,10 +974,11 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { } if ( filter->event ) { - Ref(idx); + int startpos = 0; + Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(filter->event, 3, ev, idx, val); + SendEvent(filter->event, 3, ev, predidx, val); } filter->tab->Delete(ih->idxkey); @@ -991,8 +993,6 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { filter->currDict = new PDict(InputHash); // Send event that the current update is indeed finished. - - EventHandler* handler = event_registry->Lookup("Input::update_finished"); if ( handler == 0 ) { reporter->InternalError("Input::update_finished not found!"); @@ -1077,6 +1077,7 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; + int position = filter->num_idx_fields; if ( filter->num_val_fields == 0 ) { @@ -1087,7 +1088,91 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * valval = ValueToRecordVal(vals, filter->rtype, &position); } - filter->tab->Assign(idxval, valval); + // if we have a subscribed event, we need to figure out, if this is an update or not + // same for predicates + if ( filter->pred || filter->event ) { + bool updated = false; + Val* oldval = 0; + + if ( filter->num_val_fields > 0 ) { + // in that case, we need the old value to send the event (if we send an event). + oldval = filter->tab->Lookup(idxval, false); + } + + if ( oldval != 0 ) { + // it is an update + updated = true; + Ref(oldval); // have to do that, otherwise it may disappear in assign + } + + + // predicate if we want the update or not + if ( filter->pred ) { + EnumVal* ev; + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + Ref(valval); + + if ( updated ) { + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + } + + val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. + vl.append(ev); + vl.append(predidx); + if ( filter->num_val_fields > 0 ) + vl.append(valval); + + Val* v = filter->pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // do nothing + Unref(idxval); + Unref(valval); + Unref(oldval); + return filter->num_val_fields + filter->num_idx_fields; + } + + } + + + filter->tab->Assign(idxval, valval); + + if ( filter->event ) { + EnumVal* ev; + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + + if ( updated ) { // in case of update send back the old value. + assert ( filter->num_val_fields > 0 ); + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + assert ( oldval != 0 ); + SendEvent(filter->event, 3, ev, predidx, oldval); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + Ref(valval); + if ( filter->num_val_fields == 0 ) { + SendEvent(filter->event, 3, ev, predidx); + } else { + SendEvent(filter->event, 3, ev, predidx, valval); + } + } + + } + + + + + + } else { + // no predicates or other stuff + + filter->tab->Assign(idxval, valval); + } return filter->num_idx_fields + filter->num_val_fields; } @@ -1122,8 +1207,52 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { if ( i->filters[id]->filter_type == TABLE_FILTER ) { TableFilter* filter = (TableFilter*) i->filters[id]; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); + assert(idxval != 0); readVals = filter->num_idx_fields + filter->num_val_fields; - success = ( filter->tab->Delete(idxval) != 0 ); + bool filterresult = true; + + if ( filter->pred || filter->event ) { + Val *val = filter->tab->Lookup(idxval); + + if ( filter->pred ) { + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + + val_list vl(3); + vl.append(ev); + vl.append(predidx); + vl.append(val); + Val* v = filter->pred->Call(&vl); + filterresult = v->AsBool(); + Unref(v); + + if ( filterresult == false ) { + // keep it. + Unref(idxval); + success = true; + } + + } + + // only if filter = true -> no filtering + if ( filterresult && filter->event ) { + Ref(idxval); + assert(val != 0); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(filter->event, 3, ev, idxval, val); + } + } + + // only if filter = true -> no filtering + if ( filterresult ) { + success = ( filter->tab->Delete(idxval) != 0 ); + if ( !success ) { + reporter->Error("Internal error while deleting values from input table"); + } + } } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); readVals = SendEventFilterEvent(reader, type, id, vals); diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 5a3569a95f..b0b046b75b 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -91,16 +91,22 @@ bool Ascii::DoInit(string path, int arg_mode) mode = arg_mode; mtime = 0; + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); + return false; + } + file = new ifstream(path.c_str()); if ( !file->is_open() ) { Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } - - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { - Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); + + if ( ReadHeader(false) == false ) { + Error(Fmt("Init: cannot open %s; headers are incorrect", fname.c_str())); + file->close(); return false; - } + } return true; } @@ -114,6 +120,8 @@ bool Ascii::DoStartReading() { started = true; switch ( mode ) { case MANUAL: + case REREAD: + case STREAM: DoUpdate(); break; default: @@ -157,16 +165,25 @@ bool Ascii::HasFilter(int id) { } -bool Ascii::ReadHeader() { +bool Ascii::ReadHeader(bool useCached) { // try to read the header line... string line; - if ( !GetLine(line) ) { - Error("could not read first line"); - return false; - } - map fields; + if ( !useCached ) { + if ( !GetLine(line) ) { + Error("could not read first line"); + return false; + } + + + + headerline = line; + + } else { + line = headerline; + } + // construct list of field names. istringstream splitstream(line); int pos=0; @@ -179,7 +196,7 @@ bool Ascii::ReadHeader() { pos++; } - + //printf("Updating fields from description %s\n", line.c_str()); for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { (*it).second.columnMap.clear(); @@ -433,6 +450,7 @@ bool Ascii::DoUpdate() { if ( file && file->is_open() ) { if ( mode == STREAM ) { file->clear(); // remove end of file evil bits + ReadHeader(true); // in case filters changed break; } file->close(); @@ -444,7 +462,7 @@ bool Ascii::DoUpdate() { } - if ( ReadHeader() == false ) { + if ( ReadHeader(false) == false ) { return false; } @@ -512,9 +530,14 @@ bool Ascii::DoUpdate() { fpos++; } + //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); assert ( (unsigned int) fpos == (*it).second.num_fields ); - SendEntry((*it).first, fields); + if ( mode == STREAM ) { + Put((*it).first, fields); + } else { + SendEntry((*it).first, fields); + } /* Do not do this, ownership changes to other thread * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { @@ -530,9 +553,12 @@ bool Ascii::DoUpdate() { //file->clear(); // remove end of file evil bits //file->seekg(0, ios::beg); // and seek to start. - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - EndCurrentSend((*it).first); + if ( mode != STREAM ) { + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + EndCurrentSend((*it).first); + } } + return true; } diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 017e5630d4..d2376e4fe1 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -69,7 +69,7 @@ private: TransportProto StringToProto(const string &proto); - bool ReadHeader(); + bool ReadHeader(bool useCached); threading::Value* EntryToVal(string s, FieldMapping type); bool GetLine(string& str); @@ -87,6 +87,9 @@ private: string empty_field; string unset_field; + + // keep a copy of the headerline to determine field locations when filters change + string headerline; int mode; From fe5b376d2858d599c2a14ddc8003c59c217550c5 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 20 Feb 2012 13:23:25 -0800 Subject: [PATCH 065/149] ...and update for table event testcase after fix. --- .../scripts.base.frameworks.input.tableevent/out | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out index e32a2aea00..54048a86b8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -1,21 +1,21 @@ Input::EVENT_NEW -1 +[i=1] T Input::EVENT_NEW -2 +[i=2] T Input::EVENT_NEW -3 +[i=3] F Input::EVENT_NEW -4 +[i=4] F Input::EVENT_NEW -5 +[i=5] F Input::EVENT_NEW -6 +[i=6] F Input::EVENT_NEW -7 +[i=7] T From edd30da082d288d295939429689c0a74a0787340 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 20 Feb 2012 15:30:21 -0800 Subject: [PATCH 066/149] better testcase & fix a few bugs (that took way too long to find). --- src/input/Manager.cc | 34 ++++++++----- src/input/readers/Ascii.cc | 21 +++++--- .../scripts.base.frameworks.input.reread/out | 48 +++++++++++++++++++ .../scripts/base/frameworks/input/reread.bro | 11 ++++- 4 files changed, 95 insertions(+), 19 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 66dadfdb2d..243567e0e6 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -22,7 +22,7 @@ using threading::Value; using threading::Field; struct InputHash { - HashKey* valhash; + hash_t valhash; HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. }; @@ -776,11 +776,15 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashValues(filter->num_idx_fields, vals); - //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); + //reporter->Error("Result: %d\n", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = 0; - if ( filter->num_val_fields > 0 ) - valhash = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); + + hash_t valhash = 0; + if ( filter->num_val_fields > 0 ) { + HashKey* valhashkey = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); + valhash = valhashkey->Hash(); + delete(valhashkey); + } //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); @@ -789,7 +793,7 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before - if ( filter->num_val_fields == 0 || h->valhash->Hash() == valhash->Hash() ) { + if ( filter->num_val_fields == 0 || h->valhash == valhash ) { // ok, exact duplicate filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); @@ -862,7 +866,7 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va if ( updated == true ) { assert(filter->num_val_fields > 0); // in that case, we need the old value to send the event (if we send an event). - oldval = filter->tab->Lookup(idxval); + oldval = filter->tab->Lookup(idxval, false); } //i->tab->Assign(idxval, valval); @@ -872,6 +876,8 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va return filter->num_val_fields + filter->num_idx_fields; } + if ( filter->event && updated ) + Ref(oldval); // otherwise it is no longer accessible after the assignment filter->tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); @@ -891,7 +897,6 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); - Ref(oldval); SendEvent(filter->event, 3, ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); @@ -1468,7 +1473,7 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { int length = 0; memcpy(data+startpos, (const void*) &(val->val.port_val.port), sizeof(val->val.port_val.port)); length += sizeof(val->val.port_val.port); - memcpy(data+startpos, (const void*) &(val->val.port_val.proto), sizeof(val->val.port_val.proto)); + memcpy(data+startpos+length, (const void*) &(val->val.port_val.proto), sizeof(val->val.port_val.proto)); length += sizeof(val->val.port_val.proto); return length; break; @@ -1500,7 +1505,7 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { int length = 0; memcpy(data+startpos,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); length += sizeof(val->val.subnet_val.width); - memcpy(data+startpos, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + memcpy(data+startpos+length, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); length += sizeof(val->val.subnet_val.net); return length; break; @@ -1508,7 +1513,8 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { case TYPE_TABLE: { int length = 0; - for ( int i = 0; i < val->val.set_val.size; i++ ) { + int j = val->val.set_val.size; + for ( int i = 0; i < j; i++ ) { length += CopyValue(data, startpos+length, val->val.set_val.vals[i]); } return length; @@ -1531,6 +1537,7 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { } reporter->InternalError("internal error"); + assert(false); return 0; } @@ -1550,13 +1557,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { if ( data == 0 ) { reporter->InternalError("Could not malloc?"); } + memset(data, 0, length); for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; position += CopyValue(data, position, val); } + hash_t key = HashKey::HashBytes(data, length); + assert(position == length); - return new HashKey(data, length); + return new HashKey(data, length, key, true); } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index b0b046b75b..d4b3d91e00 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -268,7 +268,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { if ( s.compare(unset_field) == 0 ) { // field is not set... return new Value(field.type, false); } - + switch ( field.type ) { case TYPE_ENUM: case TYPE_STRING: @@ -302,6 +302,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { break; case TYPE_PORT: + val->val.port_val.port = 0; val->val.port_val.port = atoi(s.c_str()); val->val.port_val.proto = TRANSPORT_UNKNOWN; break; @@ -312,19 +313,27 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; - // NOTE: dotted_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... #ifdef BROv6 if ( s.find(':') != s.npos ) { - uint32* addr = dotted_to_addr6(s.c_str()); + uint32* addr = new uint32[4]; + if ( inet_pton(AF_INET6, s.c_str(), addr) <= 0 ) { + Error(Fmt("Bad IPv6 address: %s", s.c_str())); + val->val.subnet_val.net[0] = val->val.subnet_val.net[1] = val->val.subnet_val.net[2] = val->val.subnet_val.net[3] = 0; + } copy_addr(val->val.subnet_val.net, addr); delete addr; } else { val->val.subnet_val.net[0] = val->val.subnet_val.net[1] = val->val.subnet_val.net[2] = 0; - val->val.subnet_val.net[3] = dotted_to_addr(s.c_str()); + if ( inet_aton(s.c_str(), &(val->val.subnet_val.net[3])) <= 0 ) { + Error(Fmt("Bad addres: %s", s.c_str())); + val->val.subnet_val.net[3] = 0; + } } #else - val->val.subnet_val.net = dotted_to_addr(s.c_str()); + if ( inet_aton(s.c_str(), (in_addr*) &(val->val.subnet_val.net)) <= 0 ) { + Error(Fmt("Bad addres: %s", s.c_str())); + val->val.subnet_val.net = 0; + } #endif break; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 4234a5056d..9516cb2a92 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -1,3 +1,19 @@ +============EVENT============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ { [-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, @@ -12,6 +28,22 @@ BB }, vc=[10, 20, 30], ve=[]] } +============EVENT============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ { [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, @@ -38,6 +70,22 @@ BB }, vc=[10, 20, 30], ve=[]] } +============EVENT============ +Input::EVENT_CHANGED +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ { [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 5058f4a068..58df37af84 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -70,16 +70,25 @@ global outfile: file; global try: count; +event line(tpe: Input::Event, left: Idx, right: Val) { + print outfile, "============EVENT============"; + print outfile, tpe; + print outfile, left; + print outfile, right; +} + event bro_init() { outfile = open ("../out"); try = 0; # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::REREAD]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); } + event Input::update_finished(id: Input::ID) { + print outfile, "==========SERVERS============"; print outfile, servers; try = try + 1; From d5b413c4e719eed31a998c7ddc4d5361c8fbb670 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 20 Feb 2012 17:13:41 -0800 Subject: [PATCH 067/149] reduce number of needed hash operations --- src/input/Manager.cc | 26 ++++++++------------------ src/input/Manager.h | 2 -- src/input/readers/Ascii.cc | 1 - 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 243567e0e6..a7afdc3a78 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -782,8 +782,8 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va hash_t valhash = 0; if ( filter->num_val_fields > 0 ) { HashKey* valhashkey = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); - valhash = valhashkey->Hash(); - delete(valhashkey); + valhash = valhashkey->Hash(); + delete(valhashkey); } //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); @@ -873,19 +873,17 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); - return filter->num_val_fields + filter->num_idx_fields; + assert(false); } + InputHash* ih = new InputHash(); + ih->idxkey = new HashKey(k->Key(), k->Size(), k->Hash()); + ih->valhash = valhash; + if ( filter->event && updated ) Ref(oldval); // otherwise it is no longer accessible after the assignment filter->tab->Assign(idxval, k, valval); - InputHash* ih = new InputHash(); - k = filter->tab->ComputeHash(idxval); - ih->idxkey = k; - ih->valhash = valhash; - //i->tab->Delete(k); - filter->currDict->Insert(idxhash, ih); if ( filter->event ) { @@ -1557,7 +1555,7 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { if ( data == 0 ) { reporter->InternalError("Could not malloc?"); } - memset(data, 0, length); + //memset(data, 0, length); for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; position += CopyValue(data, position, val); @@ -1695,11 +1693,3 @@ Manager::ReaderInfo* Manager::FindReader(const EnumVal* id) return 0; } - -string Manager::Hash(const string &input) { - unsigned char digest[16]; - hash_md5(input.length(), (const unsigned char*) input.c_str(), digest); - string out((const char*) digest, 16); - return out; -} - diff --git a/src/input/Manager.h b/src/input/Manager.h index be84ee416d..b4fc6cff7f 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -202,8 +202,6 @@ private: vector readers; - string Hash(const string &input); - class Filter; class TableFilter; class EventFilter; diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index d4b3d91e00..e128cd1164 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -302,7 +302,6 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { break; case TYPE_PORT: - val->val.port_val.port = 0; val->val.port_val.port = atoi(s.c_str()); val->val.port_val.proto = TRANSPORT_UNKNOWN; break; From 531189b5fdd39fb9f927b8af075957ad2892b9d4 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 22 Feb 2012 08:56:45 -0800 Subject: [PATCH 068/149] try to make ascii reader a little bit more robust to failure - mainly ignore messages after a reader has disabled itself --- src/input/ReaderBackend.cc | 20 +++++++++++++++++++- src/input/ReaderBackend.h | 7 ++++++- src/input/readers/Ascii.cc | 8 ++++++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index cfc74d33a8..8ddb6a2f42 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -203,6 +203,9 @@ bool ReaderBackend::Init(string arg_source, int mode, bool arg_autostart) } bool ReaderBackend::StartReading() { + if ( disabled ) + return false; + int success = DoStartReading(); if ( success == false ) { @@ -215,6 +218,9 @@ bool ReaderBackend::StartReading() { bool ReaderBackend::AddFilter(int id, int arg_num_fields, const Field* const * arg_fields) { + if ( disabled ) + return false; + bool success = DoAddFilter(id, arg_num_fields, arg_fields); if ( success && autostart) { autostart = false; @@ -225,6 +231,9 @@ bool ReaderBackend::AddFilter(int id, int arg_num_fields, bool ReaderBackend::RemoveFilter(int id) { + if ( disabled ) + return false; + bool success = DoRemoveFilter(id); SendOut(new FilterRemovedMessage(frontend, id)); return success; // yes, I know, noone reads this. @@ -240,11 +249,20 @@ void ReaderBackend::Finish() bool ReaderBackend::Update() { - return DoUpdate(); + if ( disabled ) + return false; + + bool success = DoUpdate(); + if ( !success ) { + DisableFrontend(); + } + + return success; } void ReaderBackend::DisableFrontend() { + disabled = true; // we also set disabled here, because there still may be other messages queued and we will dutifully ignore these from now SendOut(new DisableMessage(frontend)); } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index e34db3e559..68fd5f3a37 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -177,7 +177,12 @@ protected: * * A reader implementation must override this method but it can just ignore * calls, if a forced update does not fit the input source or the current input - * reading mode + * reading mode. + * + * If it returns false, it will be assumed that a fatal error has occured + * that prevents the reader from further operation; it will then be + * disabled and eventually deleted. When returning false, an implementation + * should also call Error to indicate what happened. */ virtual bool DoUpdate() = 0; diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index e128cd1164..73b8500d5e 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -458,7 +458,10 @@ bool Ascii::DoUpdate() { if ( file && file->is_open() ) { if ( mode == STREAM ) { file->clear(); // remove end of file evil bits - ReadHeader(true); // in case filters changed + if ( !ReadHeader(true) ) // in case filters changed + { + return false; // header reading failed + } break; } file->close(); @@ -522,6 +525,7 @@ bool Ascii::DoUpdate() { Value* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { + Error("Could not convert String value to Val"); return false; } @@ -580,7 +584,7 @@ bool Ascii::DoHeartbeat(double network_time, double current_time) break; case REREAD: case STREAM: - DoUpdate(); + Update(); // call update and not DoUpdate, because update actually checks disabled. break; default: assert(false); From 7e5f7338269d22332950b2ed9dd446c673fa21c3 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 22 Feb 2012 09:44:45 -0800 Subject: [PATCH 069/149] raw input reader for seth, which can simply read a file into string-events given a line separator. --- scripts/base/frameworks/input/__load__.bro | 1 + scripts/base/frameworks/input/readers/raw.bro | 9 + src/CMakeLists.txt | 1 + src/input.bif | 2 + src/input/Manager.cc | 2 + src/input/readers/Ascii.cc | 6 +- src/input/readers/Raw.cc | 230 ++++++++++++++++++ src/input/readers/Raw.h | 70 ++++++ src/types.bif | 1 + .../scripts.base.frameworks.input.raw/out | 8 + .../scripts/base/frameworks/input/raw.bro | 35 +++ 11 files changed, 363 insertions(+), 2 deletions(-) create mode 100644 scripts/base/frameworks/input/readers/raw.bro create mode 100644 src/input/readers/Raw.cc create mode 100644 src/input/readers/Raw.h create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.raw/out create mode 100644 testing/btest/scripts/base/frameworks/input/raw.bro diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro index a3315186d5..b41fe5e95f 100644 --- a/scripts/base/frameworks/input/__load__.bro +++ b/scripts/base/frameworks/input/__load__.bro @@ -1,3 +1,4 @@ @load ./main @load ./readers/ascii +@load ./readers/raw diff --git a/scripts/base/frameworks/input/readers/raw.bro b/scripts/base/frameworks/input/readers/raw.bro new file mode 100644 index 0000000000..45deed3eda --- /dev/null +++ b/scripts/base/frameworks/input/readers/raw.bro @@ -0,0 +1,9 @@ +##! Interface for the raw input reader. + +module InputRaw; + +export { + ## Separator between input records. + ## Please note that the separator has to be exactly one character long + const record_separator = "\n" &redef; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6a84053bce..dd294ace7c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -424,6 +424,7 @@ set(bro_SRCS input/ReaderBackend.cc input/ReaderFrontend.cc input/readers/Ascii.cc + input/readers/Raw.cc ${dns_SRCS} diff --git a/src/input.bif b/src/input.bif index 2e9324ec56..5418b7bbd4 100644 --- a/src/input.bif +++ b/src/input.bif @@ -62,3 +62,5 @@ const set_separator: string; const empty_field: string; const unset_field: string; +module InputRaw; +const record_separator: string; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index a7afdc3a78..d3009aa619 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -6,6 +6,7 @@ #include "ReaderFrontend.h" #include "ReaderBackend.h" #include "readers/Ascii.h" +#include "readers/Raw.h" #include "Event.h" #include "EventHandler.h" @@ -143,6 +144,7 @@ struct ReaderDefinition { ReaderDefinition input_readers[] = { { BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate }, + { BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate }, // End marker { BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 73b8500d5e..733cca6352 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -133,6 +133,7 @@ bool Ascii::DoStartReading() { bool Ascii::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { if ( HasFilter(id) ) { + Error("Filter was added twice, ignoring."); return false; // no, we don't want to add this a second time } @@ -147,6 +148,7 @@ bool Ascii::DoAddFilter( int id, int arg_num_fields, const Field* const* fields bool Ascii::DoRemoveFilter ( int id ) { if (!HasFilter(id) ) { + Error("Filter removal of nonexisting filter requested."); return false; } @@ -263,11 +265,11 @@ TransportProto Ascii::StringToProto(const string &proto) { Value* Ascii::EntryToVal(string s, FieldMapping field) { - Value* val = new Value(field.type, true); - if ( s.compare(unset_field) == 0 ) { // field is not set... return new Value(field.type, false); } + + Value* val = new Value(field.type, true); switch ( field.type ) { case TYPE_ENUM: diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc new file mode 100644 index 0000000000..c435624865 --- /dev/null +++ b/src/input/readers/Raw.cc @@ -0,0 +1,230 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Raw.h" +#include "NetVar.h" + +#include +#include + +#include "../../threading/SerializationTypes.h" + +#define MANUAL 0 +#define REREAD 1 +#define STREAM 2 + +#include +#include +#include + +using namespace input::reader; +using threading::Value; +using threading::Field; + +Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend) +{ + file = 0; + + //keyMap = new map(); + + separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(), BifConst::InputRaw::record_separator->Len()); + if ( separator.size() != 1 ) { + Error("separator length has to be 1. Separator will be truncated."); + } + +} + +Raw::~Raw() +{ + DoFinish(); +} + +void Raw::DoFinish() +{ + filters.empty(); + if ( file != 0 ) { + file->close(); + delete(file); + file = 0; + } +} + +bool Raw::DoInit(string path, int arg_mode) +{ + started = false; + fname = path; + mode = arg_mode; + mtime = 0; + + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); + return false; + } + + file = new ifstream(path.c_str()); + if ( !file->is_open() ) { + Error(Fmt("Init: cannot open %s", fname.c_str())); + return false; + } + + return true; +} + +bool Raw::DoStartReading() { + if ( started == true ) { + Error("Started twice"); + return false; + } + + started = true; + switch ( mode ) { + case MANUAL: + case REREAD: + case STREAM: + DoUpdate(); + break; + default: + assert(false); + } + + return true; +} + +bool Raw::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { + + if ( arg_num_fields != 1 ) { + Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); + return false; + } + + if ( fields[0]->type != TYPE_STRING ) { + Error("Filter for raw reader contains a field that is not of type string."); + return false; + } + + if ( HasFilter(id) ) { + Error("Filter was added twice, ignoring"); + return false; // no, we don't want to add this a second time + } + + Filter f; + f.num_fields = arg_num_fields; + f.fields = fields; + + filters[id] = f; + + return true; +} + +bool Raw::DoRemoveFilter ( int id ) { + if (!HasFilter(id) ) { + Error("Filter removal of nonexisting filter requested."); + return false; + } + + assert ( filters.erase(id) == 1 ); + + return true; +} + + +bool Raw::HasFilter(int id) { + map::iterator it = filters.find(id); + if ( it == filters.end() ) { + return false; + } + return true; +} + +bool Raw::GetLine(string& str) { + while ( getline(*file, str, separator[0]) ) { + return true; + } + + return false; +} + + +// read the entire file and send appropriate thingies back to InputMgr +bool Raw::DoUpdate() { + switch ( mode ) { + case REREAD: + // check if the file has changed + struct stat sb; + if ( stat(fname.c_str(), &sb) == -1 ) { + Error(Fmt("Could not get stat for %s", fname.c_str())); + return false; + } + + if ( sb.st_mtime <= mtime ) { + // no change + return true; + } + + mtime = sb.st_mtime; + // file changed. reread. + + // fallthrough + case MANUAL: + case STREAM: + + if ( file && file->is_open() ) { + if ( mode == STREAM ) { + file->clear(); // remove end of file evil bits + break; + } + file->close(); + } + file = new ifstream(fname.c_str()); + if ( !file->is_open() ) { + Error(Fmt("cannot open %s", fname.c_str())); + return false; + } + + break; + default: + assert(false); + + } + + string line; + while ( GetLine(line) ) { + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + + assert ((*it).second.num_fields == 1); + + Value** fields = new Value*[1]; + + // filter has exactly one text field. convert to it. + Value* val = new Value(TYPE_STRING, true); + val->val.string_val = new string(line); + fields[0] = val; + + Put((*it).first, fields); + + } + + } + + return true; +} + + +bool Raw::DoHeartbeat(double network_time, double current_time) +{ + ReaderBackend::DoHeartbeat(network_time, current_time); + + switch ( mode ) { + case MANUAL: + // yay, we do nothing :) + break; + case REREAD: + case STREAM: + Update(); // call update and not DoUpdate, because update actually checks disabled. + break; + default: + assert(false); + } + + return true; +} + diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h new file mode 100644 index 0000000000..e046cb2ff7 --- /dev/null +++ b/src/input/readers/Raw.h @@ -0,0 +1,70 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERS_RAW_H +#define INPUT_READERS_RAW_H + +#include +#include + +#include "../ReaderBackend.h" + +namespace input { namespace reader { + +class Raw : public ReaderBackend { +public: + Raw(ReaderFrontend* frontend); + ~Raw(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); } + +protected: + + virtual bool DoInit(string path, int mode); + + virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + + virtual bool DoRemoveFilter ( int id ); + + virtual void DoFinish(); + + virtual bool DoUpdate(); + + virtual bool DoStartReading(); + +private: + + virtual bool DoHeartbeat(double network_time, double current_time); + + struct Filter { + unsigned int num_fields; + + const threading::Field* const * fields; // raw mapping + }; + + bool HasFilter(int id); + + bool GetLine(string& str); + + ifstream* file; + string fname; + + map filters; + + // Options set from the script-level. + string separator; + + // keep a copy of the headerline to determine field locations when filters change + string headerline; + + int mode; + + bool started; + time_t mtime; + +}; + + +} +} + +#endif /* INPUT_READERS_RAW_H */ diff --git a/src/types.bif b/src/types.bif index e2a47a7ece..a9c6ecb3a8 100644 --- a/src/types.bif +++ b/src/types.bif @@ -173,6 +173,7 @@ module Input; enum Reader %{ READER_DEFAULT, READER_ASCII, + READER_RAW, %} enum Event %{ diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out new file mode 100644 index 0000000000..2059013c5d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out @@ -0,0 +1,8 @@ +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro new file mode 100644 index 0000000000..5f196648b6 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -0,0 +1,35 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Val: record { + s: string; +}; + +event line(tpe: Input::Event, s: string) { + print s; +} + +event bro_init() +{ + Input::create_stream(A::INPUT, [$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM]); + Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); +} From 93fac7a4be74d441001ec3ddb70e13655bee636c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 22 Feb 2012 10:46:35 -0800 Subject: [PATCH 070/149] fix one of the bugs seth found in the input framework. (bug in PutTable when the table contained only one element and that element should not be wrapped into a record) --- src/input/Manager.cc | 6 +- .../out | 7 + .../scripts.base.frameworks.input.reread/out | 267 ++++++++++++++++++ .../frameworks/input/predicate-stream.bro | 83 ++++++ .../scripts/base/frameworks/input/reread.bro | 32 ++- 5 files changed, 388 insertions(+), 7 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out create mode 100644 testing/btest/scripts/base/frameworks/input/predicate-stream.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d3009aa619..9a350bef20 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1083,12 +1083,11 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; - int position = filter->num_idx_fields; if ( filter->num_val_fields == 0 ) { valval = 0; - } else if ( filter->num_val_fields == 1 && !filter->want_record ) { - valval = ValueToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); + } else if ( filter->num_val_fields == 1 && filter->want_record == 0 ) { + valval = ValueToVal(vals[position], filter->rtype->FieldType(0)); } else { valval = ValueToRecordVal(vals, filter->rtype, &position); } @@ -1130,6 +1129,7 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * if ( filter->num_val_fields > 0 ) vl.append(valval); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out new file mode 100644 index 0000000000..d805f804d8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out @@ -0,0 +1,7 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 9516cb2a92..b844990978 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -110,6 +110,273 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Input::EVENT_NEW +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_NEW +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_NEW +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_NEW +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_NEW +[i=-48] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Input::EVENT_REMOVED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_REMOVED +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_REMOVED +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_REMOVED +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_REMOVED +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] } done diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro new file mode 100644 index 0000000000..f08aaef998 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -0,0 +1,83 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out +# +# only difference from predicate.bro is, that this one uses a stream source. +# the reason is, that the code-paths are quite different, because then the ascii reader uses the put and not the sendevent interface + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global servers: table[int] of Val = table(); +global ct: int; + +event line(tpe: Input::Event, left: Idx, right: bool) { + ct = ct + 1; + if ( ct < 3 ) { + return; + } + if ( ct > 3 ) { + print "Too many events"; + return; + } + + if ( 1 in servers ) { + print "VALID"; + } + if ( 2 in servers ) { + print "VALID"; + } + if ( !(3 in servers) ) { + print "VALID"; + } + if ( !(4 in servers) ) { + print "VALID"; + } + if ( !(5 in servers) ) { + print "VALID"; + } + if ( !(6 in servers) ) { + print "VALID"; + } + if ( 7 in servers ) { + print "VALID"; + } +} + +event bro_init() +{ + ct = 0; + # first read in the old stuff into the table... + Input::create_stream(A::INPUT, [$source="input.log", $mode=Input::STREAM]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } + ]); +} + diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 58df37af84..8e573494fd 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -1,11 +1,15 @@ # # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro %INPUT -# @TEST-EXEC: sleep 3 +# @TEST-EXEC: sleep 2 # @TEST-EXEC: cp input2.log input.log -# @TEST-EXEC: sleep 3 +# @TEST-EXEC: sleep 2 # @TEST-EXEC: cp input3.log input.log -# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input4.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input5.log input.log +# @TEST-EXEC: btest-bg-wait -k 2 # @TEST-EXEC: btest-diff out @TEST-START-FILE input1.log @@ -31,6 +35,26 @@ T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} @TEST-END-FILE +@TEST-START-FILE input4.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -45 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -46 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -47 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input5.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE @load frameworks/communication/listen @@ -92,7 +116,7 @@ event Input::update_finished(id: Input::ID) { print outfile, servers; try = try + 1; - if ( try == 3 ) { + if ( try == 5 ) { print outfile, "done"; close(outfile); Input::remove_tablefilter(A::INPUT, "ssh"); From d81607c3e93452595e3453cbfe9c378fb71a40fb Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 23 Feb 2012 14:36:04 -0800 Subject: [PATCH 071/149] fix empty field bug in threaded version --- src/input/Manager.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 9a350bef20..9dba56b174 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1547,7 +1547,8 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; - length += GetValueLength(val); + if ( val->present ) + length += GetValueLength(val); } //reporter->Error("Length: %d", length); @@ -1560,7 +1561,8 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { //memset(data, 0, length); for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; - position += CopyValue(data, position, val); + if ( val->present ) + position += CopyValue(data, position, val); } hash_t key = HashKey::HashBytes(data, length); From d553a3c6f69c1110b37196632e47cb6ffd10e406 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 23 Feb 2012 15:30:39 -0800 Subject: [PATCH 072/149] fix strange bug when using predicates and events at the same time on a tablefilter. Testcase is now more involved. --- src/input/Manager.cc | 52 +++-- .../scripts.base.frameworks.input.reread/out | 212 +++++++++++++++++- .../scripts/base/frameworks/input/reread.bro | 10 +- 3 files changed, 249 insertions(+), 25 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 9dba56b174..b709d26ea8 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -364,20 +364,21 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { } - Val* name = fval->Lookup(rtype->FieldOffset("name")); - Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); + Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); - RecordType *idx = fval->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = 0; if ( fval->Lookup(rtype->FieldOffset("val")) != 0 ) { - val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + val = fval->LookupWithDefault(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); } - TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); - Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); Func* event = event_val ? event_val->AsFunc() : 0; + Unref(event_val); if ( event ) { FuncType* etype = event->FType()->AsFuncType(); @@ -450,14 +451,17 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { filter->pred = pred ? pred->AsFunc() : 0; filter->num_idx_fields = idxfields; filter->num_val_fields = valfields; - filter->tab = dst->Ref()->AsTableVal(); - filter->rtype = val ? val->Ref()->AsRecordType() : 0; - filter->itype = idx->Ref()->AsRecordType(); + filter->tab = dst->AsTableVal(); + filter->rtype = val ? val->AsRecordType() : 0; + filter->itype = idx->AsRecordType(); filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter->currDict = new PDict(InputHash); filter->lastDict = new PDict(InputHash); filter->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + Unref(name); + Unref(pred); if ( valfields > 1 ) { assert(filter->want_record); @@ -861,7 +865,7 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } } - } + } Val* oldval = 0; @@ -948,16 +952,16 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { val = filter->tab->Lookup(idx); assert(val != 0); } + int startpos = 0; + Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); + EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + if ( filter->pred ) { - - bool doBreak = false; // ask predicate, if we want to expire this element... - EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - //Ref(idx); - int startpos = 0; - Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); + Ref(ev); + Ref(predidx); Ref(val); val_list vl(3); @@ -971,21 +975,23 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... + Unref(predidx); + Unref(ev); filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); continue; - } - - - } + } + } if ( filter->event ) { - int startpos = 0; - Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); + Ref(predidx); Ref(val); - EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + Ref(ev); SendEvent(filter->event, 3, ev, predidx, val); } + Unref(predidx); + Unref(ev); + filter->tab->Delete(ih->idxkey); filter->lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index b844990978..f244f11a73 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -1,3 +1,18 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW [i=-42] @@ -28,6 +43,21 @@ BB }, vc=[10, 20, 30], ve=[]] } +============PREDICATE============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW [i=-43] @@ -70,6 +100,21 @@ BB }, vc=[10, 20, 30], ve=[]] } +============PREDICATE============ +Input::EVENT_CHANGED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_CHANGED [i=-43] @@ -112,6 +157,21 @@ BB }, vc=[10, 20, 30], ve=[]] } +============PREDICATE============ +Input::EVENT_NEW +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW [i=-44] @@ -126,6 +186,21 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW @@ -142,7 +217,7 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -============EVENT============ +============PREDICATE============ Input::EVENT_NEW [i=-46] [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ @@ -159,6 +234,21 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW [i=-47] [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, @@ -171,6 +261,36 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_NEW +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-48] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW @@ -274,6 +394,96 @@ BB }, vc=[10, 20, 30], ve=[]] } +============PREDICATE============ +Input::EVENT_REMOVED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_REMOVED [i=-43] diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 8e573494fd..742d68605b 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -107,7 +107,15 @@ event bro_init() try = 0; # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::REREAD]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); } From faf5c95752c2cb32f269b624262676902a22426f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 11 Mar 2012 19:41:41 -0700 Subject: [PATCH 073/149] a couple of small fixes ( default values, all null lines) --- src/input/Manager.cc | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index a647b3c945..27580e0e82 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -104,7 +104,6 @@ struct Manager::ReaderInfo { EnumVal* type; ReaderFrontend* reader; - //list events; // events we fire when "something" happens map filters; // filters that can prevent our actions bool HasFilter(int id); @@ -160,7 +159,7 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { - reporter->Error("unknown reader when creating reader"); + reporter->Error("The reader that was requested was not found and could not be initialized."); return 0; } @@ -181,7 +180,7 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) } else { // ohok. init failed, kill factory for all eternity ir->factory = 0; - DBG_LOG(DBG_LOGGING, "failed to init input class %s", ir->name); + DBG_LOG(DBG_LOGGING, "Failed to init input class %s", ir->name); return 0; } @@ -225,8 +224,11 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) assert(reader_obj); // get the source... - const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); + Val* sourceval = description->LookupWithDefault(rtype->FieldOffset("source")); + assert ( sourceval != 0 ); + const BroString* bsource = sourceval->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); + Unref(sourceval); ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; @@ -255,13 +257,14 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { return false; } - Val* name = fval->Lookup(rtype->FieldOffset("name")); - RecordType *fields = fval->Lookup(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); + Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); + RecordType *fields = fval->LookupWithDefault(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); - Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); Func* event = event_val->AsFunc(); + Unref(event_val); { FuncType* etype = event->FType()->AsFuncType(); @@ -330,8 +333,10 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { logf[i] = fieldsV[i]; } + Unref(fields); // ref'd by lookupwithdefault EventFilter* filter = new EventFilter(); filter->name = name->AsString()->CheckString(); + Unref(name); // ref'd by lookupwithdefault filter->id = id->Ref()->AsEnumVal(); filter->num_fields = fieldsV.size(); filter->fields = fields->Ref()->AsRecordType(); @@ -369,8 +374,9 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = 0; - if ( fval->Lookup(rtype->FieldOffset("val")) != 0 ) { + if ( fval->LookupWithDefault(rtype->FieldOffset("val")) != 0 ) { val = fval->LookupWithDefault(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + Unref(val); // The lookupwithdefault in the if-clause ref'ed val. } TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); @@ -780,10 +786,12 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va assert(i->filters[id]->filter_type == TABLE_FILTER); TableFilter* filter = (TableFilter*) i->filters[id]; - //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashValues(filter->num_idx_fields, vals); - //reporter->Error("Result: %d\n", (uint64_t) idxhash->Hash()); - //reporter->Error("Hashing %d val fields", i->num_val_fields); + + if ( idxhash == 0 ) { + reporter->Error("Could not hash line. Ignoring"); + return filter->num_val_fields + filter->num_idx_fields; + } hash_t valhash = 0; if ( filter->num_val_fields > 0 ) { @@ -792,10 +800,6 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va delete(valhashkey); } - //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); - - //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before @@ -1609,14 +1613,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { length += GetValueLength(val); } - //reporter->Error("Length: %d", length); + if ( length == 0 ) { + reporter->Error("Input reader sent line where all elements are null values. Ignoring line"); + return NULL; + } int position = 0; char *data = (char*) malloc(length); if ( data == 0 ) { reporter->InternalError("Could not malloc?"); } - //memset(data, 0, length); for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; if ( val->present ) From 92555badd4d6bca6a3f9ed96c3dab1087aac9940 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 11 Mar 2012 20:43:26 -0700 Subject: [PATCH 074/149] cleanup, more sanity tests, a little bit more documentation --- doc/input.rst | 27 +++-- scripts/base/frameworks/input/main.bro | 3 - src/input/Manager.cc | 159 ++++++++++++++----------- src/input/Manager.h | 3 + 4 files changed, 110 insertions(+), 82 deletions(-) diff --git a/doc/input.rst b/doc/input.rst index 78e96fe06e..e201af9fed 100644 --- a/doc/input.rst +++ b/doc/input.rst @@ -34,9 +34,11 @@ very similar to the abstracts used in the logging framework: Readers A reader defines the input format for the specific input stream. - At the moment, Bro comes with only one type of reader, which can - read the tab seperated ASCII logfiles that were generated by the + At the moment, Bro comes with two types of reader. The default reader is READER_ASCII, + which can read the tab seperated ASCII logfiles that were generated by the logging framework. + READER_RAW can files containing records separated by a character(like e.g. newline) and send + one event per line. Basics @@ -68,7 +70,21 @@ The fields that can be set when creating a stream are: ``reader`` The reader used for this stream. Default is ``READER_ASCII``. - + + ``mode`` + The mode in which the stream is opened. Possible values are ``MANUAL``, ``REREAD`` and ``STREAM``. + Default is ``MANUAL``. + ``MANUAL`` means, that the files is not updated after it has been read. Changes to the file will not + be reflected in the data bro knows. + ``REREAD`` means that the whole file is read again each time a change is found. This should be used for + files that are mapped to a table where individual lines can change. + ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new + data is added to the file. + + ``autostart`` + If set to yes, the first update operation is triggered automatically after the first filter has been added to the stream. + This has to be set to false if several filters are added to the input source. + In this case Input::force_update has to be called manually once after all filters have been added. Filters ======= @@ -101,9 +117,6 @@ could be defined as follows: ... Input::add_eventfilter(Foo::INPUT, [$name="input", $fields=Val, $ev=line]); - - # read the file after all filters have been set - Input::force_update(Foo::INPUT); } The fields that can be set for an event filter are: @@ -156,7 +169,7 @@ an approach similar to this: Input::add_tablefilter(Foo::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=conn_attempts]); - # read the file after all filters have been set + # read the file after all filters have been set (only needed if autostart is set to false) Input::force_update(Foo::INPUT); } diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 445f947106..c6995121bd 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -182,9 +182,6 @@ function read_table(description: Input::StreamDescription, filter: Input::TableF if ( ok ) { ok = add_tablefilter(id, filter); } - if ( ok ) { - ok = force_update(id); - } if ( ok ) { ok = remove_stream(id); } else { diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 27580e0e82..db98cb7a33 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -327,7 +327,6 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { return false; } - Field** logf = new Field*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { logf[i] = fieldsV[i]; @@ -380,6 +379,30 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { } TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); + // check if index fields match tabla description + { + int num = idx->NumFields(); + const type_list* tl = dst->Type()->AsTableType()->IndexTypes(); + + loop_over_list(*tl, j) + { + if ( j >= num ) { + reporter->Error("Table type has more indexes than index definition"); + return false; + } + + if ( !same_type(idx->FieldType(j), (*tl)[j]) ) { + reporter->Error("Table type does not match index type"); + return false; + } + } + + if ( num != j ) { + reporter->Error("Table has less elements than index definition"); + return false; + } + } + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); @@ -571,7 +594,6 @@ bool Manager::RemoveStreamContinuation(const ReaderFrontend* reader) { reporter->Error("Stream not found in RemoveStreamContinuation"); return false; - } bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { @@ -738,7 +760,6 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu idxval = l; } - //reporter->Error("Position: %d, num_fields: %d", position, num_fields); assert ( position == num_fields ); return idxval; @@ -771,8 +792,6 @@ void Manager::SendEntry(const ReaderFrontend* reader, const int id, Value* *vals delete vals[i]; } delete [] vals; - - } int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { @@ -846,17 +865,15 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); } + + bool result; + if ( filter->num_val_fields > 0 ) { // we have values + result = CallPred(filter->pred, 3, ev, predidx, valval); + } else { + // no values + result = CallPred(filter->pred, 2, ev, predidx); + } - val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. - vl.append(ev); - vl.append(predidx); - if ( filter->num_val_fields > 0 ) - vl.append(valval); - - Val* v = filter->pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... @@ -968,14 +985,8 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { Ref(predidx); Ref(val); - val_list vl(3); - vl.append(ev); - vl.append(predidx); - vl.append(val); - Val* v = filter->pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - + bool result = CallPred(filter->pred, 3, ev, predidx, val); + if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... @@ -1038,7 +1049,6 @@ void Manager::Put(const ReaderFrontend* reader, int id, Value* *vals) { } else { assert(false); } - } int Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { @@ -1132,18 +1142,15 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); } + + bool result; + if ( filter->num_val_fields > 0 ) { // we have values + result = CallPred(filter->pred, 3, ev, predidx, valval); + } else { + // no values + result = CallPred(filter->pred, 2, ev, predidx); + } - val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. - vl.append(ev); - vl.append(predidx); - if ( filter->num_val_fields > 0 ) - vl.append(valval); - - - Val* v = filter->pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - if ( result == false ) { // do nothing Unref(idxval); @@ -1154,7 +1161,6 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * } - filter->tab->Assign(idxval, valval); if ( filter->event ) { @@ -1176,13 +1182,9 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * SendEvent(filter->event, 3, ev, predidx, valval); } } - } - - - } else { // no predicates or other stuff @@ -1192,6 +1194,7 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * return filter->num_idx_fields + filter->num_val_fields; } +// Todo:: perhaps throw some kind of clear-event? void Manager::Clear(const ReaderFrontend* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -1207,6 +1210,7 @@ void Manager::Clear(const ReaderFrontend* reader, int id) { filter->tab->RemoveAll(); } +// put interface: delete old entry from table. bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -1235,13 +1239,7 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - val_list vl(3); - vl.append(ev); - vl.append(predidx); - vl.append(val); - Val* v = filter->pred->Call(&vl); - filterresult = v->AsBool(); - Unref(v); + filterresult = CallPred(filter->pred, 3, ev, predidx, val); if ( filterresult == false ) { // keep it. @@ -1285,6 +1283,26 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { return success; } +bool Manager::CallPred(Func* pred_func, const int numvals, ...) +{ + bool result; + val_list vl(numvals); + + va_list lP; + va_start(lP, numvals); + for ( int i = 0; i < numvals; i++ ) + { + vl.append( va_arg(lP, Val*) ); + } + va_end(lP); + + Val* v = pred_func->Call(&vl); + result = v->AsBool(); + Unref(v); + + return(result); +} + bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); @@ -1341,8 +1359,15 @@ void Manager::SendEvent(EventHandlerPtr ev, list events) mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } - +// Convert a bro list value to a bro record value. I / we could think about moving this functionality to val.cc RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) { + assert(position != 0 ); // we need the pointer to point to data; + + if ( request_type->Tag() != TYPE_RECORD ) { + reporter->InternalError("ListValToRecordVal called on non-record-value."); + return 0; + } + RecordVal* rec = new RecordVal(request_type->AsRecordType()); int maxpos = list->Length(); @@ -1364,20 +1389,14 @@ RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, return rec; } - - +// Convert a threading value to a record value RecordVal* Manager::ValueToRecordVal(const Value* const *vals, RecordType *request_type, int* position) { - if ( position == 0 ) { - reporter->InternalError("Need position"); - return 0; - } + assert(position != 0); // we need the pointer to point to data. - /* if ( request_type->Tag() != TYPE_RECORD ) { - reporter->InternalError("I only work with records"); + reporter->InternalError("ValueToRecordVal called on non-record-value."); return 0; - } */ - + } RecordVal* rec = new RecordVal(request_type->AsRecordType()); for ( int i = 0; i < request_type->NumFields(); i++ ) { @@ -1394,11 +1413,12 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals, RecordType *reque } return rec; - } - +// Count the length of the values +// used to create a correct length buffer for hashing later int Manager::GetValueLength(const Value* val) { + assert( val->present ); // presence has to be checked elsewhere int length = 0; switch (val->type) { @@ -1485,19 +1505,20 @@ int Manager::GetValueLength(const Value* val) { } +// Given a threading::value, copy the raw data bytes into *data and return how many bytes were copied. +// Used for hashing the values for lookup in the bro table int Manager::CopyValue(char *data, const int startpos, const Value* val) { + assert( val->present ); // presence has to be checked elsewhere + switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: - //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, startpos); memcpy(data+startpos, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); - //*(data+startpos) = val->val.int_val; return sizeof(val->val.int_val); break; case TYPE_COUNT: case TYPE_COUNTER: - //*(data+startpos) = val->val.uint_val; memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); return sizeof(val->val.uint_val); break; @@ -1516,7 +1537,6 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - //*(data+startpos) = val->val.double_val; memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); return sizeof(val->val.double_val); break; @@ -1598,12 +1618,11 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { return 0; } - reporter->InternalError("internal error"); assert(false); return 0; - } +// Hash num_elements threading values and return the HashKey for them. At least one of the vals has to be ->present. HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { int length = 0; @@ -1633,10 +1652,9 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { assert(position == length); return new HashKey(data, length, key, true); - - } +// convert threading value to Bro value Val* Manager::ValueToVal(const Value* val, BroType* request_type) { if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { @@ -1647,7 +1665,6 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { if ( !val->present ) { return 0; // unset field } - switch ( val->type ) { case TYPE_BOOL: @@ -1760,8 +1777,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { reporter->InternalError("unsupported type for input_read"); } - - reporter->InternalError("Impossible error"); + assert(false); return NULL; } @@ -1778,7 +1794,6 @@ Manager::ReaderInfo* Manager::FindReader(const ReaderFrontend* reader) return 0; } - Manager::ReaderInfo* Manager::FindReader(const EnumVal* id) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) diff --git a/src/input/Manager.h b/src/input/Manager.h index b4fc6cff7f..96ea0e43db 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -177,6 +177,9 @@ private: void SendEvent(EventHandlerPtr ev, const int numvals, ...); void SendEvent(EventHandlerPtr ev, list events); + // Call predicate function and return result + bool CallPred(Func* pred_func, const int numvals, ...); + // get a hashkey for a set of threading::Values HashKey* HashValues(const int num_elements, const threading::Value* const *vals); From b4e6971aab46054e68887836e1a1be40cfd4b9c5 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 14 Mar 2012 14:45:53 -0700 Subject: [PATCH 075/149] Add regular debugging output for interesting operations (stream/filter operations) to input framework (this was way overdue) --- scripts/base/frameworks/input/main.bro | 25 +++++++- src/DebugLogger.cc | 3 +- src/DebugLogger.h | 1 + src/input/Manager.cc | 88 +++++++++++++++++++++++++- 4 files changed, 114 insertions(+), 3 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index c6995121bd..1df8563d94 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,7 +5,7 @@ module Input; export { - redef enum Input::ID += { TABLE_READ }; + redef enum Input::ID += { TABLE_READ, EVENT_READ }; ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; @@ -123,6 +123,8 @@ export { ## filter: the `TableFilter` record describing the filter. global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; + global read_event: function(description: Input::StreamDescription, filter: Input::EventFilter) : bool; + global update_finished: event(id: Input::ID); } @@ -182,6 +184,27 @@ function read_table(description: Input::StreamDescription, filter: Input::TableF if ( ok ) { ok = add_tablefilter(id, filter); } + if ( ok ) { + ok = remove_tablefilter(id, filter$name); + } + if ( ok ) { + ok = remove_stream(id); + } else { + remove_stream(id); + } + + return ok; +} + +function read_event(description: Input::StreamDescription, filter: Input::EventFilter) : bool { + local ok: bool = T; + # since we create and delete it ourselves this should be ok... at least for singlethreaded operation + local id: Input::ID = Input::EVENT_READ; + + ok = create_stream(id, description); + if ( ok ) { + ok = add_eventfilter(id, filter); + } if ( ok ) { ok = remove_stream(id); } else { diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index c41a0552c6..3394486ff2 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -15,7 +15,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, - { "logging", 0, false }, { "threading", 0, false } + { "logging", 0, false }, {"input", 0, false }, + { "threading", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 71e21bfa26..ca422072c5 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -24,6 +24,7 @@ enum DebugStream { DBG_DPD, // Dynamic application detection framework DBG_TM, // Time-machine packet input via Brocolli DBG_LOGGING, // Logging streams + DBG_INPUT, // Input streams DBG_THREADING, // Threading system NUM_DBGS // Has to be last diff --git a/src/input/Manager.cc b/src/input/Manager.cc index db98cb7a33..aa50453bdf 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -205,12 +205,24 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) // create a new input reader object to be used at whomevers leisure lateron. ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) { + { + ReaderInfo *i = FindReader(id); + if ( i != 0 ) { + ODesc desc; + id->Describe(&desc); + reporter->Error("Trying create already existing input stream %s", desc.Description()); + return 0; + } + } + ReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) { - reporter->Error("Streamdescription argument not of right type"); + ODesc desc; + id->Describe(&desc); + reporter->Error("Streamdescription argument not of right type for new input stream %s", desc.Description()); return 0; } @@ -239,6 +251,13 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) reader_obj->Init(source, mode->InternalInt(), do_autostart); +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", + desc.Description()); +#endif + return reader_obj; } @@ -503,6 +522,13 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { i->filters[filterid] = filter; i->reader->AddFilter( filterid, fieldsV.size(), fields ); +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Successfully created new table filter %s for stream", + filter->name.c_str(), desc.Description()); +#endif + return true; } @@ -574,6 +600,13 @@ bool Manager::RemoveStream(const EnumVal* id) { i->reader->Finish(); +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Successfully queued removal of stream %s", + desc.Description()); +#endif + return true; } @@ -586,6 +619,12 @@ bool Manager::RemoveStreamContinuation(const ReaderFrontend* reader) { if ( (*s)->reader && (*s)->reader == reader ) { i = *s; +#ifdef DEBUG + ODesc desc; + i->id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Successfully executed removal of stream %s", + desc.Description()); +#endif delete(i); readers.erase(s); return true; @@ -651,6 +690,13 @@ bool Manager::ForceUpdate(const EnumVal* id) i->reader->Update(); +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Forcing update of stream %s", + desc.Description()); +#endif + return true; // update is async :( } @@ -685,6 +731,13 @@ bool Manager::RemoveTableFilter(EnumVal* id, const string &name) { i->reader->RemoveFilter(filterId); +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Queued removal of tablefilter %s for stream %s", + name.c_str(), desc.Description()); +#endif + return true; } @@ -701,6 +754,13 @@ bool Manager::RemoveFilterContinuation(const ReaderFrontend* reader, const int f return false; } +#ifdef DEBUG + ODesc desc; + i->id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Executed removal of (table|event)-filter %s for stream %s", + (*it).second->name.c_str(), desc.Description()); +#endif + delete (*it).second; i->filters.erase(it); @@ -736,6 +796,13 @@ bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { } i->reader->RemoveFilter(filterId); + +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Queued removal of eventfilter %s for stream %s", + name.c_str(), desc.Description()); +#endif return true; } @@ -948,6 +1015,13 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { assert(i->HasFilter(id)); +#ifdef DEBUG + ODesc desc; + i->id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Got EndCurrentSend for filter %d and stream %s", + id, desc.Description()); +#endif + if ( i->filters[id]->filter_type == EVENT_FILTER ) { // nothing to do.. return; @@ -1018,6 +1092,11 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { filter->lastDict = filter->currDict; filter->currDict = new PDict(InputHash); +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "EndCurrentSend complete for filter %d and stream %s, queueing update_finished event", + id, desc.Description()); +#endif + // Send event that the current update is indeed finished. EventHandler* handler = event_registry->Lookup("Input::update_finished"); if ( handler == 0 ) { @@ -1202,6 +1281,13 @@ void Manager::Clear(const ReaderFrontend* reader, int id) { return; } +#ifdef DEBUG + ODesc desc; + i->id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Got Clear for filter %d and stream %s", + id, desc.Description()); +#endif + assert(i->HasFilter(id)); assert(i->filters[id]->filter_type == TABLE_FILTER); From 57ffe1be777f395de2cb94c95c02b9f234109744 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 15 Mar 2012 18:41:51 -0700 Subject: [PATCH 076/149] completely change interface again. compiles, not really tested. basic test works 70% of the time, coredumps in the other 30 - but was not easy to debug on a first glance (most interestingly the crash happens in the logging framework - I wonder how that works). Other tests are not adjusted to the new interface yet. --- scripts/base/frameworks/input/main.bro | 190 ++----- src/input.bif | 39 +- src/input/Manager.cc | 511 ++++++------------ src/input/Manager.h | 96 +--- src/input/ReaderBackend.cc | 115 +--- src/input/ReaderBackend.h | 80 +-- src/input/ReaderFrontend.cc | 83 +-- src/input/ReaderFrontend.h | 27 +- src/input/readers/Ascii.cc | 182 +++---- src/input/readers/Ascii.h | 24 +- src/input/readers/Raw.cc | 89 +-- src/input/readers/Raw.h | 23 +- src/types.bif | 4 - .../scripts/base/frameworks/input/basic.bro | 12 +- 14 files changed, 403 insertions(+), 1072 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 1df8563d94..4f7f9983d1 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,15 +5,15 @@ module Input; export { - redef enum Input::ID += { TABLE_READ, EVENT_READ }; - ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; const default_mode = MANUAL &redef; - ## Stream decription type used for the `create_stream` method - type StreamDescription: record { + ## TableFilter description type used for the `table` method. + type TableDescription: record { + ## Common definitions for tables and events + ## String that allows the reader to find the source. ## For `READER_ASCII`, this is the filename. source: string; @@ -26,13 +26,12 @@ export { ## Automatically start the input stream after the first filter has been added autostart: bool &default=T; - }; - ## TableFilter description type used for the `add_tablefilter` method. - type TableFilter: record { ## Descriptive name. Used to remove a filter at a later time name: string; + ## Special definitions for tables + ## Table which will contain the data read by the input framework destination: any; ## Record that defines the values used as the index of the table @@ -55,11 +54,28 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; }; - ## EventFilter description type used for the `add_eventfilter` method. - type EventFilter: record { - ## Descriptive name. Used to remove a filter at a later time - name: string; + ## EventFilter description type used for the `event` method. + type EventDescription: record { + ## Common definitions for tables and events + + ## String that allows the reader to find the source. + ## For `READER_ASCII`, this is the filename. + source: string; + + ## Reader to use for this steam + reader: Reader &default=default_reader; + ## Read mode to use for this stream + mode: Mode &default=default_mode; + + ## Automatically start the input stream after the first filter has been added + autostart: bool &default=T; + + ## Descriptive name. Used to remove a filter at a later time + name: string; + + ## Special definitions for events + ## Record describing the fields to be retrieved from the source input. fields: any; ## If want_record if false (default), the event receives each value in fields as a seperate argument. @@ -72,61 +88,29 @@ export { }; - #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - - ## Create a new input stream from a given source. Returns true on success. + ## Create a new table input from a given source. Returns true on success. ## - ## id: `Input::ID` enum value identifying this stream - ## description: `StreamDescription` record describing the source. - global create_stream: function(id: Input::ID, description: Input::StreamDescription) : bool; - - ## Remove a current input stream. Returns true on success. + ## description: `TableDescription` record describing the source. + global add_table: function(description: Input::TableDescription) : bool; + + ## Create a new event input from a given source. Returns true on success. ## - ## id: `Input::ID` enum value identifying the stream to be removed - global remove_stream: function(id: Input::ID) : bool; + ## description: `TableDescription` record describing the source. + global add_event: function(description: Input::EventDescription) : bool; + + ## Remove a input stream. Returns true on success and false if the named stream was not found. + ## + ## id: string value identifying the stream to be removed + global remove: function(id: string) : bool; ## Forces the current input to be checked for changes. + ## Returns true on success and false if the named stream was not found ## - ## id: `Input::ID` enum value identifying the stream - global force_update: function(id: Input::ID) : bool; - - ## Adds a table filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## filter: the `TableFilter` record describing the filter. - global add_tablefilter: function(id: Input::ID, filter: Input::TableFilter) : bool; - - ## Removes a named table filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## name: the name of the filter to be removed. - global remove_tablefilter: function(id: Input::ID, name: string) : bool; - - ## Adds an event filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## filter: the `EventFilter` record describing the filter. - global add_eventfilter: function(id: Input::ID, filter: Input::EventFilter) : bool; - - ## Removes a named event filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## name: the name of the filter to be removed. - global remove_eventfilter: function(id: Input::ID, name: string) : bool; - #global get_filter: function(id: ID, name: string) : Filter; - - ## Convenience function for reading a specific input source exactly once using - ## exactly one tablefilter - ## - ## id: `Input::ID` enum value identifying the stream - ## description: `StreamDescription` record describing the source. - ## filter: the `TableFilter` record describing the filter. - global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; - - global read_event: function(description: Input::StreamDescription, filter: Input::EventFilter) : bool; - - global update_finished: event(id: Input::ID); + ## id: string value identifying the stream + global force_update: function(id: string) : bool; + ## Event that is called, when the update of a specific source is finished + global update_finished: event(id: string); } @load base/input.bif @@ -134,90 +118,26 @@ export { module Input; -#global filters: table[ID, string] of Filter; +#global streams: table[string] of Filter; +# ^ change to set containing the names -function create_stream(id: Input::ID, description: Input::StreamDescription) : bool +function add_table(description: Input::TableDescription) : bool { - return __create_stream(id, description); + return __create_table_stream(description); } -function remove_stream(id: Input::ID) : bool +function add_event(description: Input::EventDescription) : bool + { + return __create_event_stream(description); + } + +function remove(id: string) : bool { return __remove_stream(id); } -function force_update(id: Input::ID) : bool +function force_update(id: string) : bool { return __force_update(id); } -function add_tablefilter(id: Input::ID, filter: Input::TableFilter) : bool - { -# filters[id, filter$name] = filter; - return __add_tablefilter(id, filter); - } - -function remove_tablefilter(id: Input::ID, name: string) : bool - { -# delete filters[id, name]; - return __remove_tablefilter(id, name); - } - -function add_eventfilter(id: Input::ID, filter: Input::EventFilter) : bool - { -# filters[id, filter$name] = filter; - return __add_eventfilter(id, filter); - } - -function remove_eventfilter(id: Input::ID, name: string) : bool - { -# delete filters[id, name]; - return __remove_eventfilter(id, name); - } - -function read_table(description: Input::StreamDescription, filter: Input::TableFilter) : bool { - local ok: bool = T; - # since we create and delete it ourselves this should be ok... at least for singlethreaded operation - local id: Input::ID = Input::TABLE_READ; - - ok = create_stream(id, description); - if ( ok ) { - ok = add_tablefilter(id, filter); - } - if ( ok ) { - ok = remove_tablefilter(id, filter$name); - } - if ( ok ) { - ok = remove_stream(id); - } else { - remove_stream(id); - } - - return ok; -} - -function read_event(description: Input::StreamDescription, filter: Input::EventFilter) : bool { - local ok: bool = T; - # since we create and delete it ourselves this should be ok... at least for singlethreaded operation - local id: Input::ID = Input::EVENT_READ; - - ok = create_stream(id, description); - if ( ok ) { - ok = add_eventfilter(id, filter); - } - if ( ok ) { - ok = remove_stream(id); - } else { - remove_stream(id); - } - - return ok; -} - -#function get_filter(id: ID, name: string) : Filter -# { -# if ( [id, name] in filters ) -# return filters[id, name]; -# -# return no_filter; -# } diff --git a/src/input.bif b/src/input.bif index 5418b7bbd4..1157b7b62b 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,52 +7,33 @@ module Input; #include "NetVar.h" %%} -type StreamDescription: record; -type TableFilter: record; -type EventFilter: record; +type TableDescription: record; +type EventDescription: record; -function Input::__create_stream%(id: Input::ID, description: Input::StreamDescription%) : bool +function Input::__create_table_stream%(description: Input::TableDescription%) : bool %{ - input::ReaderFrontend *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); - return new Val( the_reader != 0, TYPE_BOOL ); - %} - -function Input::__remove_stream%(id: Input::ID%) : bool - %{ - bool res = input_mgr->RemoveStream(id->AsEnumVal()); + bool res = input_mgr->CreateTableStream(description->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__force_update%(id: Input::ID%) : bool +function Input::__create_event_stream%(description: Input::EventDescription%) : bool %{ - bool res = input_mgr->ForceUpdate(id->AsEnumVal()); + bool res = input_mgr->CreateEventStream(description->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_tablefilter%(id: Input::ID, filter: Input::TableFilter%) : bool +function Input::__remove_stream%(id: string%) : bool %{ - bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); + bool res = input_mgr->RemoveStream(id->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_tablefilter%(id: Input::ID, name: string%) : bool +function Input::__force_update%(id: string%) : bool %{ - bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL); - %} - -function Input::__add_eventfilter%(id: Log::ID, filter: Input::EventFilter%) : bool - %{ - bool res = input_mgr->AddEventFilter(id->AsEnumVal(), filter->AsRecordVal()); + bool res = input_mgr->ForceUpdate(id->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_eventfilter%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->RemoveEventFilter(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL); - %} - # Options for Ascii Reader module InputAscii; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index aa50453bdf..44b5bf44db 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -31,14 +31,25 @@ declare(PDict, InputHash); class Manager::Filter { public: - EnumVal* id; string name; + string source; + + int mode; FilterType filter_type; // to distinguish between event and table filters + EnumVal* type; + ReaderFrontend* reader; + virtual ~Filter(); }; +Manager::Filter::~Filter() { + Unref(type); + + delete(reader); +} + class Manager::TableFilter: public Manager::Filter { public: @@ -85,10 +96,6 @@ Manager::EventFilter::EventFilter() { filter_type = EVENT_FILTER; } -Manager::Filter::~Filter() { - Unref(id); -} - Manager::TableFilter::~TableFilter() { Unref(tab); Unref(itype); @@ -99,41 +106,6 @@ Manager::TableFilter::~TableFilter() { delete lastDict; } -struct Manager::ReaderInfo { - EnumVal* id; - EnumVal* type; - ReaderFrontend* reader; - - map filters; // filters that can prevent our actions - - bool HasFilter(int id); - - ~ReaderInfo(); - }; - -Manager::ReaderInfo::~ReaderInfo() { - map::iterator it = filters.begin(); - - while ( it != filters.end() ) { - delete (*it).second; - ++it; - } - - Unref(type); - Unref(id); - - delete(reader); -} - -bool Manager::ReaderInfo::HasFilter(int id) { - map::iterator it = filters.find(id); - if ( it == filters.end() ) { - return false; - } - return true; -} - - struct ReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -203,37 +175,34 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) } // create a new input reader object to be used at whomevers leisure lateron. -ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) +bool Manager::CreateStream(Filter* info, RecordVal* description) { - { - ReaderInfo *i = FindReader(id); - if ( i != 0 ) { - ODesc desc; - id->Describe(&desc); - reporter->Error("Trying create already existing input stream %s", desc.Description()); - return 0; - } - } - ReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) + if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) || same_type(rtype, BifType::Record::Input::EventDescription, 0) ) ) { - ODesc desc; - id->Describe(&desc); - reporter->Error("Streamdescription argument not of right type for new input stream %s", desc.Description()); - return 0; + reporter->Error("Streamdescription argument not of right type for new input stream"); + return false; + } + + Val* name_val = description->LookupWithDefault(rtype->FieldOffset("name")); + string name = name_val->AsString()->CheckString(); + Unref(name_val); + + { + Filter *i = FindFilter(name); + if ( i != 0 ) { + reporter->Error("Trying create already existing input stream %s", name.c_str()); + return false; + } } EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); - EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); Val *autostart = description->LookupWithDefault(rtype->FieldOffset("autostart")); - bool do_autostart = ( autostart->InternalInt() == 1 ); - Unref(autostart); // Ref'd by LookupWithDefault - - ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); - assert(reader_obj); + + ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); + assert(reader_obj); // get the source... Val* sourceval = description->LookupWithDefault(rtype->FieldOffset("source")); @@ -241,42 +210,45 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) const BroString* bsource = sourceval->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); Unref(sourceval); + + EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); + info->mode = mode->InternalInt(); + Unref(mode); - ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault - info->id = id->Ref()->AsEnumVal(); + info->name = name; + info->source = source; - readers.push_back(info); - - reader_obj->Init(source, mode->InternalInt(), do_autostart); #ifdef DEBUG - ODesc desc; - id->Describe(&desc); DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", - desc.Description()); + name.c_str()); #endif - return reader_obj; + return true; } -bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Stream not found"); - return false; - } +bool Manager::CreateEventStream(RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::EventFilter, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::EventDescription, 0) ) { reporter->Error("filter argument not of right type"); return false; } + + EventFilter* filter = new EventFilter(); + { + bool res = CreateStream(filter, fval); + if ( res == false ) { + delete filter; + return false; + } + } + - Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); RecordType *fields = fval->LookupWithDefault(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); @@ -352,42 +324,36 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { } Unref(fields); // ref'd by lookupwithdefault - EventFilter* filter = new EventFilter(); - filter->name = name->AsString()->CheckString(); - Unref(name); // ref'd by lookupwithdefault - filter->id = id->Ref()->AsEnumVal(); filter->num_fields = fieldsV.size(); filter->fields = fields->Ref()->AsRecordType(); filter->event = event_registry->Lookup(event->GetID()->Name()); filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault - int filterid = 0; - if ( i->filters.size() > 0 ) { - filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> - } - i->filters[filterid] = filter; - i->reader->AddFilter( filterid, fieldsV.size(), logf ); + assert(filter->reader); + filter->reader->Init(filter->source, filter->mode, filter->num_fields, logf ); + readers[filter->reader] = filter; return true; } -bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Stream not found"); - return false; - } - +bool Manager::CreateTableStream(RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::TableFilter, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::TableDescription, 0) ) { reporter->Error("filter argument not of right type"); return false; } + TableFilter* filter = new TableFilter(); + { + bool res = CreateStream(filter, fval); + if ( res == false ) { + delete filter; + return false; + } + } - Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); @@ -493,9 +459,6 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } - TableFilter* filter = new TableFilter(); - filter->name = name->AsString()->CheckString(); - filter->id = id->Ref()->AsEnumVal(); filter->pred = pred ? pred->AsFunc() : 0; filter->num_idx_fields = idxfields; filter->num_val_fields = valfields; @@ -508,25 +471,22 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault - Unref(name); Unref(pred); if ( valfields > 1 ) { assert(filter->want_record); } + + + assert(filter->reader); + filter->reader->Init(filter->source, filter->mode, fieldsV.size(), fields ); + + readers[filter->reader] = filter; - int filterid = 0; - if ( i->filters.size() > 0 ) { - filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> - } - i->filters[filterid] = filter; - i->reader->AddFilter( filterid, fieldsV.size(), fields ); #ifdef DEBUG - ODesc desc; - id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Successfully created new table filter %s for stream", - filter->name.c_str(), desc.Description()); + DBG_LOG(DBG_INPUT, "Successfully created table stream %s", + filter->name.c_str()); #endif return true; @@ -583,16 +543,8 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) } -bool Manager::RemoveStream(const EnumVal* id) { - ReaderInfo *i = 0; - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) - { - if ( (*s)->id == id ) - { - i = (*s); - break; - } - } +bool Manager::RemoveStream(const string &name) { + Filter *i = FindFilter(name); if ( i == 0 ) { return false; // not found @@ -601,38 +553,29 @@ bool Manager::RemoveStream(const EnumVal* id) { i->reader->Finish(); #ifdef DEBUG - ODesc desc; - id->Describe(&desc); DBG_LOG(DBG_INPUT, "Successfully queued removal of stream %s", - desc.Description()); + name.c_str()); #endif return true; } -bool Manager::RemoveStreamContinuation(const ReaderFrontend* reader) { - ReaderInfo *i = 0; +bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) { + Filter *i = FindFilter(reader); - - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) - { - if ( (*s)->reader && (*s)->reader == reader ) - { - i = *s; -#ifdef DEBUG - ODesc desc; - i->id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Successfully executed removal of stream %s", - desc.Description()); -#endif - delete(i); - readers.erase(s); - return true; - } + if ( i == 0 ) { + reporter->Error("Stream not found in RemoveStreamContinuation"); + return false; } - - reporter->Error("Stream not found in RemoveStreamContinuation"); - return false; + + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Successfully executed removal of stream %s", + i->name.c_str()); +#endif + readers.erase(reader); + delete(i); + return true; } bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { @@ -680,132 +623,24 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co return true; } -bool Manager::ForceUpdate(const EnumVal* id) +bool Manager::ForceUpdate(const string &name) { - ReaderInfo *i = FindReader(id); + Filter *i = FindFilter(name); if ( i == 0 ) { - reporter->Error("Reader not found"); + reporter->Error("Stream %s not found", name.c_str()); return false; } i->reader->Update(); #ifdef DEBUG - ODesc desc; - id->Describe(&desc); DBG_LOG(DBG_INPUT, "Forcing update of stream %s", - desc.Description()); + name.c_str()); #endif return true; // update is async :( } -bool Manager::RemoveTableFilter(EnumVal* id, const string &name) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - bool found = false; - int filterId; - - for ( map::iterator it = i->filters.begin(); it != i->filters.end(); ++it ) { - if ( (*it).second->name == name ) { - found = true; - filterId = (*it).first; - - if ( (*it).second->filter_type != TABLE_FILTER ) { - reporter->Error("Trying to remove filter %s of wrong type", name.c_str()); - return false; - } - - break; - } - } - - if ( !found ) { - reporter->Error("Trying to remove nonexisting filter %s", name.c_str()); - return false; - } - - i->reader->RemoveFilter(filterId); - -#ifdef DEBUG - ODesc desc; - id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Queued removal of tablefilter %s for stream %s", - name.c_str(), desc.Description()); -#endif - - return true; -} - -bool Manager::RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId) { - ReaderInfo *i = FindReader(reader); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - map::iterator it = i->filters.find(filterId); - if ( it == i->filters.end() ) { - reporter->Error("Got RemoveFilterContinuation where filter nonexistant for %d", filterId); - return false; - } - -#ifdef DEBUG - ODesc desc; - i->id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Executed removal of (table|event)-filter %s for stream %s", - (*it).second->name.c_str(), desc.Description()); -#endif - - delete (*it).second; - i->filters.erase(it); - - return true; -} - -bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - bool found = false; - int filterId; - for ( map::iterator it = i->filters.begin(); it != i->filters.end(); ++it ) { - if ( (*it).second->name == name ) { - found = true; - filterId = (*it).first; - - if ( (*it).second->filter_type != EVENT_FILTER ) { - reporter->Error("Trying to remove filter %s of wrong type", name.c_str()); - return false; - } - - break; - } - } - - if ( !found ) { - reporter->Error("Trying to remove nonexisting filter %s", name.c_str()); - return false; - } - - i->reader->RemoveFilter(filterId); - -#ifdef DEBUG - ODesc desc; - id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Queued removal of eventfilter %s for stream %s", - name.c_str(), desc.Description()); -#endif - return true; -} - Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) { Val* idxval; int position = 0; @@ -833,24 +668,19 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu } -void Manager::SendEntry(const ReaderFrontend* reader, const int id, Value* *vals) { - ReaderInfo *i = FindReader(reader); +void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); - return; - } - - if ( !i->HasFilter(id) ) { - reporter->InternalError("Unknown filter"); + reporter->InternalError("Unknown reader in SendEntry"); return; } int readFields; - if ( i->filters[id]->filter_type == TABLE_FILTER ) { - readFields = SendEntryTable(reader, id, vals); - } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + if ( i->filter_type == TABLE_FILTER ) { + readFields = SendEntryTable(i, vals); + } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - readFields = SendEventFilterEvent(reader, type, id, vals); + readFields = SendEventFilterEvent(i, type, vals); } else { assert(false); } @@ -861,16 +691,13 @@ void Manager::SendEntry(const ReaderFrontend* reader, const int id, Value* *vals delete [] vals; } -int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { - ReaderInfo *i = FindReader(reader); - +int Manager::SendEntryTable(Filter* i, const Value* const *vals) { bool updated = false; assert(i); - assert(i->HasFilter(id)); - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; HashKey* idxhash = HashValues(filter->num_idx_fields, vals); @@ -1006,29 +833,25 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } -void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { - ReaderInfo *i = FindReader(reader); +void Manager::EndCurrentSend(ReaderFrontend* reader) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in EndCurrentSend"); return; } - assert(i->HasFilter(id)); - #ifdef DEBUG - ODesc desc; - i->id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Got EndCurrentSend for filter %d and stream %s", - id, desc.Description()); + DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", + i->name.c_str()); #endif - if ( i->filters[id]->filter_type == EVENT_FILTER ) { + if ( i->filter_type == EVENT_FILTER ) { // nothing to do.. return; } - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = filter->lastDict->InitForIteration(); @@ -1093,8 +916,8 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { filter->currDict = new PDict(InputHash); #ifdef DEBUG - DBG_LOG(DBG_INPUT, "EndCurrentSend complete for filter %d and stream %s, queueing update_finished event", - id, desc.Description()); + DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s, queueing update_finished event", + i->name.c_str()); #endif // Send event that the current update is indeed finished. @@ -1104,42 +927,40 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { } - Ref(i->id); - SendEvent(handler, 1, i->id); + SendEvent(handler, 1, new BroString(i->name)); } -void Manager::Put(const ReaderFrontend* reader, int id, Value* *vals) { - ReaderInfo *i = FindReader(reader); +void Manager::Put(ReaderFrontend* reader, Value* *vals) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in Put"); return; } - if ( !i->HasFilter(id) ) { - reporter->InternalError("Unknown filter"); - return; - } - - if ( i->filters[id]->filter_type == TABLE_FILTER ) { - PutTable(reader, id, vals); - } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + int readFields; + if ( i->filter_type == TABLE_FILTER ) { + readFields = PutTable(i, vals); + } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - SendEventFilterEvent(reader, type, id, vals); + readFields = SendEventFilterEvent(i, type, vals); } else { assert(false); } + + for ( int i = 0; i < readFields; i++ ) { + delete vals[i]; + } + delete [] vals; + } -int Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { - ReaderInfo *i = FindReader(reader); - +int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const *vals) { bool updated = false; assert(i); - assert(i->HasFilter(id)); - assert(i->filters[id]->filter_type == EVENT_FILTER); - EventFilter* filter = (EventFilter*) i->filters[id]; + assert(i->filter_type == EVENT_FILTER); + EventFilter* filter = (EventFilter*) i; Val *val; list out_vals; @@ -1170,14 +991,11 @@ int Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, i } -int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const *vals) { - ReaderInfo *i = FindReader(reader); - +int Manager::PutTable(Filter* i, const Value* const *vals) { assert(i); - assert(i->HasFilter(id)); - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; @@ -1274,43 +1092,37 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * } // Todo:: perhaps throw some kind of clear-event? -void Manager::Clear(const ReaderFrontend* reader, int id) { - ReaderInfo *i = FindReader(reader); +void Manager::Clear(ReaderFrontend* reader) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in Clear"); return; } #ifdef DEBUG - ODesc desc; - i->id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Got Clear for filter %d and stream %s", - id, desc.Description()); + DBG_LOG(DBG_INPUT, "Got Clear for stream %s", + i->name.c_str()); #endif - assert(i->HasFilter(id)); - - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; filter->tab->RemoveAll(); } // put interface: delete old entry from table. -bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { - ReaderInfo *i = FindReader(reader); +bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in Delete"); return false; } - assert(i->HasFilter(id)); - bool success = false; int readVals = 0; - if ( i->filters[id]->filter_type == TABLE_FILTER ) { - TableFilter* filter = (TableFilter*) i->filters[id]; + if ( i->filter_type == TABLE_FILTER ) { + TableFilter* filter = (TableFilter*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); assert(idxval != 0); readVals = filter->num_idx_fields + filter->num_val_fields; @@ -1352,9 +1164,9 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { reporter->Error("Internal error while deleting values from input table"); } } - } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - readVals = SendEventFilterEvent(reader, type, id, vals); + readVals = SendEventFilterEvent(i, type, vals); success = true; } else { assert(false); @@ -1867,29 +1679,24 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { return NULL; } -Manager::ReaderInfo* Manager::FindReader(const ReaderFrontend* reader) +Manager::Filter* Manager::FindFilter(const string &name) { - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->reader && (*s)->reader == reader ) + if ( (*s).second->name == name ) { - return *s; + return (*s).second; } } return 0; } -Manager::ReaderInfo* Manager::FindReader(const EnumVal* id) - { - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) - { - if ( (*s)->id && (*s)->id->AsEnum() == id->AsEnum() ) - { - return *s; - } - } - - return 0; +Manager::Filter* Manager::FindFilter(ReaderFrontend* reader) +{ + map::iterator s = readers.find(reader); + if ( s != readers.end() ) { + return s->second; } - + return 0; +} diff --git a/src/input/Manager.h b/src/input/Manager.h index 96ea0e43db..71169c4bc2 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -11,7 +11,7 @@ #include "../EventHandler.h" #include "../RemoteSerializer.h" -#include +#include namespace input { @@ -35,6 +35,9 @@ public: /** * Creates a new input stream. + * Add a filter to an input source, which will write the data from the data source into + * a Bro table. + * Add a filter to an input source, which sends events for read input data. * * @param id The enum value corresponding the input stream. * @@ -43,7 +46,9 @@ public: * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ - ReaderFrontend* CreateStream(EnumVal* id, RecordVal* description); + bool CreateTableStream(RecordVal* description); + bool CreateEventStream(RecordVal* description); + /** * Force update on a input stream. @@ -57,7 +62,7 @@ public: * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ - bool ForceUpdate(const EnumVal* id); + bool ForceUpdate(const string &id); /** * Deletes an existing input stream @@ -67,53 +72,8 @@ public: * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ - bool RemoveStream(const EnumVal* id); + bool RemoveStream(const string &id); - /** - * Add a filter to an input source, which will write the data from the data source into - * a Bro table. - * - * @param id The enum value corresponding the input stream. - * - * @param description A record of script type \c Input:TableFilter. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool AddTableFilter(EnumVal *id, RecordVal* filter); - - /** - * Removes a tablefilter from the log stream - * - * @param id The enum value corresponding the input stream. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool RemoveTableFilter(EnumVal* id, const string &name); - - /** - * Add a filter to an input source, which sends events for read input data. - * - * @param id The enum value corresponding the input stream. - * - * @param description A record of script type \c Input:EventFilter. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool AddEventFilter(EnumVal *id, RecordVal* filter); - - /** - * Removes a eventfilter from the log stream - * - * @param id The enum value corresponding the input stream. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool RemoveEventFilter(EnumVal* id, const string &name); - protected: friend class ReaderFrontend; friend class PutMessage; @@ -122,19 +82,18 @@ protected: friend class SendEventMessage; friend class SendEntryMessage; friend class EndCurrentSendMessage; - friend class FilterRemovedMessage; friend class ReaderFinishedMessage; // For readers to write to input stream in direct mode (reporting new/deleted values directly) // Functions take ownership of threading::Value fields - void Put(const ReaderFrontend* reader, int id, threading::Value* *vals); - void Clear(const ReaderFrontend* reader, int id); - bool Delete(const ReaderFrontend* reader, int id, threading::Value* *vals); + void Put(ReaderFrontend* reader, threading::Value* *vals); + void Clear(ReaderFrontend* reader); + bool Delete(ReaderFrontend* reader, threading::Value* *vals); // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) // Functions take ownership of threading::Value fields - void SendEntry(const ReaderFrontend* reader, const int id, threading::Value* *vals); - void EndCurrentSend(const ReaderFrontend* reader, const int id); + void SendEntry(ReaderFrontend* reader, threading::Value* *vals); + void EndCurrentSend(ReaderFrontend* reader); // Allows readers to directly send Bro events. // The num_vals and vals must be the same the named event expects. @@ -150,20 +109,23 @@ protected: // Used to prevent race conditions where data for a specific filter is still in the queue when the // RemoveFilter directive is executed by the main thread. // This makes sure all data that has ben queued for a filter is still received. - bool RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId); - bool RemoveStreamContinuation(const ReaderFrontend* reader); + bool RemoveStreamContinuation(ReaderFrontend* reader); private: - struct ReaderInfo; + class Filter; + class TableFilter; + class EventFilter; + + bool CreateStream(Filter*, RecordVal* description); // SendEntry implementation for Tablefilter - int SendEntryTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + int SendEntryTable(Filter* i, const threading::Value* const *vals); // Put implementation for Tablefilter - int PutTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + int PutTable(Filter* i, const threading::Value* const *vals); // SendEntry and Put implementation for Eventfilter - int SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const threading::Value* const *vals); + int SendEventFilterEvent(Filter* i, EnumVal* type, const threading::Value* const *vals); // Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types // from the log framework @@ -200,16 +162,12 @@ private: // Converts a Bro ListVal to a RecordVal given the record type RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); - ReaderInfo* FindReader(const ReaderFrontend* reader); - ReaderInfo* FindReader(const EnumVal* id); - - vector readers; - - class Filter; - class TableFilter; - class EventFilter; + Filter* FindFilter(const string &name); + Filter* FindFilter(ReaderFrontend* reader); enum FilterType { TABLE_FILTER, EVENT_FILTER }; + + map readers; }; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 5af02b1acc..b33e19d297 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -11,48 +11,44 @@ namespace input { class PutMessage : public threading::OutputMessage { public: - PutMessage(ReaderFrontend* reader, int id, Value* *val) + PutMessage(ReaderFrontend* reader, Value* *val) : threading::OutputMessage("Put", reader), - id(id), val(val) {} + val(val) {} virtual bool Process() { - input_mgr->Put(Object(), id, val); + input_mgr->Put(Object(), val); return true; } private: - int id; Value* *val; }; class DeleteMessage : public threading::OutputMessage { public: - DeleteMessage(ReaderFrontend* reader, int id, Value* *val) + DeleteMessage(ReaderFrontend* reader, Value* *val) : threading::OutputMessage("Delete", reader), - id(id), val(val) {} + val(val) {} virtual bool Process() { - return input_mgr->Delete(Object(), id, val); + return input_mgr->Delete(Object(), val); } private: - int id; Value* *val; }; class ClearMessage : public threading::OutputMessage { public: - ClearMessage(ReaderFrontend* reader, int id) - : threading::OutputMessage("Clear", reader), - id(id) {} + ClearMessage(ReaderFrontend* reader) + : threading::OutputMessage("Clear", reader) {} virtual bool Process() { - input_mgr->Clear(Object(), id); + input_mgr->Clear(Object()); return true; } private: - int id; }; class SendEventMessage : public threading::OutputMessage { @@ -73,47 +69,30 @@ private: class SendEntryMessage : public threading::OutputMessage { public: - SendEntryMessage(ReaderFrontend* reader, const int id, Value* *val) + SendEntryMessage(ReaderFrontend* reader, Value* *val) : threading::OutputMessage("SendEntry", reader), - id(id), val(val) { } + val(val) { } virtual bool Process() { - input_mgr->SendEntry(Object(), id, val); + input_mgr->SendEntry(Object(), val); return true; } private: - const int id; Value* *val; }; class EndCurrentSendMessage : public threading::OutputMessage { public: - EndCurrentSendMessage(ReaderFrontend* reader, const int id) - : threading::OutputMessage("EndCurrentSend", reader), - id(id) {} + EndCurrentSendMessage(ReaderFrontend* reader) + : threading::OutputMessage("EndCurrentSend", reader) {} virtual bool Process() { - input_mgr->EndCurrentSend(Object(), id); + input_mgr->EndCurrentSend(Object()); return true; } private: - const int id; -}; - -class FilterRemovedMessage : public threading::OutputMessage { -public: - FilterRemovedMessage(ReaderFrontend* reader, const int id) - : threading::OutputMessage("FilterRemoved", reader), - id(id) {} - - virtual bool Process() { - return input_mgr->RemoveFilterContinuation(Object(), id); - } - -private: - const int id; }; class ReaderFinishedMessage : public threading::OutputMessage { @@ -155,19 +134,19 @@ ReaderBackend::~ReaderBackend() } -void ReaderBackend::Put(int id, Value* *val) +void ReaderBackend::Put(Value* *val) { - SendOut(new PutMessage(frontend, id, val)); + SendOut(new PutMessage(frontend, val)); } -void ReaderBackend::Delete(int id, Value* *val) +void ReaderBackend::Delete(Value* *val) { - SendOut(new DeleteMessage(frontend, id, val)); + SendOut(new DeleteMessage(frontend, val)); } -void ReaderBackend::Clear(int id) +void ReaderBackend::Clear() { - SendOut(new ClearMessage(frontend, id)); + SendOut(new ClearMessage(frontend)); } void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) @@ -175,70 +154,32 @@ void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *va SendOut(new SendEventMessage(frontend, name, num_vals, vals)); } -void ReaderBackend::EndCurrentSend(int id) +void ReaderBackend::EndCurrentSend() { - SendOut(new EndCurrentSendMessage(frontend, id)); + SendOut(new EndCurrentSendMessage(frontend)); } -void ReaderBackend::SendEntry(int id, Value* *vals) +void ReaderBackend::SendEntry(Value* *vals) { - SendOut(new SendEntryMessage(frontend, id, vals)); + SendOut(new SendEntryMessage(frontend, vals)); } -bool ReaderBackend::Init(string arg_source, int mode, bool arg_autostart) +bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* arg_fields) { source = arg_source; - autostart = arg_autostart; SetName("InputReader/"+source); // disable if DoInit returns error. - disabled = !DoInit(arg_source, mode); + int success = DoInit(arg_source, mode, arg_num_fields, arg_fields); - if ( disabled ) { + if ( !success ) { Error("Init failed"); DisableFrontend(); } - return !disabled; -} - -bool ReaderBackend::StartReading() { - if ( disabled ) - return false; - - int success = DoStartReading(); - - if ( success == false ) { - DisableFrontend(); - } - return success; } -bool ReaderBackend::AddFilter(int id, int arg_num_fields, - const Field* const * arg_fields) -{ - if ( disabled ) - return false; - - bool success = DoAddFilter(id, arg_num_fields, arg_fields); - if ( success && autostart) { - autostart = false; - return StartReading(); - } - return success; -} - -bool ReaderBackend::RemoveFilter(int id) -{ - if ( disabled ) - return false; - - bool success = DoRemoveFilter(id); - SendOut(new FilterRemovedMessage(frontend, id)); - return success; // yes, I know, noone reads this. -} - void ReaderBackend::Finish() { DoFinish(); diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index e167f8ff47..28fd99f2b9 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -53,46 +53,14 @@ public: * * @param mode the opening mode for the input source * - * @param autostart automatically start the input source after the first filter has been added - * - * @return False if an error occured. - */ - bool Init(string arg_source, int mode, bool autostart); - - /** - * One-time start method of the reader. - * - * This method is called from the scripting layer, after all filters have been added. - * No data should be read before this method is called. - * - * If autostart in Init is set to true, this method is called automatically by the backend after - * the first filter has been added. - */ - bool StartReading(); - - /** - * Add an input filter to the input stream - * - * @param id identifier of the input stream - * * @param arg_num_fields number of fields contained in \a fields * * @param fields the types and names of the fields to be retrieved from the input source * * @return False if an error occured. */ - bool AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + bool Init(string arg_source, int mode, int arg_num_fields, const threading::Field* const* fields); - - /** - * Remove an input filter to the input stream - * - * @param id identifier of the input stream - * - * @return False if an error occured. - */ - bool RemoveFilter ( int id ); - /** * Finishes reading from this input stream in a regular fashion. Must not be * called if an error has been indicated earlier. After calling this, @@ -131,33 +99,7 @@ protected: * disabled and eventually deleted. When returning false, an * implementation should also call Error() to indicate what happened. */ - virtual bool DoInit(string arg_sources, int mode) = 0; - - /** - * Reader-specific start method. After this function has been called, data may be read from - * the input source and be sent to the specified filters - * - * A reader implementation must override this method. - * If it returns false, it will be assumed that a fatal error has occured - * that prevents the reader from further operation; it will then be - * disabled and eventually deleted. When returning false, an implementation - * should also call Error to indicate what happened. - */ - virtual bool DoStartReading() = 0; - - /** - * Reader-specific method to add a filter. - * - * A reader implementation must override this method. - */ - virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ) = 0; - - /** - * Reader-specific method to remove a filter. - * - * A reader implementation must override this method. - */ - virtual bool DoRemoveFilter( int id ) = 0; + virtual bool DoInit(string arg_sources, int mode, int arg_num_fields, const threading::Field* const* fields) = 0; /** * Reader-specific method implementing input finalization at @@ -209,31 +151,26 @@ protected: * * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised * - * @param id the input filter id for which the values are sent - * * @param val list of threading::Values expected by the filter */ - void Put(int id, threading::Value* *val); + void Put(threading::Value* *val); /** * Method allowing a reader to delete a specific value from a bro table. * * If the receiving filter is an event, only a removed event is raised * - * @param id the input filter id for which the values are sent - * * @param val list of threading::Values expected by the filter */ - void Delete(int id, threading::Value* *val); + void Delete(threading::Value* *val); /** * Method allowing a reader to clear a value from a bro table. * * If the receiving filter is an event, this is ignored. * - * @param id the input filter id for which the values are sent */ - void Clear(int id); + void Clear(); // Content-sending-functions (tracking mode): Only changed lines are propagated. @@ -243,11 +180,9 @@ protected: * * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised. * - * @param id the input filter id for which the values are sent - * * @param val list of threading::Values expected by the filter */ - void SendEntry(int id, threading::Value* *vals); + void SendEntry(threading::Value* *vals); /** * Method telling the manager, that the current list of entries sent by SendEntry is finished. @@ -255,9 +190,8 @@ protected: * For table filters, all entries that were not updated since the last EndCurrentSend will be deleted, because they are no longer * present in the input source * - * @param id the input filter id for which the values are sent */ - void EndCurrentSend(int id); + void EndCurrentSend(); /** * Triggered by regular heartbeat messages from the main thread. diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index f7fc23bf72..9711997821 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -12,16 +12,17 @@ namespace input { class InitMessage : public threading::InputMessage { public: - InitMessage(ReaderBackend* backend, const string source, const int mode, const bool autostart) + InitMessage(ReaderBackend* backend, const string source, const int mode, const int num_fields, const threading::Field* const* fields) : threading::InputMessage("Init", backend), - source(source), mode(mode), autostart(autostart) { } + source(source), mode(mode), num_fields(num_fields), fields(fields) { } - virtual bool Process() { return Object()->Init(source, mode, autostart); } + virtual bool Process() { return Object()->Init(source, mode, num_fields, fields); } private: const string source; const int mode; - const bool autostart; + const int num_fields; + const threading::Field* const* fields; }; class UpdateMessage : public threading::InputMessage @@ -44,44 +45,6 @@ public: virtual bool Process() { Object()->Finish(); return true; } }; -class StartReadingMessage : public threading::InputMessage -{ -public: - StartReadingMessage(ReaderBackend* backend) - : threading::InputMessage("StartReading", backend) - { } - - virtual bool Process() { Object()->StartReading(); return true; } -}; - -class AddFilterMessage : public threading::InputMessage -{ -public: - AddFilterMessage(ReaderBackend* backend, const int id, const int num_fields, const threading::Field* const* fields) - : threading::InputMessage("AddFilter", backend), - id(id), num_fields(num_fields), fields(fields) { } - - virtual bool Process() { return Object()->AddFilter(id, num_fields, fields); } - -private: - const int id; - const int num_fields; - const threading::Field* const* fields; -}; - -class RemoveFilterMessage : public threading::InputMessage -{ -public: - RemoveFilterMessage(ReaderBackend* backend, const int id) - : threading::InputMessage("RemoveFilter", backend), - id(id) { } - - virtual bool Process() { return Object()->RemoveFilter(id); } - -private: - const int id; -}; - ReaderFrontend::ReaderFrontend(bro_int_t type) { disabled = initialized = false; @@ -95,7 +58,7 @@ ReaderFrontend::ReaderFrontend(bro_int_t type) { ReaderFrontend::~ReaderFrontend() { } -void ReaderFrontend::Init(string arg_source, int mode, bool autostart) { +void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, const threading::Field* const* fields) { if ( disabled ) return; @@ -105,37 +68,33 @@ void ReaderFrontend::Init(string arg_source, int mode, bool autostart) { source = arg_source; initialized = true; - backend->SendIn(new InitMessage(backend, arg_source, mode, autostart)); + backend->SendIn(new InitMessage(backend, arg_source, mode, num_fields, fields)); } void ReaderFrontend::Update() { if ( disabled ) return; + if ( !initialized ) { + reporter->Error("Tried to call update on uninitialized reader"); + return; + } + backend->SendIn(new UpdateMessage(backend)); } void ReaderFrontend::Finish() { if ( disabled ) return; + + if ( !initialized ) { + reporter->Error("Tried to call finish on uninitialized reader"); + return; + } backend->SendIn(new FinishMessage(backend)); } -void ReaderFrontend::AddFilter(const int id, const int arg_num_fields, const threading::Field* const* fields) { - if ( disabled ) - return; - - backend->SendIn(new AddFilterMessage(backend, id, arg_num_fields, fields)); -} - -void ReaderFrontend::RemoveFilter(const int id) { - if ( disabled ) - return; - - backend->SendIn(new RemoveFilterMessage(backend, id)); -} - string ReaderFrontend::Name() const { if ( source.size() ) @@ -144,13 +103,5 @@ string ReaderFrontend::Name() const return ty_name + "/" + source; } -void ReaderFrontend::StartReading() { - if ( disabled ) - return; - - backend->SendIn(new StartReadingMessage(backend)); } -} - - diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index 0df4c00e8c..1c3306e0c1 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -49,18 +49,7 @@ public: * See ReaderBackend::Init() for arguments. * This method must only be called from the main thread. */ - void Init(string arg_source, int mode, bool autostart); - - /** - * Start the reader. - * - * This methods starts the reader, after all necessary filters have been added. - * It is not necessary to call this function, if autostart has been set. - * If autostart has been set, the reader will be initialized automatically after the first filter has been added - * - * This method must only be called from the main thread. - */ - void StartReading(); + void Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* fields); /** * Force an update of the current input source. Actual action depends on @@ -72,20 +61,6 @@ public: */ void Update(); - /** - * Add a filter to the current input source. - * - * See ReaderBackend::AddFilter for arguments. - * - * The method takes ownership of \a fields - */ - void AddFilter( const int id, const int arg_num_fields, const threading::Field* const* fields ); - - /** - * Removes a filter to the current input source. - */ - void RemoveFilter ( const int id ); - /** * Finalizes writing to this tream. * diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index ab26442524..bb59b3fc1d 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -76,7 +76,6 @@ Ascii::~Ascii() void Ascii::DoFinish() { - filters.empty(); if ( file != 0 ) { file->close(); delete(file); @@ -84,9 +83,8 @@ void Ascii::DoFinish() } } -bool Ascii::DoInit(string path, int arg_mode) +bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) { - started = false; fname = path; mode = arg_mode; mtime = 0; @@ -107,17 +105,10 @@ bool Ascii::DoInit(string path, int arg_mode) file->close(); return false; } + + num_fields = arg_num_fields; + fields = arg_fields; - return true; -} - -bool Ascii::DoStartReading() { - if ( started == true ) { - Error("Started twice"); - return false; - } - - started = true; switch ( mode ) { case MANUAL: case REREAD: @@ -131,46 +122,11 @@ bool Ascii::DoStartReading() { return true; } -bool Ascii::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { - if ( HasFilter(id) ) { - Error("Filter was added twice, ignoring."); - return false; // no, we don't want to add this a second time - } - - Filter f; - f.num_fields = arg_num_fields; - f.fields = fields; - - filters[id] = f; - - return true; -} - -bool Ascii::DoRemoveFilter ( int id ) { - if (!HasFilter(id) ) { - Error("Filter removal of nonexisting filter requested."); - return false; - } - - assert ( filters.erase(id) == 1 ); - - return true; -} - - -bool Ascii::HasFilter(int id) { - map::iterator it = filters.find(id); - if ( it == filters.end() ) { - return false; - } - return true; -} - bool Ascii::ReadHeader(bool useCached) { // try to read the header line... string line; - map fields; + map ifields; if ( !useCached ) { if ( !GetLine(line) ) { @@ -194,37 +150,35 @@ bool Ascii::ReadHeader(bool useCached) { if ( !getline(splitstream, s, separator[0])) break; - fields[s] = pos; + ifields[s] = pos; pos++; } //printf("Updating fields from description %s\n", line.c_str()); - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - (*it).second.columnMap.clear(); - - for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - const Field* field = (*it).second.fields[i]; - - map::iterator fit = fields.find(field->name); - if ( fit == fields.end() ) { - Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); - return false; - } - - - FieldMapping f(field->name, field->type, field->subtype, fields[field->name]); - if ( field->secondary_name != "" ) { - map::iterator fit2 = fields.find(field->secondary_name); - if ( fit2 == fields.end() ) { - Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); - return false; - } - f.secondary_position = fields[field->secondary_name]; - } - (*it).second.columnMap.push_back(f); + columnMap.clear(); + + for ( unsigned int i = 0; i < num_fields; i++ ) { + const Field* field = fields[i]; + + map::iterator fit = ifields.find(field->name); + if ( fit == ifields.end() ) { + Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); + return false; } + + FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]); + if ( field->secondary_name != "" ) { + map::iterator fit2 = ifields.find(field->secondary_name); + if ( fit2 == ifields.end() ) { + Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); + return false; + } + f.secondary_position = ifields[field->secondary_name]; + } + columnMap.push_back(f); } + // well, that seems to have worked... return true; @@ -461,57 +415,55 @@ bool Ascii::DoUpdate() { pos--; // for easy comparisons of max element. - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - Value** fields = new Value*[(*it).second.num_fields]; + Value** fields = new Value*[num_fields]; - int fpos = 0; - for ( vector::iterator fit = (*it).second.columnMap.begin(); - fit != (*it).second.columnMap.end(); - fit++ ){ + int fpos = 0; + for ( vector::iterator fit = columnMap.begin(); + fit != columnMap.end(); + fit++ ){ - if ( (*fit).position > pos || (*fit).secondary_position > pos ) { - Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); - return false; - } - - Value* val = EntryToVal(stringfields[(*fit).position], *fit); - if ( val == 0 ) { - Error("Could not convert String value to Val"); - return false; - } - - if ( (*fit).secondary_position != -1 ) { - // we have a port definition :) - assert(val->type == TYPE_PORT ); - // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - - val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); - } - - fields[fpos] = val; - - fpos++; + if ( (*fit).position > pos || (*fit).secondary_position > pos ) { + Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); + return false; } - //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); - assert ( (unsigned int) fpos == (*it).second.num_fields ); + Value* val = EntryToVal(stringfields[(*fit).position], *fit); + if ( val == 0 ) { + Error("Could not convert String value to Val"); + return false; + } + + if ( (*fit).secondary_position != -1 ) { + // we have a port definition :) + assert(val->type == TYPE_PORT ); + // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - if ( mode == STREAM ) { - Put((*it).first, fields); - } else { - SendEntry((*it).first, fields); + val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); } - /* Do not do this, ownership changes to other thread - * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - delete fields[i]; - } - delete [] fields; - */ + fields[fpos] = val; + + fpos++; } + //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); + assert ( (unsigned int) fpos == num_fields ); + + if ( mode == STREAM ) { + Put(fields); + } else { + SendEntry(fields); + } + + /* Do not do this, ownership changes to other thread + * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; + */ + } @@ -519,9 +471,7 @@ bool Ascii::DoUpdate() { //file->seekg(0, ios::beg); // and seek to start. if ( mode != STREAM ) { - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - EndCurrentSend((*it).first); - } + EndCurrentSend(); } return true; diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 3bb0e91853..40f92be717 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -39,33 +39,22 @@ public: protected: - virtual bool DoInit(string path, int mode); - - virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); - - virtual bool DoRemoveFilter ( int id ); + virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); virtual void DoFinish(); virtual bool DoUpdate(); - virtual bool DoStartReading(); - private: virtual bool DoHeartbeat(double network_time, double current_time); - struct Filter { - unsigned int num_fields; + unsigned int num_fields; - const threading::Field* const * fields; // raw mapping + const threading::Field* const * fields; // raw mapping - // map columns in the file to columns to send back to the manager - vector columnMap; - - }; - - bool HasFilter(int id); + // map columns in the file to columns to send back to the manager + vector columnMap; bool ReadHeader(bool useCached); threading::Value* EntryToVal(string s, FieldMapping type); @@ -75,8 +64,6 @@ private: ifstream* file; string fname; - map filters; - // Options set from the script-level. string separator; @@ -91,7 +78,6 @@ private: int mode; - bool started; time_t mtime; }; diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index f2892e7af5..27415b525f 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -40,7 +40,6 @@ Raw::~Raw() void Raw::DoFinish() { - filters.empty(); if ( file != 0 ) { file->close(); delete(file); @@ -48,9 +47,8 @@ void Raw::DoFinish() } } -bool Raw::DoInit(string path, int arg_mode) +bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) { - started = false; fname = path; mode = arg_mode; mtime = 0; @@ -66,16 +64,19 @@ bool Raw::DoInit(string path, int arg_mode) return false; } - return true; -} - -bool Raw::DoStartReading() { - if ( started == true ) { - Error("Started twice"); + if ( arg_num_fields != 1 ) { + Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); return false; - } + } + + if ( fields[0]->type != TYPE_STRING ) { + Error("Filter for raw reader contains a field that is not of type string."); + return false; + } + + num_fields = arg_num_fields; + fields = arg_fields; - started = true; switch ( mode ) { case MANUAL: case REREAD: @@ -89,51 +90,6 @@ bool Raw::DoStartReading() { return true; } -bool Raw::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { - - if ( arg_num_fields != 1 ) { - Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); - return false; - } - - if ( fields[0]->type != TYPE_STRING ) { - Error("Filter for raw reader contains a field that is not of type string."); - return false; - } - - if ( HasFilter(id) ) { - Error("Filter was added twice, ignoring"); - return false; // no, we don't want to add this a second time - } - - Filter f; - f.num_fields = arg_num_fields; - f.fields = fields; - - filters[id] = f; - - return true; -} - -bool Raw::DoRemoveFilter ( int id ) { - if (!HasFilter(id) ) { - Error("Filter removal of nonexisting filter requested."); - return false; - } - - assert ( filters.erase(id) == 1 ); - - return true; -} - - -bool Raw::HasFilter(int id) { - map::iterator it = filters.find(id); - if ( it == filters.end() ) { - return false; - } - return true; -} bool Raw::GetLine(string& str) { while ( getline(*file, str, separator[0]) ) { @@ -188,21 +144,16 @@ bool Raw::DoUpdate() { string line; while ( GetLine(line) ) { - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + assert (num_fields == 1); + + Value** fields = new Value*[1]; - assert ((*it).second.num_fields == 1); + // filter has exactly one text field. convert to it. + Value* val = new Value(TYPE_STRING, true); + val->val.string_val = new string(line); + fields[0] = val; - Value** fields = new Value*[1]; - - // filter has exactly one text field. convert to it. - Value* val = new Value(TYPE_STRING, true); - val->val.string_val = new string(line); - fields[0] = val; - - Put((*it).first, fields); - - } - + Put(fields); } return true; diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index e046cb2ff7..ace4e0ee88 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -19,37 +19,21 @@ public: protected: - virtual bool DoInit(string path, int mode); - - virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); - - virtual bool DoRemoveFilter ( int id ); + virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); virtual void DoFinish(); virtual bool DoUpdate(); - virtual bool DoStartReading(); - private: virtual bool DoHeartbeat(double network_time, double current_time); - struct Filter { - unsigned int num_fields; - - const threading::Field* const * fields; // raw mapping - }; - - bool HasFilter(int id); - bool GetLine(string& str); ifstream* file; string fname; - map filters; - // Options set from the script-level. string separator; @@ -58,8 +42,11 @@ private: int mode; - bool started; time_t mtime; + + unsigned int num_fields; + + const threading::Field* const * fields; // raw mapping }; diff --git a/src/types.bif b/src/types.bif index a9c6ecb3a8..26850bfa93 100644 --- a/src/types.bif +++ b/src/types.bif @@ -182,10 +182,6 @@ enum Event %{ EVENT_REMOVED, %} -enum ID %{ - Unknown, -%} - enum Mode %{ MANUAL = 0, REREAD = 1, diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 156898edca..827b1ce283 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -45,12 +41,10 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); - Input::remove_tablefilter(A::INPUT, "ssh"); - Input::remove_stream(A::INPUT); + Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(id: string) { print servers; } From 367c4b4a7e7d8522d16a64f1d113a56104cab316 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Mar 2012 07:53:29 -0700 Subject: [PATCH 077/149] make raw reading work. apparently there was a crash in the reader plugin, but main bro did not notice but waited for eternity for it do to something. --- src/input/Manager.cc | 18 ++++++++---------- src/input/readers/Raw.cc | 9 +++++++-- .../scripts/base/frameworks/input/raw.bro | 9 ++------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 44b5bf44db..af82b676c6 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -220,11 +220,8 @@ bool Manager::CreateStream(Filter* info, RecordVal* description) info->name = name; info->source = source; - -#ifdef DEBUG - DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", - name.c_str()); -#endif + DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", + name.c_str()); return true; @@ -334,6 +331,10 @@ bool Manager::CreateEventStream(RecordVal* fval) { filter->reader->Init(filter->source, filter->mode, filter->num_fields, logf ); readers[filter->reader] = filter; + + DBG_LOG(DBG_INPUT, "Successfully created event stream %s", + filter->name.c_str()); + return true; } @@ -482,12 +483,9 @@ bool Manager::CreateTableStream(RecordVal* fval) { filter->reader->Init(filter->source, filter->mode, fieldsV.size(), fields ); readers[filter->reader] = filter; - -#ifdef DEBUG - DBG_LOG(DBG_INPUT, "Successfully created table stream %s", - filter->name.c_str()); -#endif + DBG_LOG(DBG_INPUT, "Successfully created table stream %s", + filter->name.c_str()); return true; } diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index 27415b525f..a83314c491 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -63,6 +63,9 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } + + num_fields = arg_num_fields; + fields = arg_fields; if ( arg_num_fields != 1 ) { Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); @@ -74,8 +77,9 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con return false; } - num_fields = arg_num_fields; - fields = arg_fields; +#ifdef DEBUG + Debug(DBG_INPUT, "Raw reader created, will perform first update"); +#endif switch ( mode ) { case MANUAL: @@ -87,6 +91,7 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con assert(false); } + return true; } diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 5f196648b6..4de5c3450e 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -16,10 +16,6 @@ sdf module A; -export { - redef enum Input::ID += { INPUT }; -} - type Val: record { s: string; }; @@ -30,6 +26,5 @@ event line(tpe: Input::Event, s: string) { event bro_init() { - Input::create_stream(A::INPUT, [$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM]); - Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); + Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); } From 842f635695d67f027ff2911aa41272ec9ceabf71 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Mar 2012 08:10:28 -0700 Subject: [PATCH 078/149] give EventDescripion field back to events --- src/input/Manager.cc | 25 ++++++++++++++----- .../scripts/base/frameworks/input/raw.bro | 3 ++- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index af82b676c6..7ac1de92c2 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -41,11 +41,14 @@ public: EnumVal* type; ReaderFrontend* reader; + RecordVal* description; + virtual ~Filter(); }; Manager::Filter::~Filter() { Unref(type); + Unref(description); delete(reader); } @@ -219,6 +222,8 @@ bool Manager::CreateStream(Filter* info, RecordVal* description) info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault info->name = name; info->source = source; + Ref(description); + info->description = description; DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", name.c_str()); @@ -274,27 +279,33 @@ bool Manager::CreateEventStream(RecordVal* fval) { reporter->Error("events first attribute must be of type Input::Event"); return false; } + + if ( ! same_type((*args)[1], BifType::Record::Input::EventDescription, 0) ) + { + reporter->Error("events second attribute must be of type Input::EventDescription"); + return false; + } if ( want_record->InternalInt() == 0 ) { - if ( args->length() != fields->NumFields() + 1 ) { - reporter->Error("events has wrong number of arguments"); + if ( args->length() != fields->NumFields() + 2 ) { + reporter->Error("event has wrong number of arguments"); return false; } for ( int i = 0; i < fields->NumFields(); i++ ) { - if ( !same_type((*args)[i+1], fields->FieldType(i) ) ) { + if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) { reporter->Error("Incompatible type for event"); return false; } } } else if ( want_record->InternalInt() == 1 ) { - if ( args->length() != 2 ) { - reporter->Error("events has wrong number of arguments"); + if ( args->length() != 3 ) { + reporter->Error("event has wrong number of arguments"); return false; } - if ( !same_type((*args)[1], fields ) ) { + if ( !same_type((*args)[2], fields ) ) { reporter->Error("Incompatible type for event"); return false; } @@ -965,6 +976,8 @@ int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const * // no tracking, send everything with a new event... //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); out_vals.push_back(type); + Ref(filter->description); + out_vals.push_back(filter->description); int position = 0; if ( filter->want_record ) { diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 4de5c3450e..0399eb301d 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -20,7 +20,8 @@ type Val: record { s: string; }; -event line(tpe: Input::Event, s: string) { +event line(tpe: Input::Event, description: Input::EventDescription, s: string) { + print description; print s; } From e59aed6ce35afc89bdf0d8114a0a2a330dee83ec Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Mar 2012 08:31:19 -0700 Subject: [PATCH 079/149] for seth - reverse order of event arguments --- src/input/Manager.cc | 12 ++++++------ testing/btest/scripts/base/frameworks/input/raw.bro | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 7ac1de92c2..d0db846769 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -274,15 +274,15 @@ bool Manager::CreateEventStream(RecordVal* fval) { return false; } - if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) { - reporter->Error("events first attribute must be of type Input::Event"); + reporter->Error("events second attribute must be of type Input::Event"); return false; } - if ( ! same_type((*args)[1], BifType::Record::Input::EventDescription, 0) ) + if ( ! same_type((*args)[0], BifType::Record::Input::EventDescription, 0) ) { - reporter->Error("events second attribute must be of type Input::EventDescription"); + reporter->Error("events first attribute must be of type Input::EventDescription"); return false; } @@ -973,11 +973,11 @@ int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const * Val *val; list out_vals; + Ref(filter->description); + out_vals.push_back(filter->description); // no tracking, send everything with a new event... //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); out_vals.push_back(type); - Ref(filter->description); - out_vals.push_back(filter->description); int position = 0; if ( filter->want_record ) { diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 0399eb301d..6b9fb8ef96 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -20,7 +20,7 @@ type Val: record { s: string; }; -event line(tpe: Input::Event, description: Input::EventDescription, s: string) { +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { print description; print s; } From 29f56b4986bfe3b4fff59458fe0aacfef572c1e6 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Mar 2012 23:43:13 -0700 Subject: [PATCH 080/149] continue finetuning of interface + adjust tests. streaming + re-reading do not seem to work completely correctly + there are still some strange random crashes. --- .../scripts.base.frameworks.input.event/out | 49 ++++++++ .../scripts.base.frameworks.input.raw/out | 56 ++++++++++ .../out | 105 ++++++++++++++++++ .../scripts/base/frameworks/input/basic.bro | 2 +- .../scripts/base/frameworks/input/event.bro | 13 +-- .../frameworks/input/onecolumn-norecord.bro | 13 +-- .../frameworks/input/onecolumn-record.bro | 11 +- .../scripts/base/frameworks/input/port.bro | 15 +-- .../frameworks/input/predicate-stream.bro | 11 +- .../base/frameworks/input/predicate.bro | 12 +- .../scripts/base/frameworks/input/raw.bro | 2 + .../scripts/base/frameworks/input/reread.bro | 15 +-- .../scripts/base/frameworks/input/stream.bro | 12 +- .../base/frameworks/input/tableevent.bro | 15 +-- .../base/frameworks/input/twofilters.bro | 6 +- 15 files changed, 251 insertions(+), 86 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out index e32a2aea00..59070cd88e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.event/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out @@ -1,21 +1,70 @@ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 1 T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 2 T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 3 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 4 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 5 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 6 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 7 T diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out index 2059013c5d..34a5599dc9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out @@ -1,8 +1,64 @@ +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW 3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out index 54048a86b8..56b36a1a0e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -1,21 +1,126 @@ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=1] T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=2] T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=3] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=4] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=5] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=6] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=7] T diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 827b1ce283..8d4028a12e 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -45,6 +45,6 @@ event bro_init() Input::remove("ssh"); } -event Input::update_finished(id: string) { +event Input::update_finished(name: string, source:string) { print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index 41eba1613c..dca75334d0 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -19,16 +19,13 @@ module A; -export { - redef enum Input::ID += { INPUT }; -} - type Val: record { i: int; b: bool; }; -event line(tpe: Input::Event, i: int, b: bool) { +event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool) { + print description; print tpe; print i; print b; @@ -36,6 +33,6 @@ event line(tpe: Input::Event, i: int, b: bool) { event bro_init() { - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); + Input::add_event([$source="input.log", $name="input", $fields=Val, $ev=line]); + Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index bcbba05a3e..d6c81cb2db 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -30,12 +26,11 @@ global servers: table[int] of Val = table(); event bro_init() { - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 1c532ba6a9..ca1e956f35 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -30,12 +26,11 @@ global servers: table[int] of Val = table(); event bro_init() { - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); + Input::add_table([$name="input", $source="input.log", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 801d6bac3f..88e86eb5dc 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -13,10 +13,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: addr; }; @@ -29,17 +25,14 @@ global servers: table[addr] of Val = table(); event bro_init() { - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers]); print servers[1.2.3.4]; print servers[1.2.3.5]; print servers[1.2.3.6]; - Input::remove_tablefilter(A::INPUT, "input"); - Input::remove_stream(A::INPUT); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print servers[1.2.3.4]; print servers[1.2.3.5]; print servers[1.2.3.6]; diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro index f08aaef998..20c69131cb 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -23,10 +23,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -38,7 +34,7 @@ type Val: record { global servers: table[int] of Val = table(); global ct: int; -event line(tpe: Input::Event, left: Idx, right: bool) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { ct = ct + 1; if ( ct < 3 ) { return; @@ -75,9 +71,10 @@ event bro_init() { ct = 0; # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log", $mode=Input::STREAM]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line, + Input::add_table([$source="input.log", $mode=Input::STREAM, $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); + Input::remove("input"); + } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 009911e6a8..278ac7418e 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -20,10 +20,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -37,13 +33,13 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { if ( 1 in servers ) { print "VALID"; } diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 6b9fb8ef96..8ec6c12a78 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -22,10 +22,12 @@ type Val: record { event line(description: Input::EventDescription, tpe: Input::Event, s: string) { print description; + print tpe; print s; } event bro_init() { Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); + Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 742d68605b..0930cdcb34 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -62,10 +62,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -94,8 +90,9 @@ global outfile: file; global try: count; -event line(tpe: Input::Event, left: Idx, right: Val) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { print outfile, "============EVENT============"; + #print outfile, description; print outfile, tpe; print outfile, left; print outfile, right; @@ -106,8 +103,7 @@ event bro_init() outfile = open ("../out"); try = 0; # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::REREAD]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, $pred(typ: Input::Event, left: Idx, right: Val) = { print outfile, "============PREDICATE============"; print outfile, typ; @@ -119,7 +115,7 @@ event bro_init() } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print outfile, "==========SERVERS============"; print outfile, servers; @@ -127,7 +123,6 @@ event Input::update_finished(id: Input::ID) { if ( try == 5 ) { print outfile, "done"; close(outfile); - Input::remove_tablefilter(A::INPUT, "ssh"); - Input::remove_stream(A::INPUT); + Input::remove("input"); } } diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro index db368074aa..571a2273c1 100644 --- a/testing/btest/scripts/base/frameworks/input/stream.bro +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -28,10 +28,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -60,7 +56,7 @@ global outfile: file; global try: count; -event line(tpe: Input::Event, left: Idx, right: Val) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { print outfile, "============EVENT============"; print outfile, tpe; print outfile, left; @@ -73,8 +69,7 @@ event line(tpe: Input::Event, left: Idx, right: Val) { if ( try == 3 ) { print outfile, "done"; close(outfile); - Input::remove_tablefilter(A::INPUT, "ssh"); - Input::remove_stream(A::INPUT); + Input::remove("input"); } } @@ -83,7 +78,6 @@ event bro_init() outfile = open ("../out"); try = 0; # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::STREAM]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); + Input::add_table([$source="../input.log", $mode=Input::STREAM, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); } diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index 0c86ac94b8..e40485dd12 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -18,12 +18,6 @@ redef InputAscii::empty_field = "EMPTY"; -module A; - -export { - redef enum Log::ID += { LOG }; -} - type Idx: record { i: int; }; @@ -34,7 +28,8 @@ type Val: record { global destination: table[int] of Val = table(); -event line(tpe: Input::Event, left: Idx, right: bool) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { + print description; print tpe; print left; print right; @@ -42,6 +37,6 @@ event line(tpe: Input::Event, left: Idx, right: bool) { event bro_init() { - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro index 260f73e58f..5e94aafba9 100644 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -20,10 +20,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -49,7 +45,7 @@ event bro_init() Input::force_update(A::INPUT); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { if ( done == T ) { return; } From 3286d013c9d37b2fbeb9bcdbf171d83fae12f168 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Mar 2012 23:45:10 -0700 Subject: [PATCH 081/149] forgot two files. --- scripts/base/frameworks/input/main.bro | 2 +- src/input/Manager.cc | 38 +++++++++++++++----------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4f7f9983d1..e06dfae005 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -110,7 +110,7 @@ export { global force_update: function(id: string) : bool; ## Event that is called, when the update of a specific source is finished - global update_finished: event(id: string); + global update_finished: event(name: string, source:string); } @load base/input.bif diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d0db846769..28b6afe63f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -376,7 +376,7 @@ bool Manager::CreateTableStream(RecordVal* fval) { } TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); - // check if index fields match tabla description + // check if index fields match table description { int num = idx->NumFields(); const type_list* tl = dst->Type()->AsTableType()->IndexTypes(); @@ -416,29 +416,35 @@ bool Manager::CreateTableStream(RecordVal* fval) { const type_list* args = etype->ArgTypes()->Types(); - if ( args->length() != 3 ) + if ( args->length() != 4 ) { - reporter->Error("Table event must take 3 arguments"); + reporter->Error("Table event must take 4 arguments"); return false; } - if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + if ( ! same_type((*args)[0], BifType::Record::Input::TableDescription, 0) ) { - reporter->Error("table events first attribute must be of type Input::Event"); + reporter->Error("table events first attribute must be of type Input::TableDescription"); return false; } - if ( ! same_type((*args)[1], idx) ) + if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("table events second attribute must be of type Input::Event"); + return false; + } + + if ( ! same_type((*args)[2], idx) ) { reporter->Error("table events index attributes do not match"); return false; } - if ( want_record->InternalInt() == 1 && ! same_type((*args)[2], val) ) + if ( want_record->InternalInt() == 1 && ! same_type((*args)[3], val) ) { reporter->Error("table events value attributes do not match"); return false; - } else if ( want_record->InternalInt() == 0 && !same_type((*args)[2], val->FieldType(0) ) ) { + } else if ( want_record->InternalInt() == 0 && !same_type((*args)[3], val->FieldType(0) ) ) { reporter->Error("table events value attribute does not match"); return false; } @@ -825,14 +831,15 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); - SendEvent(filter->event, 3, ev, predidx, oldval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); if ( filter->num_val_fields == 0 ) { - SendEvent(filter->event, 3, ev, predidx); + Ref(filter->description); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx); } else { - SendEvent(filter->event, 3, ev, predidx, valval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval); } } } @@ -936,7 +943,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { } - SendEvent(handler, 1, new BroString(i->name)); + SendEvent(handler, 2, new BroString(i->name), new BroString(i->source)); } void Manager::Put(ReaderFrontend* reader, Value* *vals) { @@ -1080,14 +1087,13 @@ int Manager::PutTable(Filter* i, const Value* const *vals) { assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); - SendEvent(filter->event, 3, ev, predidx, oldval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - Ref(valval); if ( filter->num_val_fields == 0 ) { - SendEvent(filter->event, 3, ev, predidx); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx); } else { - SendEvent(filter->event, 3, ev, predidx, valval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval->Ref()); } } } From bf597012f89bbcfd374e84e98a85b48d637254f6 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 18 Mar 2012 10:50:10 -0700 Subject: [PATCH 082/149] fix some stupid, not that easy to find bugs. Functionality seems to work completely again - including all tests passing. --- src/input/Manager.cc | 10 +- src/input/ReaderBackend.cc | 2 + src/input/readers/Ascii.cc | 8 +- .../scripts.base.frameworks.input.repeat/out | 160 +++++ .../scripts.base.frameworks.input.reread/out | 662 +++++++++++++----- .../out | 15 - .../scripts/base/frameworks/input/repeat.bro | 41 ++ .../scripts/base/frameworks/input/reread.bro | 6 +- .../base/frameworks/input/twofilters.bro | 100 --- 9 files changed, 690 insertions(+), 314 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.repeat/out delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out create mode 100644 testing/btest/scripts/base/frameworks/input/repeat.bro delete mode 100644 testing/btest/scripts/base/frameworks/input/twofilters.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 28b6afe63f..fb7ea6edca 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -837,7 +837,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { Ref(valval); if ( filter->num_val_fields == 0 ) { Ref(filter->description); - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx); + SendEvent(filter->event, 3, filter->description->Ref(), ev, predidx); } else { SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval); } @@ -898,7 +898,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { Ref(predidx); Ref(val); - bool result = CallPred(filter->pred, 3, ev, predidx, val); + bool result = CallPred(filter->pred, 4, filter->description->Ref(), ev, predidx, val); if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict @@ -943,7 +943,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { } - SendEvent(handler, 2, new BroString(i->name), new BroString(i->source)); + SendEvent(handler, 2, new StringVal(i->name.c_str()), new StringVal(i->source.c_str())); } void Manager::Put(ReaderFrontend* reader, Value* *vals) { @@ -1154,7 +1154,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - filterresult = CallPred(filter->pred, 3, ev, predidx, val); + filterresult = CallPred(filter->pred, 4, filter->description->Ref(), ev, predidx, val); if ( filterresult == false ) { // keep it. @@ -1170,7 +1170,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { assert(val != 0); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(filter->event, 3, ev, idxval, val); + SendEvent(filter->event, 4, filter->description->Ref(), ev, idxval, val); } } diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index b33e19d297..0a6ff37dc2 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -177,6 +177,8 @@ bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, DisableFrontend(); } + disabled = !success; + return success; } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index bb59b3fc1d..a04a40e780 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -89,6 +89,9 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c mode = arg_mode; mtime = 0; + num_fields = arg_num_fields; + fields = arg_fields; + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; @@ -106,9 +109,6 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c return false; } - num_fields = arg_num_fields; - fields = arg_fields; - switch ( mode ) { case MANUAL: case REREAD: @@ -480,7 +480,7 @@ bool Ascii::DoUpdate() { bool Ascii::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - + switch ( mode ) { case MANUAL: // yay, we do nothing :) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out b/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out new file mode 100644 index 0000000000..71de0d2570 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out @@ -0,0 +1,160 @@ +input0 +input.log +{ +[1] = T +} +input1 +input.log +{ +[1] = T +} +input2 +input.log +{ +[1] = T +} +input3 +input.log +{ +[1] = T +} +input4 +input.log +{ +[1] = T +} +input5 +input.log +{ +[1] = T +} +input6 +input.log +{ +[1] = T +} +input7 +input.log +{ +[1] = T +} +input8 +input.log +{ +[1] = T +} +input9 +input.log +{ +[1] = T +} +input10 +input.log +{ +[1] = T +} +input11 +input.log +{ +[1] = T +} +input12 +input.log +{ +[1] = T +} +input13 +input.log +{ +[1] = T +} +input14 +input.log +{ +[1] = T +} +input15 +input.log +{ +[1] = T +} +input16 +input.log +{ +[1] = T +} +input17 +input.log +{ +[1] = T +} +input18 +input.log +{ +[1] = T +} +input19 +input.log +{ +[1] = T +} +input20 +input.log +{ +[1] = T +} +input21 +input.log +{ +[1] = T +} +input22 +input.log +{ +[1] = T +} +input23 +input.log +{ +[1] = T +} +input24 +input.log +{ +[1] = T +} +input25 +input.log +{ +[1] = T +} +input26 +input.log +{ +[1] = T +} +input27 +input.log +{ +[1] = T +} +input28 +input.log +{ +[1] = T +} +input29 +input.log +{ +[1] = T +} +input30 +input.log +{ +[1] = T +} +input31 +input.log +{ +[1] = T +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index f244f11a73..545a1cb781 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -14,8 +14,44 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-42] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -59,8 +95,56 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-43] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -116,8 +200,56 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_CHANGED +Left [i=-43] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -173,8 +305,68 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-44] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -203,8 +395,80 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-45] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -233,8 +497,92 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-46] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -263,8 +611,104 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-47] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -293,22 +737,8 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ -Input::EVENT_NEW -[i=-48] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -==========SERVERS============ -{ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -393,10 +823,30 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -} -============PREDICATE============ -Input::EVENT_REMOVED -[i=-43] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-48] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -408,175 +858,10 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-46] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-44] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-47] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-45] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-42] -[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-43] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-46] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-44] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-47] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-45] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-42] -[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ==========SERVERS============ { -[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, 1, @@ -587,6 +872,5 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -} -done +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out deleted file mode 100644 index 5b1ee5e983..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out +++ /dev/null @@ -1,15 +0,0 @@ -VALID -VALID -VALID -VALID -VALID -VALID -VALID -MARK -VALID -VALID -VALID -VALID -VALID -VALID -VALID diff --git a/testing/btest/scripts/base/frameworks/input/repeat.bro b/testing/btest/scripts/base/frameworks/input/repeat.bro new file mode 100644 index 0000000000..58ce9a1675 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/repeat.bro @@ -0,0 +1,41 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +const one_to_32: vector of count = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32}; + +event bro_init() +{ + for ( i in one_to_32 ) { + Input::add_table([$source="input.log", $name=fmt("input%d", i), $idx=Idx, $val=Val, $destination=destination, $want_record=F]); + Input::remove(fmt("input%d", i)); + } +} + +event Input::update_finished(name: string, source: string) { + print name; + print source; + print destination; +} diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 0930cdcb34..f33b060fe0 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -92,9 +92,13 @@ global try: count; event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { print outfile, "============EVENT============"; - #print outfile, description; + print outfile, "Description"; + print outfile, description; + print outfile, "Type"; print outfile, tpe; + print outfile, "Left"; print outfile, left; + print outfile, "Right"; print outfile, right; } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro deleted file mode 100644 index 5e94aafba9..0000000000 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ /dev/null @@ -1,100 +0,0 @@ -# -# @TEST-EXEC: bro %INPUT >out -# @TEST-EXEC: btest-diff out - -@TEST-START-FILE input.log -#separator \x09 -#path ssh -#fields i b -#types int bool -1 T -2 T -3 F -4 F -5 F -6 F -7 T -@TEST-END-FILE - -redef InputAscii::empty_field = "EMPTY"; - -module A; - -type Idx: record { - i: int; -}; - -type Val: record { - b: bool; -}; - -global destination1: table[int] of Val = table(); -global destination2: table[int] of Val = table(); - -global done: bool = F; - -event bro_init() -{ - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log", $autostart=F]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, - $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } - ]); - Input::add_tablefilter(A::INPUT, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); - - Input::force_update(A::INPUT); -} - -event Input::update_finished(name: string, source: string) { - if ( done == T ) { - return; - } - - done = T; - - if ( 1 in destination1 ) { - print "VALID"; - } - if ( 2 in destination1 ) { - print "VALID"; - } - if ( !(3 in destination1) ) { - print "VALID"; - } - if ( !(4 in destination1) ) { - print "VALID"; - } - if ( !(5 in destination1) ) { - print "VALID"; - } - if ( !(6 in destination1) ) { - print "VALID"; - } - if ( 7 in destination1 ) { - print "VALID"; - } - - print "MARK"; - - if ( 2 in destination2 ) { - print "VALID"; - } - if ( 2 in destination2 ) { - print "VALID"; - } - if ( 3 in destination2 ) { - print "VALID"; - } - if ( 4 in destination2 ) { - print "VALID"; - } - if ( 5 in destination2 ) { - print "VALID"; - } - if ( 6 in destination2 ) { - print "VALID"; - } - if ( 7 in destination2 ) { - print "VALID"; - } -} From aa6026c1a7d40b9fa8bc553ad4f42a7150a1cbeb Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 18 Mar 2012 10:52:23 -0700 Subject: [PATCH 083/149] forgot to undo this - this idea did not work, because records cannot reference themselves. --- src/input/Manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index fb7ea6edca..d0386fbb3f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -898,7 +898,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { Ref(predidx); Ref(val); - bool result = CallPred(filter->pred, 4, filter->description->Ref(), ev, predidx, val); + bool result = CallPred(filter->pred, 3, ev, predidx, val); if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict @@ -1154,7 +1154,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - filterresult = CallPred(filter->pred, 4, filter->description->Ref(), ev, predidx, val); + filterresult = CallPred(filter->pred, 3, ev, predidx, val); if ( filterresult == false ) { // keep it. From 88e0cea598e5e57d87150c05e3d59989b6102fee Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 18 Mar 2012 15:31:47 -0700 Subject: [PATCH 084/149] add execute-mode support to the raw reader - allows to directly call commands and read their output. Note that fdstream.h is from boost and has a separate license: * (C) Copyright Nicolai M. Josuttis 2001. * Permission to copy, use, modify, sell and distribute this software * is granted provided this copyright notice appears in all copies. * This software is provided "as is" without express or implied * warranty, and with no claim as to its suitability for any purpose. --- src/input/fdstream.h | 184 ++++++++++++++++++ src/input/readers/Raw.cc | 63 ++++-- src/input/readers/Raw.h | 4 + src/types.bif | 1 + .../base/frameworks/input/executeraw.bro | 33 ++++ 5 files changed, 267 insertions(+), 18 deletions(-) create mode 100644 src/input/fdstream.h create mode 100644 testing/btest/scripts/base/frameworks/input/executeraw.bro diff --git a/src/input/fdstream.h b/src/input/fdstream.h new file mode 100644 index 0000000000..585e03d10b --- /dev/null +++ b/src/input/fdstream.h @@ -0,0 +1,184 @@ +/* The following code declares classes to read from and write to + * file descriptore or file handles. + * + * See + * http://www.josuttis.com/cppcode + * for details and the latest version. + * + * - open: + * - integrating BUFSIZ on some systems? + * - optimized reading of multiple characters + * - stream for reading AND writing + * - i18n + * + * (C) Copyright Nicolai M. Josuttis 2001. + * Permission to copy, use, modify, sell and distribute this software + * is granted provided this copyright notice appears in all copies. + * This software is provided "as is" without express or implied + * warranty, and with no claim as to its suitability for any purpose. + * + * Version: Jul 28, 2002 + * History: + * Jul 28, 2002: bugfix memcpy() => memmove() + * fdinbuf::underflow(): cast for return statements + * Aug 05, 2001: first public version + */ +#ifndef BOOST_FDSTREAM_HPP +#define BOOST_FDSTREAM_HPP + +#include +#include +#include +// for EOF: +#include +// for memmove(): +#include + + +// low-level read and write functions +#ifdef _MSC_VER +# include +#else +# include +//extern "C" { +// int write (int fd, const char* buf, int num); +// int read (int fd, char* buf, int num); +//} +#endif + + +// BEGIN namespace BOOST +namespace boost { + + +/************************************************************ + * fdostream + * - a stream that writes on a file descriptor + ************************************************************/ + + +class fdoutbuf : public std::streambuf { + protected: + int fd; // file descriptor + public: + // constructor + fdoutbuf (int _fd) : fd(_fd) { + } + protected: + // write one character + virtual int_type overflow (int_type c) { + if (c != EOF) { + char z = c; + if (write (fd, &z, 1) != 1) { + return EOF; + } + } + return c; + } + // write multiple characters + virtual + std::streamsize xsputn (const char* s, + std::streamsize num) { + return write(fd,s,num); + } +}; + +class fdostream : public std::ostream { + protected: + fdoutbuf buf; + public: + fdostream (int fd) : std::ostream(0), buf(fd) { + rdbuf(&buf); + } +}; + + +/************************************************************ + * fdistream + * - a stream that reads on a file descriptor + ************************************************************/ + +class fdinbuf : public std::streambuf { + protected: + int fd; // file descriptor + protected: + /* data buffer: + * - at most, pbSize characters in putback area plus + * - at most, bufSize characters in ordinary read buffer + */ + static const int pbSize = 4; // size of putback area + static const int bufSize = 1024; // size of the data buffer + char buffer[bufSize+pbSize]; // data buffer + + public: + /* constructor + * - initialize file descriptor + * - initialize empty data buffer + * - no putback area + * => force underflow() + */ + fdinbuf (int _fd) : fd(_fd) { + setg (buffer+pbSize, // beginning of putback area + buffer+pbSize, // read position + buffer+pbSize); // end position + } + + protected: + // insert new characters into the buffer + virtual int_type underflow () { +#ifndef _MSC_VER + using std::memmove; +#endif + + // is read position before end of buffer? + if (gptr() < egptr()) { + return traits_type::to_int_type(*gptr()); + } + + /* process size of putback area + * - use number of characters read + * - but at most size of putback area + */ + int numPutback; + numPutback = gptr() - eback(); + if (numPutback > pbSize) { + numPutback = pbSize; + } + + /* copy up to pbSize characters previously read into + * the putback area + */ + memmove (buffer+(pbSize-numPutback), gptr()-numPutback, + numPutback); + + // read at most bufSize new characters + int num; + num = read (fd, buffer+pbSize, bufSize); + if (num <= 0) { + // ERROR or EOF + return EOF; + } + + // reset buffer pointers + setg (buffer+(pbSize-numPutback), // beginning of putback area + buffer+pbSize, // read position + buffer+pbSize+num); // end of buffer + + // return next character + return traits_type::to_int_type(*gptr()); + } +}; + +class fdistream : public std::istream { + protected: + fdinbuf buf; + public: + fdistream (int fd) : std::istream(0), buf(fd) { + rdbuf(&buf); + } +}; + + +} // END namespace boost + +#endif /*BOOST_FDSTREAM_HPP*/ diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index a83314c491..777acb5951 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -7,10 +7,12 @@ #include #include "../../threading/SerialTypes.h" +#include "../fdstream.h" #define MANUAL 0 #define REREAD 1 #define STREAM 2 +#define EXECUTE 3 #include #include @@ -23,6 +25,7 @@ using threading::Field; Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend) { file = 0; + in = 0; //keyMap = new map(); @@ -41,9 +44,15 @@ Raw::~Raw() void Raw::DoFinish() { if ( file != 0 ) { - file->close(); - delete(file); + if ( mode != EXECUTE ) { + file->close(); + delete(file); + } else { // mode == EXECUTE + delete(in); + pclose(pfile); + } file = 0; + in = 0; } } @@ -53,15 +62,29 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con mode = arg_mode; mtime = 0; - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) && ( mode != EXECUTE ) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; } - file = new ifstream(path.c_str()); - if ( !file->is_open() ) { - Error(Fmt("Init: cannot open %s", fname.c_str())); - return false; + if ( mode != EXECUTE ) { + + file = new ifstream(path.c_str()); + if ( !file->is_open() ) { + Error(Fmt("Init: cannot open %s", fname.c_str())); + return false; + } + in = file; + + } else { // mode == EXECUTE + + pfile = popen(path.c_str(), "r"); + if ( pfile == NULL ) { + Error(Fmt("Could not execute command %s", path.c_str())); + return false; + } + + in = new boost::fdistream(fileno(pfile)); } num_fields = arg_num_fields; @@ -81,23 +104,14 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con Debug(DBG_INPUT, "Raw reader created, will perform first update"); #endif - switch ( mode ) { - case MANUAL: - case REREAD: - case STREAM: - DoUpdate(); - break; - default: - assert(false); - } - + DoUpdate(); return true; } bool Raw::GetLine(string& str) { - while ( getline(*file, str, separator[0]) ) { + while ( getline(*in, str, separator[0]) ) { return true; } @@ -141,6 +155,18 @@ bool Raw::DoUpdate() { return false; } + break; + case EXECUTE: + // re-execute it... + pclose(pfile); + + pfile = popen(fname.c_str(), "r"); + if ( pfile == NULL ) { + Error(Fmt("Could not execute command %s", fname.c_str())); + return false; + } + + in = new boost::fdistream(fileno(pfile)); break; default: assert(false); @@ -171,6 +197,7 @@ bool Raw::DoHeartbeat(double network_time, double current_time) switch ( mode ) { case MANUAL: + case EXECUTE: // yay, we do nothing :) break; case REREAD: diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index ace4e0ee88..55d14d956d 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -31,7 +31,11 @@ private: bool GetLine(string& str); + istream* in; ifstream* file; + + FILE* pfile; + string fname; // Options set from the script-level. diff --git a/src/types.bif b/src/types.bif index 26850bfa93..ebd206c6fa 100644 --- a/src/types.bif +++ b/src/types.bif @@ -186,6 +186,7 @@ enum Mode %{ MANUAL = 0, REREAD = 1, STREAM = 2, + EXECUTE = 3, %} module GLOBAL; diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro new file mode 100644 index 0000000000..85c1415bf3 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print description; + print tpe; + print s; +} + +event bro_init() +{ + Input::add_event([$source="wc input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $mode=Input::EXECUTE]); + Input::remove("input"); +} From 08e1771682da9d90a1deeae3918f3f2960b38772 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 20 Mar 2012 12:07:37 -0700 Subject: [PATCH 085/149] update to execute raw. support reading from commands by adppending | to the filename. support streaming reads from command. Fix something to make rearead work better. (magically happened) --- src/input/fdstream.h | 5 + src/input/readers/Raw.cc | 202 ++-- src/input/readers/Raw.h | 7 +- src/types.bif | 1 - .../out | 145 +++ .../scripts.base.frameworks.input.reread/out | 966 +++++++++++++----- .../out | 128 +++ .../out | 120 +++ .../base/frameworks/input/executeraw.bro | 2 +- .../frameworks/input/executestreamraw.bro | 58 ++ .../base/frameworks/input/rereadraw.bro | 34 + .../base/frameworks/input/streamraw.bro | 56 + 12 files changed, 1364 insertions(+), 360 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out create mode 100644 testing/btest/scripts/base/frameworks/input/executestreamraw.bro create mode 100644 testing/btest/scripts/base/frameworks/input/rereadraw.bro create mode 100644 testing/btest/scripts/base/frameworks/input/streamraw.bro diff --git a/src/input/fdstream.h b/src/input/fdstream.h index 585e03d10b..cda767dd52 100644 --- a/src/input/fdstream.h +++ b/src/input/fdstream.h @@ -35,10 +35,12 @@ #include + // low-level read and write functions #ifdef _MSC_VER # include #else +# include # include //extern "C" { // int write (int fd, const char* buf, int num); @@ -154,6 +156,9 @@ class fdinbuf : public std::streambuf { // read at most bufSize new characters int num; num = read (fd, buffer+pbSize, bufSize); + if ( num == EAGAIN ) { + return 0; + } if (num <= 0) { // ERROR or EOF return EOF; diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index 777acb5951..fb9243e713 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -12,11 +12,11 @@ #define MANUAL 0 #define REREAD 1 #define STREAM 2 -#define EXECUTE 3 #include #include #include +#include using namespace input::reader; using threading::Value; @@ -44,52 +44,73 @@ Raw::~Raw() void Raw::DoFinish() { if ( file != 0 ) { - if ( mode != EXECUTE ) { - file->close(); - delete(file); - } else { // mode == EXECUTE - delete(in); - pclose(pfile); - } - file = 0; - in = 0; + Close(); } } +bool Raw::Open() +{ + if ( execute ) { + file = popen(fname.c_str(), "r"); + if ( file == NULL ) { + Error(Fmt("Could not execute command %s", fname.c_str())); + return false; + } + } else { + file = fopen(fname.c_str(), "r"); + if ( file == NULL ) { + Error(Fmt("Init: cannot open %s", fname.c_str())); + return false; + } + } + + in = new boost::fdistream(fileno(file)); + + if ( execute && mode == STREAM ) { + fcntl(fileno(file), F_SETFL, O_NONBLOCK); + } + + return true; +} + +bool Raw::Close() +{ + if ( file == NULL ) { + InternalError(Fmt("Trying to close closed file for stream %s", fname.c_str())); + return false; + } + + if ( execute ) { + delete(in); + pclose(file); + } else { + delete(in); + fclose(file); + } + + in = NULL; + file = NULL; + + return true; +} + bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) { fname = path; mode = arg_mode; mtime = 0; - - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) && ( mode != EXECUTE ) ) { - Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); - return false; - } + execute = false; + firstrun = true; + bool result; - if ( mode != EXECUTE ) { - - file = new ifstream(path.c_str()); - if ( !file->is_open() ) { - Error(Fmt("Init: cannot open %s", fname.c_str())); - return false; - } - in = file; - - } else { // mode == EXECUTE - - pfile = popen(path.c_str(), "r"); - if ( pfile == NULL ) { - Error(Fmt("Could not execute command %s", path.c_str())); - return false; - } - - in = new boost::fdistream(fileno(pfile)); - } - num_fields = arg_num_fields; fields = arg_fields; + if ( path.length() == 0 ) { + Error("No source path provided"); + return false; + } + if ( arg_num_fields != 1 ) { Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); return false; @@ -100,12 +121,45 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con return false; } + // do Initialization + char last = path[path.length()-1]; + if ( last == '|' ) { + execute = true; + fname = path.substr(0, fname.length() - 1); + + if ( ( mode != MANUAL ) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s in execution mode", mode, fname.c_str())); + return false; + } + + result = Open(); + + } else { + execute = false; + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, fname.c_str())); + return false; + } + + result = Open(); + + } + + if ( result == false ) { + return result; + } + + #ifdef DEBUG Debug(DBG_INPUT, "Raw reader created, will perform first update"); #endif + // after initialization - do update DoUpdate(); +#ifdef DEBUG + Debug(DBG_INPUT, "First update went through"); +#endif return true; } @@ -121,56 +175,45 @@ bool Raw::GetLine(string& str) { // read the entire file and send appropriate thingies back to InputMgr bool Raw::DoUpdate() { - switch ( mode ) { - case REREAD: - // check if the file has changed - struct stat sb; - if ( stat(fname.c_str(), &sb) == -1 ) { - Error(Fmt("Could not get stat for %s", fname.c_str())); - return false; - } + if ( firstrun ) { + firstrun = false; + } else { + switch ( mode ) { + case REREAD: + // check if the file has changed + struct stat sb; + if ( stat(fname.c_str(), &sb) == -1 ) { + Error(Fmt("Could not get stat for %s", fname.c_str())); + return false; + } - if ( sb.st_mtime <= mtime ) { - // no change - return true; - } + if ( sb.st_mtime <= mtime ) { + // no change + return true; + } - mtime = sb.st_mtime; - // file changed. reread. + mtime = sb.st_mtime; + // file changed. reread. - // fallthrough - case MANUAL: - case STREAM: - - if ( file && file->is_open() ) { - if ( mode == STREAM ) { - file->clear(); // remove end of file evil bits + // fallthrough + case MANUAL: + case STREAM: + Debug(DBG_INPUT, "Updating"); + if ( mode == STREAM && file != NULL && in != NULL ) { + fpurge(file); + in->clear(); // remove end of file evil bits break; } - file->close(); - } - file = new ifstream(fname.c_str()); - if ( !file->is_open() ) { - Error(Fmt("cannot open %s", fname.c_str())); - return false; - } - break; - case EXECUTE: - // re-execute it... - pclose(pfile); - - pfile = popen(fname.c_str(), "r"); - if ( pfile == NULL ) { - Error(Fmt("Could not execute command %s", fname.c_str())); - return false; - } - - in = new boost::fdistream(fileno(pfile)); - break; - default: - assert(false); + Close(); + if ( !Open() ) { + return false; + } + break; + default: + assert(false); + } } string line; @@ -195,9 +238,10 @@ bool Raw::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); + Debug(DBG_INPUT, "Heartbeat"); + switch ( mode ) { case MANUAL: - case EXECUTE: // yay, we do nothing :) break; case REREAD: diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index 55d14d956d..1cbeff4f83 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -28,13 +28,14 @@ protected: private: virtual bool DoHeartbeat(double network_time, double current_time); + bool Open(); + bool Close(); bool GetLine(string& str); istream* in; - ifstream* file; - FILE* pfile; + FILE* file; string fname; @@ -45,6 +46,8 @@ private: string headerline; int mode; + bool execute; + bool firstrun; time_t mtime; diff --git a/src/types.bif b/src/types.bif index ebd206c6fa..26850bfa93 100644 --- a/src/types.bif +++ b/src/types.bif @@ -186,7 +186,6 @@ enum Mode %{ MANUAL = 0, REREAD = 1, STREAM = 2, - EXECUTE = 3, %} module GLOBAL; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out new file mode 100644 index 0000000000..06e28de441 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out @@ -0,0 +1,145 @@ +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +q3r3057fdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfs\d +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +dfsdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 545a1cb781..46a30f387f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -303,81 +303,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ -[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}] -Type -Input::EVENT_NEW -Left -[i=-44] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_NEW @@ -393,93 +318,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ -[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}] -Type -Input::EVENT_NEW -Left -[i=-45] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_NEW @@ -495,105 +333,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ -[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}] -Type -Input::EVENT_NEW -Left -[i=-46] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_NEW @@ -609,6 +348,21 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-48] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description @@ -636,6 +390,387 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-44] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-45] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-46] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]], [-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, @@ -720,21 +855,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_NEW -[i=-48] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description @@ -873,4 +993,296 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]], -[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, \ No newline at end of file +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============PREDICATE============ +Input::EVENT_REMOVED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-43] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-46] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-44] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-47] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-45] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-42] +Left +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +==========SERVERS============ +{ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out new file mode 100644 index 0000000000..d85c8f2e83 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out @@ -0,0 +1,128 @@ +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW + +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW + +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out new file mode 100644 index 0000000000..937acf428e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out @@ -0,0 +1,120 @@ +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +q3r3057fdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfs\d +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +dfsdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro index 85c1415bf3..6fceebf885 100644 --- a/testing/btest/scripts/base/frameworks/input/executeraw.bro +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -28,6 +28,6 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string) { event bro_init() { - Input::add_event([$source="wc input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $mode=Input::EXECUTE]); + Input::add_event([$source="wc input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]); Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/executestreamraw.bro b/testing/btest/scripts/base/frameworks/input/executestreamraw.bro new file mode 100644 index 0000000000..d97a7b26a0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/executestreamraw.bro @@ -0,0 +1,58 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +@TEST-END-FILE + +@TEST-START-FILE input3.log +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. + +@TEST-END-FILE + +@load frameworks/communication/listen + +module A; + +type Val: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print outfile, description; + print outfile, tpe; + print outfile, s; + try = try + 1; + + if ( try == 9 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + Input::add_event([$source="tail -f ../input.log |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); +} diff --git a/testing/btest/scripts/base/frameworks/input/rereadraw.bro b/testing/btest/scripts/base/frameworks/input/rereadraw.bro new file mode 100644 index 0000000000..33361ad27e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/rereadraw.bro @@ -0,0 +1,34 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print description; + print tpe; + print s; +} + +event bro_init() +{ + Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::REREAD, $name="input", $fields=Val, $ev=line]); + Input::force_update("input"); + Input::remove("input"); +} diff --git a/testing/btest/scripts/base/frameworks/input/streamraw.bro b/testing/btest/scripts/base/frameworks/input/streamraw.bro new file mode 100644 index 0000000000..cc0afd5ae8 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/streamraw.bro @@ -0,0 +1,56 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +@TEST-END-FILE + +@TEST-START-FILE input3.log +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + +@load frameworks/communication/listen + +module A; + +type Val: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print outfile, description; + print outfile, tpe; + print outfile, s; + + if ( try == 3 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); +} From d39a389201180c730d6a7d24a2f0c6426ff07fe9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 20 Mar 2012 14:11:59 -0700 Subject: [PATCH 086/149] make optional fields possible for input framework. This do not have to be present in the input file and are marked as &optional in the record description. Those can e.g. be used to create field values on the file in a predicate while reading a file - example: Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; } --- src/input/Manager.cc | 10 ++++- src/input/readers/Ascii.cc | 24 ++++++++-- src/input/readers/Ascii.h | 1 + src/threading/SerialTypes.cc | 4 +- src/threading/SerialTypes.h | 5 ++- .../out | 9 ++++ .../base/frameworks/input/optional.bro | 45 +++++++++++++++++++ 7 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.optional/out create mode 100644 testing/btest/scripts/base/frameworks/input/optional.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d0386fbb3f..f6ba6f9f49 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -492,7 +492,11 @@ bool Manager::CreateTableStream(RecordVal* fval) { Unref(pred); if ( valfields > 1 ) { - assert(filter->want_record); + if ( ! filter->want_record ) { + reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", filter->name.c_str()); + delete filter; + return false; + } } @@ -631,6 +635,10 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co field->secondary_name = c->AsStringVal()->AsString()->CheckString(); } + if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) { + field->optional = true; + } + fields->push_back(field); } } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index a04a40e780..20ae79ab19 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -26,6 +26,7 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int { position = arg_position; secondary_position = -1; + present = true; } FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) @@ -33,10 +34,11 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, cons { position = arg_position; secondary_position = -1; + present = true; } FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type), subtype(arg.subtype) + : name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present) { position = arg.position; secondary_position = arg.secondary_position; @@ -162,7 +164,15 @@ bool Ascii::ReadHeader(bool useCached) { map::iterator fit = ifields.find(field->name); if ( fit == ifields.end() ) { - Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); + if ( field->optional ) { + // we do not really need this field. mark it as not present and always send an undef back. + FieldMapping f(field->name, field->type, field->subtype, -1); + f.present = false; + columnMap.push_back(f); + continue; + } + + Error(Fmt("Did not find requested field %s in input data file %s.", field->name.c_str(), fname.c_str())); return false; } @@ -220,7 +230,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { } else if ( s == "F" ) { val->val.int_val = 0; } else { - Error(Fmt("Invalid value for boolean: %s", s.c_str())); + Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str())); return false; } break; @@ -423,6 +433,14 @@ bool Ascii::DoUpdate() { fit != columnMap.end(); fit++ ){ + if ( ! fit->present ) { + // add non-present field + fields[fpos] = new Value((*fit).type, false); + fpos++; + continue; + } + + assert(fit->position >= 0 ); if ( (*fit).position > pos || (*fit).secondary_position > pos ) { Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 40f92be717..a9b14768fb 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -19,6 +19,7 @@ struct FieldMapping { int position; // for ports: pos of the second field int secondary_position; + bool present; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index 78556e5271..1d7255d695 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -13,7 +13,7 @@ bool Field::Read(SerializationFormat* fmt) int st; bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && - fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); + fmt->Read(&t, "type") && fmt->Read(&st, "subtype") && fmt->Read(&optional, "optional")); type = (TypeTag) t; subtype = (TypeTag) st; @@ -23,7 +23,7 @@ bool Field::Read(SerializationFormat* fmt) bool Field::Write(SerializationFormat* fmt) const { return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && - fmt->Write((int)subtype, "subtype")); + fmt->Write((int)subtype, "subtype"), fmt->Write(optional, "optional")); } Value::~Value() diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index ac34f3e476..bee84f2b54 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -24,17 +24,18 @@ struct Field { string secondary_name; TypeTag type; //! Type of the field. TypeTag subtype; //! Inner type for sets. + bool optional; //! needed by input framework. Is the field optional or does it have to be present in the input data /** * Constructor. */ - Field() { subtype = TYPE_VOID; } + Field() { subtype = TYPE_VOID; optional = false; } /** * Copy constructor. */ Field(const Field& other) - : name(other.name), type(other.type), subtype(other.subtype) { } + : name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) { } /** * Unserializes a field. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.optional/out b/testing/btest/Baseline/scripts.base.frameworks.input.optional/out new file mode 100644 index 0000000000..7a304fc918 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.optional/out @@ -0,0 +1,9 @@ +{ +[2] = [b=T, notb=F], +[4] = [b=F, notb=T], +[6] = [b=F, notb=T], +[7] = [b=T, notb=F], +[1] = [b=T, notb=F], +[5] = [b=F, notb=T], +[3] = [b=F, notb=T] +} diff --git a/testing/btest/scripts/base/frameworks/input/optional.bro b/testing/btest/scripts/base/frameworks/input/optional.bro new file mode 100644 index 0000000000..c354f7c3ab --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/optional.bro @@ -0,0 +1,45 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + notb: bool &optional; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, + $pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; } + ]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers; +} From 51ddc9f572a1ab8524ecc27ec5ee6de494d2fc56 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 21 Mar 2012 15:51:21 -0700 Subject: [PATCH 087/149] fix bug that crashed input framework when creating already existing stream (tried to free not yet alloccated data) + write twotables test --- src/input/Manager.cc | 48 ++- .../out | 349 ++++++++++++++++++ .../base/frameworks/input/twotables.bro | 113 ++++++ 3 files changed, 500 insertions(+), 10 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.twotables/out create mode 100644 testing/btest/scripts/base/frameworks/input/twotables.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index f6ba6f9f49..f62e87d937 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -43,14 +43,24 @@ public: RecordVal* description; + Filter(); virtual ~Filter(); }; -Manager::Filter::~Filter() { - Unref(type); - Unref(description); +Manager::Filter::Filter() { + type = 0; + reader = 0; + description = 0; +} - delete(reader); +Manager::Filter::~Filter() { + if ( type ) + Unref(type); + if ( description ) + Unref(description); + + if ( reader ) + delete(reader); } class Manager::TableFilter: public Manager::Filter { @@ -85,28 +95,46 @@ public: bool want_record; EventFilter(); + ~EventFilter(); }; -Manager::TableFilter::TableFilter() { +Manager::TableFilter::TableFilter() : Manager::Filter::Filter() { filter_type = TABLE_FILTER; tab = 0; itype = 0; rtype = 0; + + currDict = 0; + lastDict = 0; + + pred = 0; } -Manager::EventFilter::EventFilter() { +Manager::EventFilter::EventFilter() : Manager::Filter::Filter() { + fields = 0; filter_type = EVENT_FILTER; } +Manager::EventFilter::~EventFilter() { + if ( fields ) { + Unref(fields); + } +} + Manager::TableFilter::~TableFilter() { - Unref(tab); - Unref(itype); + if ( tab ) + Unref(tab); + if ( itype ) + Unref(itype); if ( rtype ) // can be 0 for sets Unref(rtype); - delete currDict; - delete lastDict; + if ( currDict != 0 ) + delete currDict; + + if ( lastDict != 0 ) + delete lastDict; } struct ReaderDefinition { diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out new file mode 100644 index 0000000000..a61a4a2993 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out @@ -0,0 +1,349 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE 2============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-42] +Right +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Description +[source=../input2.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh2, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE 2============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-43] +Right +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============PREDICATE============ +Input::EVENT_NEW +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-44] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-42] +Left +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro new file mode 100644 index 0000000000..6f18e0e939 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/twotables.bro @@ -0,0 +1,113 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: btest-bg-wait -k 2 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { + print outfile, "============EVENT============"; + print outfile, "Description"; + print outfile, description; + print outfile, "Type"; + print outfile, tpe; + print outfile, "Left"; + print outfile, left; + print outfile, "Right"; + print outfile, right; +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); + Input::add_table([$source="../input2.log", $mode=Input::REREAD, $name="ssh2", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE 2============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); +} + + +event Input::update_finished(name: string, source: string) { + print outfile, "==========SERVERS============"; + print outfile, servers; + + try = try + 1; + if ( try == 5 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} From 0db89bed3ba6248354315f6d3f1b77ab84e65a41 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 10:33:49 -0700 Subject: [PATCH 088/149] fix crash when deleting data from source where there are no events or predicates... (that happens when all testcases are too complicated and use all features..) --- src/input/Manager.cc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index f62e87d937..c1fa060b2b 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -915,18 +915,20 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { ListVal * idx = 0; Val *val = 0; + + Val* predidx = 0; + EnumVal* ev = 0; + int startpos = 0; if ( filter->pred || filter->event ) { idx = filter->tab->RecoverIndex(ih->idxkey); assert(idx != 0); val = filter->tab->Lookup(idx); assert(val != 0); + predidx = ListValToRecordVal(idx, filter->itype, &startpos); + ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); } - int startpos = 0; - Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); - EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - if ( filter->pred ) { // ask predicate, if we want to expire this element... @@ -953,8 +955,10 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { SendEvent(filter->event, 3, ev, predidx, val); } - Unref(predidx); - Unref(ev); + if ( predidx ) // if we have a filter or an event... + Unref(predidx); + if ( ev ) + Unref(ev); filter->tab->Delete(ih->idxkey); filter->lastDict->Remove(lastDictIdxKey); // deletex in next line @@ -1321,6 +1325,7 @@ RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, RecordVal* rec = new RecordVal(request_type->AsRecordType()); + assert(list != 0); int maxpos = list->Length(); for ( int i = 0; i < request_type->NumFields(); i++ ) { From 14c6c4004289e715101dcddc1a42955ac6f86e8c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 10:59:36 -0700 Subject: [PATCH 089/149] fix crash when all value fields of imported table are uninitialized. --- .../out | 4 ++ .../base/frameworks/input/emptyvals.bro | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out create mode 100644 testing/btest/scripts/base/frameworks/input/emptyvals.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out b/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out new file mode 100644 index 0000000000..f75248cf97 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out @@ -0,0 +1,4 @@ +{ +[2] = [b=], +[1] = [b=T] +} diff --git a/testing/btest/scripts/base/frameworks/input/emptyvals.bro b/testing/btest/scripts/base/frameworks/input/emptyvals.bro new file mode 100644 index 0000000000..77659d13ec --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/emptyvals.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +##types bool int +T 1 +- 2 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); +} + +event Input::update_finished(name: string, source:string) { + print servers; +} From 5f5209fcfb12cf96dbb9bb027a93133657f0d9ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 11:00:51 -0700 Subject: [PATCH 090/149] ...forgotten file. --- src/input/Manager.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index c1fa060b2b..32b60f05f2 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -760,8 +760,13 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { hash_t valhash = 0; if ( filter->num_val_fields > 0 ) { HashKey* valhashkey = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); - valhash = valhashkey->Hash(); - delete(valhashkey); + if ( valhashkey == 0 ) { + // empty line. index, but no values. + // hence we also have no hash value... + } else { + valhash = valhashkey->Hash(); + delete(valhashkey); + } } InputHash *h = filter->lastDict->Lookup(idxhash); From 7e4cbbc0735a13343062c9bc18e01f8d8d7b17fb Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 12:45:11 -0700 Subject: [PATCH 091/149] remove forgotten debug statements --- src/input/readers/Raw.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index fb9243e713..f416fbe94a 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -198,7 +198,6 @@ bool Raw::DoUpdate() { // fallthrough case MANUAL: case STREAM: - Debug(DBG_INPUT, "Updating"); if ( mode == STREAM && file != NULL && in != NULL ) { fpurge(file); in->clear(); // remove end of file evil bits @@ -238,8 +237,6 @@ bool Raw::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - Debug(DBG_INPUT, "Heartbeat"); - switch ( mode ) { case MANUAL: // yay, we do nothing :) From 6c4a40f176c74735d33bb8afe66ac429d9f416ed Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 13:09:53 -0700 Subject: [PATCH 092/149] missing include on linux --- src/input/Manager.cc | 2 +- src/input/readers/Raw.cc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 32b60f05f2..d8bf63505b 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1540,7 +1540,7 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { memcpy(data + startpos, (const char*) &(val->val.subnet_val.prefix.in.in4), length); break; case IPv6: - length += sizeof(val->val.addr_val.in.in6); + length = sizeof(val->val.addr_val.in.in6); memcpy(data + startpos, (const char*) &(val->val.subnet_val.prefix.in.in4), length); break; default: diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index f416fbe94a..cba0a29f3e 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -17,6 +17,7 @@ #include #include #include +#include using namespace input::reader; using threading::Value; @@ -199,7 +200,7 @@ bool Raw::DoUpdate() { case MANUAL: case STREAM: if ( mode == STREAM && file != NULL && in != NULL ) { - fpurge(file); + //fpurge(file); in->clear(); // remove end of file evil bits break; } From f73de0bc8c531f9b2ec0d06b9c63054dc4cf3589 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 15:11:42 -0700 Subject: [PATCH 093/149] fix small memory leak (field description given to readers was never freed). --- src/input/ReaderBackend.cc | 13 +++++++++++++ src/input/ReaderBackend.h | 3 +++ src/input/readers/Ascii.cc | 1 - 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 0a6ff37dc2..ce79ecfd39 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -169,6 +169,9 @@ bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, source = arg_source; SetName("InputReader/"+source); + num_fields = arg_num_fields; + fields = arg_fields; + // disable if DoInit returns error. int success = DoInit(arg_source, mode, arg_num_fields, arg_fields); @@ -188,6 +191,16 @@ void ReaderBackend::Finish() disabled = true; DisableFrontend(); SendOut(new ReaderFinishedMessage(frontend)); + + if ( fields != 0 ) { + + for ( unsigned int i = 0; i < num_fields; i++ ) { + delete(fields[i]); + } + + delete[] (fields); + fields = 0; + } } bool ReaderBackend::Update() diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 28fd99f2b9..5742f72368 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -228,6 +228,9 @@ private: char* buf; unsigned int buf_len; bool autostart; + + unsigned int num_fields; + const threading::Field* const * fields; // raw mapping }; } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 20ae79ab19..553a4ada81 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -73,7 +73,6 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) Ascii::~Ascii() { DoFinish(); - } void Ascii::DoFinish() From 94d439b0cba75fee93a3b08bb3b57041e694b46e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 17:17:41 -0700 Subject: [PATCH 094/149] enable predicate modification of index of value which is currently being added/removed Todo: test if this works for removal ( I think it should ). --- src/input/Manager.cc | 45 +++++++++++++++-- src/input/Manager.h | 1 + .../out | 4 ++ .../base/frameworks/input/predicatemodify.bro | 50 +++++++++++++++++++ 4 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out create mode 100644 testing/btest/scripts/base/frameworks/input/predicatemodify.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d8bf63505b..2201c69995 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -692,6 +692,31 @@ bool Manager::ForceUpdate(const string &name) return true; // update is async :( } + +Val* Manager::RecordValToIndexVal(RecordVal *r) { + Val* idxval; + + RecordType *type = r->Type()->AsRecordType(); + + int num_fields = type->NumFields(); + + if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) { + idxval = r->Lookup(0); + } else { + ListVal *l = new ListVal(TYPE_ANY); + for ( int j = 0 ; j < num_fields; j++ ) { + Val* rval = r->Lookup(j); + assert(rval != 0); + l->Append(r->LookupWithDefault(j)); + } + idxval = l; + } + + + return idxval; +} + + Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) { Val* idxval; int position = 0; @@ -788,8 +813,8 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { } - Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; + RecordVal* predidx = 0; int position = filter->num_idx_fields; if ( filter->num_val_fields == 0 ) { @@ -806,8 +831,10 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { if ( filter->pred ) { EnumVal* ev; //Ref(idxval); - int startpos = 0; - Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + int startpos = 0; + //Val* predidx = ListValToRecordVal(idxval->AsListVal(), filter->itype, &startpos); + predidx = ValueToRecordVal(vals, filter->itype, &startpos); + //ValueToRecordVal(vals, filter->itype, &startpos); Ref(valval); if ( updated ) { @@ -818,13 +845,14 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { bool result; if ( filter->num_val_fields > 0 ) { // we have values - result = CallPred(filter->pred, 3, ev, predidx, valval); + result = CallPred(filter->pred, 3, ev, predidx->Ref(), valval); } else { // no values - result = CallPred(filter->pred, 2, ev, predidx); + result = CallPred(filter->pred, 2, ev, predidx->Ref()); } if ( result == false ) { + Unref(predidx); if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... delete(filter->currDict->RemoveEntry(idxhash)); @@ -839,6 +867,13 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { } + Val* idxval; + if ( predidx != 0 ) { + idxval = RecordValToIndexVal(predidx); + Unref(predidx); + } else { + idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); + } Val* oldval = 0; if ( updated == true ) { assert(filter->num_val_fields > 0); diff --git a/src/input/Manager.h b/src/input/Manager.h index 71169c4bc2..c6dd40bd95 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -158,6 +158,7 @@ private: // Converts a threading::value to a record type. mostly used by ValueToVal RecordVal* ValueToRecordVal(const threading::Value* const *vals, RecordType *request_type, int* position); + Val* RecordValToIndexVal(RecordVal *r); // Converts a Bro ListVal to a RecordVal given the record type RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out new file mode 100644 index 0000000000..c648e63710 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out @@ -0,0 +1,4 @@ +{ +[2, idxmodified] = [b=T, s=test2], +[1, idx1] = [b=T, s=testmodified] +} diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro new file mode 100644 index 0000000000..c3198d8483 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro @@ -0,0 +1,50 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 T test1 idx1 +2 T test2 idx2 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; + ss: string; +}; + +type Val: record { + b: bool; + s: string; +}; + +global servers: table[int, string] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, + $pred(typ: Input::Event, left: Idx, right: Val) = { + if ( left$i == 1 ) { + right$s = "testmodified"; + } + + if ( left$i == 2 ) { + left$ss = "idxmodified"; + } + return T; + } + ]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers; +} From 03116d779eef498d2fcc0ab7e2822a94cbfb43f4 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 18:08:59 -0700 Subject: [PATCH 095/149] one unref to many ... apparently --- src/input/Manager.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 2201c69995..8eaca07e78 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -705,8 +705,8 @@ Val* Manager::RecordValToIndexVal(RecordVal *r) { } else { ListVal *l = new ListVal(TYPE_ANY); for ( int j = 0 ; j < num_fields; j++ ) { - Val* rval = r->Lookup(j); - assert(rval != 0); + //Val* rval = r->Lookup(j); + //assert(rval != 0); l->Append(r->LookupWithDefault(j)); } idxval = l; @@ -870,7 +870,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { Val* idxval; if ( predidx != 0 ) { idxval = RecordValToIndexVal(predidx); - Unref(predidx); + // I think there is an unref missing here. But if I insert is, it crashes :) } else { idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); } From 315948dbc8fc3b34a0add6a46686814474ff2fb7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 23 Mar 2012 11:40:59 -0700 Subject: [PATCH 096/149] add test for update functionality of tables where a predicate modifies values / indexes. Seems to work fine for all cases... --- .../out | 23 ++++ .../input/predicatemodifyandreread.bro | 107 ++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out create mode 100644 testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out new file mode 100644 index 0000000000..0adccc1856 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out @@ -0,0 +1,23 @@ +Update_finished for input, try 1 +{ +[2, idxmodified] = [b=T, s=test2], +[1, idx1] = [b=T, s=testmodified] +} +Update_finished for input, try 2 +{ +[2, idxmodified] = [b=T, s=test2], +[1, idx1] = [b=F, s=testmodified] +} +Update_finished for input, try 3 +{ +[2, idxmodified] = [b=F, s=test2], +[1, idx1] = [b=F, s=testmodified] +} +Update_finished for input, try 4 +{ +[2, idxmodified] = [b=F, s=test2] +} +Update_finished for input, try 5 +{ +[1, idx1] = [b=T, s=testmodified] +} diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro new file mode 100644 index 0000000000..1606ff6a27 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro @@ -0,0 +1,107 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input2.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input4.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input5.log input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out +# + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 T test1 idx1 +2 T test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 F test1 idx1 +2 T test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 F test1 idx1 +2 F test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input4.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +2 F test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input5.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 T test1 idx1 +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; + ss: string; +}; + +type Val: record { + b: bool; + s: string; +}; + +global servers: table[int, string] of Val = table(); +global outfile: file; +global try: count; + +event bro_init() +{ + try = 0; + outfile = open ("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $mode=Input::REREAD, + $pred(typ: Input::Event, left: Idx, right: Val) = { + if ( left$i == 1 ) { + right$s = "testmodified"; + } + + if ( left$i == 2 ) { + left$ss = "idxmodified"; + } + return T; + } + ]); +} + +event Input::update_finished(name: string, source: string) { + try = try + 1; + print outfile, fmt("Update_finished for %s, try %d", name, try); + print outfile, servers; + + if ( try == 5 ) { + close (outfile); + Input::remove("input"); + } +} From 872ad195f789a155c3dd79a4f3786388203f3dce Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 23 Mar 2012 12:30:54 -0700 Subject: [PATCH 097/149] prevent several remove operations for the same thread to be queued and output errors in that case. --- src/input/Manager.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 8eaca07e78..6d97c2f50b 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -33,6 +33,7 @@ class Manager::Filter { public: string name; string source; + bool removed; int mode; @@ -51,6 +52,7 @@ Manager::Filter::Filter() { type = 0; reader = 0; description = 0; + removed = false; } Manager::Filter::~Filter() { @@ -597,6 +599,13 @@ bool Manager::RemoveStream(const string &name) { return false; // not found } + if ( i->removed ) { + reporter->Error("Stream %s is already queued for removal. Ignoring", name.c_str()); + return false; + } + + i->removed = true; + i->reader->Finish(); #ifdef DEBUG From 9732859d44ea66098f541fd879c0781b8362b718 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 26 Mar 2012 12:20:39 -0700 Subject: [PATCH 098/149] add first simple benchmark reader (it simply spews random data, amount of lines specified in source). --- src/CMakeLists.txt | 2 +- src/input/Manager.cc | 9 +- src/input/readers/Ascii.cc | 10 +- src/input/readers/Benchmark.cc | 198 +++++++++++++++++++++++++++++++++ src/input/readers/Benchmark.h | 47 ++++++++ src/types.bif | 1 + 6 files changed, 256 insertions(+), 11 deletions(-) create mode 100644 src/input/readers/Benchmark.cc create mode 100644 src/input/readers/Benchmark.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d9ec76f8d2..9b075decd5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -425,7 +425,7 @@ set(bro_SRCS input/ReaderFrontend.cc input/readers/Ascii.cc input/readers/Raw.cc - + input/readers/Benchmark.cc ${dns_SRCS} ${openssl_SRCS} diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 6d97c2f50b..f8ad493e11 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -7,6 +7,7 @@ #include "ReaderBackend.h" #include "readers/Ascii.h" #include "readers/Raw.h" +#include "readers/Benchmark.h" #include "Event.h" #include "EventHandler.h" @@ -149,6 +150,7 @@ struct ReaderDefinition { ReaderDefinition input_readers[] = { { BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate }, { BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate }, + { BifEnum::Input::READER_BENCHMARK, "Benchmark", 0, reader::Benchmark::Instantiate }, // End marker { BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 } @@ -600,7 +602,7 @@ bool Manager::RemoveStream(const string &name) { } if ( i->removed ) { - reporter->Error("Stream %s is already queued for removal. Ignoring", name.c_str()); + reporter->Error("Stream %s is already queued for removal. Ignoring remove.", name.c_str()); return false; } @@ -690,6 +692,11 @@ bool Manager::ForceUpdate(const string &name) reporter->Error("Stream %s not found", name.c_str()); return false; } + + if ( i->removed ) { + reporter->Error("Stream %s is already queued for removal. Ignoring force update.", name.c_str()); + return false; + } i->reader->Update(); diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 553a4ada81..17391afe73 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -110,15 +110,7 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c return false; } - switch ( mode ) { - case MANUAL: - case REREAD: - case STREAM: - DoUpdate(); - break; - default: - assert(false); - } + DoUpdate(); return true; } diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc new file mode 100644 index 0000000000..b48074c146 --- /dev/null +++ b/src/input/readers/Benchmark.cc @@ -0,0 +1,198 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Benchmark.h" +#include "NetVar.h" + +#include "../../threading/SerialTypes.h" + +#define MANUAL 0 +#define REREAD 1 +#define STREAM 2 + +#include +#include +#include + +using namespace input::reader; +using threading::Value; +using threading::Field; + + + +Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) +{ +} + +Benchmark::~Benchmark() +{ + DoFinish(); +} + +void Benchmark::DoFinish() +{ +} + +bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) +{ + mode = arg_mode; + + num_fields = arg_num_fields; + fields = arg_fields; + num_lines = atoi(path.c_str()); + + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); + return false; + } + + DoUpdate(); + + return true; +} + +string Benchmark::RandomString(const int len) { + string s; + + s.reserve(len); + + static const char values[] = + "0123456789!@#$%^&*()-_=+{}[]\\|" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + for (int i = 0; i < len; ++i) { + s[i] = values[rand() / (RAND_MAX / sizeof(values))]; + } + + return s; +} + +// read the entire file and send appropriate thingies back to InputMgr +bool Benchmark::DoUpdate() { + for ( int i = 0; i < num_lines; i++ ) { + Value** field = new Value*[num_fields]; + for (unsigned int j = 0; j < num_fields; j++ ) { + field[j] = EntryToVal(fields[j]->type, fields[j]->subtype); + } + SendEntry(field); + } + + EndCurrentSend(); + + return true; +} + +threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { + Value* val = new Value(type, true); + + // basically construct something random from the fields that we want. + + switch ( type ) { + case TYPE_ENUM: + assert(false); // no enums, please. + case TYPE_STRING: + val->val.string_val = new string(RandomString(10)); + break; + + case TYPE_BOOL: + val->val.int_val = 1; // we never lie. + break; + + case TYPE_INT: + val->val.int_val = rand(); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = random(); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + val->val.uint_val = rand(); + break; + + case TYPE_PORT: + val->val.port_val.port = rand() / (RAND_MAX / 60000); + val->val.port_val.proto = TRANSPORT_UNKNOWN; + break; + + case TYPE_SUBNET: { + val->val.subnet_val.prefix = StringToAddr("192.168.17.1"); + val->val.subnet_val.length = 16; + } + break; + + case TYPE_ADDR: + val->val.addr_val = StringToAddr("192.168.17.1"); + break; + + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { + // how many entries do we have... + unsigned int length = rand() / (RAND_MAX / 15); + + Value** lvals = new Value* [length]; + + if ( type == TYPE_TABLE ) { + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } else if ( type == TYPE_VECTOR ) { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } else { + assert(false); + } + + if ( length == 0 ) + break; //empty + + for ( unsigned int pos = 0; pos < length; pos++ ) { + + Value* newval = EntryToVal(subtype, TYPE_ENUM); + if ( newval == 0 ) { + Error("Error while reading set"); + return 0; + } + lvals[pos] = newval; + } + + break; + } + + + default: + Error(Fmt("unsupported field format %d", type)); + return 0; + } + + return val; + +} + + +bool Benchmark::DoHeartbeat(double network_time, double current_time) +{ + ReaderBackend::DoHeartbeat(network_time, current_time); + + switch ( mode ) { + case MANUAL: + // yay, we do nothing :) + break; + case REREAD: + case STREAM: + Update(); // call update and not DoUpdate, because update actually checks disabled. + break; + default: + assert(false); + } + + return true; +} + diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h new file mode 100644 index 0000000000..5a82c5b726 --- /dev/null +++ b/src/input/readers/Benchmark.h @@ -0,0 +1,47 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERS_BENCHMARK_H +#define INPUT_READERS_BENCHMARK_H + + +#include "../ReaderBackend.h" + +namespace input { namespace reader { + +class Benchmark : public ReaderBackend { +public: + Benchmark(ReaderFrontend* frontend); + ~Benchmark(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Benchmark(frontend); } + +protected: + + virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); + + virtual void DoFinish(); + + virtual bool DoUpdate(); + +private: + + virtual bool DoHeartbeat(double network_time, double current_time); + + unsigned int num_fields; + + const threading::Field* const * fields; // raw mapping + + threading::Value* EntryToVal(TypeTag Type, TypeTag subtype); + + int mode; + int num_lines; + + string RandomString(const int len); + +}; + + +} +} + +#endif /* INPUT_READERS_BENCHMARK_H */ diff --git a/src/types.bif b/src/types.bif index 26850bfa93..682170e3a6 100644 --- a/src/types.bif +++ b/src/types.bif @@ -174,6 +174,7 @@ enum Reader %{ READER_DEFAULT, READER_ASCII, READER_RAW, + READER_BENCHMARK, %} enum Event %{ From 016a2540a5afc867bbfb68dd92e5dad667bf920d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 26 Mar 2012 12:41:59 -0700 Subject: [PATCH 099/149] ...and spread out streaming reads over time. --- src/input/readers/Benchmark.cc | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index b48074c146..a4cf5f6818 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -51,9 +51,7 @@ bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Fiel } string Benchmark::RandomString(const int len) { - string s; - - s.reserve(len); + string s(len, ' '); static const char values[] = "0123456789!@#$%^&*()-_=+{}[]\\|" @@ -74,10 +72,19 @@ bool Benchmark::DoUpdate() { for (unsigned int j = 0; j < num_fields; j++ ) { field[j] = EntryToVal(fields[j]->type, fields[j]->subtype); } - SendEntry(field); + + if ( mode == STREAM ) { + // do not do tracking, spread out elements over the second that we have... + Put(field); + usleep(900000/num_lines); + } else { + SendEntry(field); + } } - EndCurrentSend(); + //if ( mode != STREAM ) { // well, does not really make sense in the streaming sense - but I like getting the event. + EndCurrentSend(); + //} return true; } From 28f3fa01444b0c5595bafc6585320cdd11168b0f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 26 Mar 2012 13:52:58 -0700 Subject: [PATCH 100/149] make time types always return current time for benchmark reader --- src/input/readers/Benchmark.cc | 13 ++++++++++++- src/input/readers/Benchmark.h | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index a4cf5f6818..07ee7eb9bc 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -65,6 +65,14 @@ string Benchmark::RandomString(const int len) { return s; } +double Benchmark::CurrTime() { + struct timeval tv; + assert ( gettimeofday(&tv, 0) >= 0 ); + + return double(tv.tv_sec) + double(tv.tv_usec) / 1e6; +} + + // read the entire file and send appropriate thingies back to InputMgr bool Benchmark::DoUpdate() { for ( int i = 0; i < num_lines; i++ ) { @@ -109,8 +117,11 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { val->val.int_val = rand(); break; - case TYPE_DOUBLE: case TYPE_TIME: + val->val.double_val = CurrTime(); + break; + + case TYPE_DOUBLE: case TYPE_INTERVAL: val->val.double_val = random(); break; diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index 5a82c5b726..e8de4ac773 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -29,6 +29,8 @@ private: unsigned int num_fields; + double CurrTime(); + const threading::Field* const * fields; // raw mapping threading::Value* EntryToVal(TypeTag Type, TypeTag subtype); From 6a60f484f9faa3925cfc297e38f1f06561f41607 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 29 Mar 2012 09:03:33 -0700 Subject: [PATCH 101/149] make heart beat interval for threading configureable from scripting layer --- scripts/base/init-bare.bro | 8 ++++++++ src/const.bif | 1 + src/threading/Manager.cc | 6 +++++- src/threading/Manager.h | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 9f4e0355f0..77f90cae5f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1484,6 +1484,14 @@ export { }; } # end export +module Threading; + +export { + ## The heart beat interval used by the threading framework. + ## Changing this should usually not be neccessary and will break several tests. + const heart_beat_interval = 1.0 &redef; +} + module GLOBAL; ## An NTP message. diff --git a/src/const.bif b/src/const.bif index bc960caeb6..aadb1e1ab7 100644 --- a/src/const.bif +++ b/src/const.bif @@ -12,3 +12,4 @@ const NFS3::return_data: bool; const NFS3::return_data_max: count; const NFS3::return_data_first_only: bool; +const Threading::heart_beat_interval: double; diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d008d2e5e8..8546ca3948 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -1,5 +1,6 @@ #include "Manager.h" +#include "NetVar.h" using namespace threading; @@ -11,6 +12,9 @@ Manager::Manager() next_beat = 0; terminating = false; idle = false; + + heart_beat_interval = double(BifConst::Threading::heart_beat_interval); + DBG_LOG(DBG_THREADING, "Heart beat interval set to %f", heart_beat_interval); } Manager::~Manager() @@ -73,7 +77,7 @@ void Manager::GetFds(int* read, int* write, int* except) double Manager::NextTimestamp(double* network_time) { if ( ::network_time && ! next_beat ) - next_beat = ::network_time + HEART_BEAT_INTERVAL; + next_beat = ::network_time + heart_beat_interval; // fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat); diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 7d9ba766d4..29729f6a7a 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -120,7 +120,7 @@ protected: virtual const char* Tag() { return "threading::Manager"; } private: - static const int HEART_BEAT_INTERVAL = 1; + int heart_beat_interval; typedef std::list all_thread_list; all_thread_list all_threads; From ead30e423d0316e1e2e05ae5334f7771d1582f5c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 30 Mar 2012 08:40:38 -0700 Subject: [PATCH 102/149] change type of heart_beat_interval to interval (makes much more sese) --- scripts/base/init-bare.bro | 2 +- src/const.bif | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 3a5b2023dd..4637580337 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1489,7 +1489,7 @@ module Threading; export { ## The heart beat interval used by the threading framework. ## Changing this should usually not be neccessary and will break several tests. - const heart_beat_interval = 1.0 &redef; + const heart_beat_interval = 1.0 secs &redef; } module GLOBAL; diff --git a/src/const.bif b/src/const.bif index aadb1e1ab7..f9e5f61644 100644 --- a/src/const.bif +++ b/src/const.bif @@ -12,4 +12,4 @@ const NFS3::return_data: bool; const NFS3::return_data_max: count; const NFS3::return_data_first_only: bool; -const Threading::heart_beat_interval: double; +const Threading::heart_beat_interval: interval; From 355b85fcd7fc638ad54cf8b8d3614fe3c6ecc9e5 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 30 Mar 2012 09:08:08 -0700 Subject: [PATCH 103/149] most of the stuff we should need for benchmarking. next: search memory leaks, after 1.5million simulated inputs we are leaking about 1Gb of ram... --- scripts/base/frameworks/input/__load__.bro | 1 + .../base/frameworks/input/readers/benchmark.bro | 8 ++++++++ src/input.bif | 3 +++ src/input/ReaderBackend.cc | 7 ++++++- src/input/readers/Benchmark.cc | 15 ++++++++++++++- src/input/readers/Benchmark.h | 2 ++ 6 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 scripts/base/frameworks/input/readers/benchmark.bro diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro index b41fe5e95f..0e7d8ffb73 100644 --- a/scripts/base/frameworks/input/__load__.bro +++ b/scripts/base/frameworks/input/__load__.bro @@ -1,4 +1,5 @@ @load ./main @load ./readers/ascii @load ./readers/raw +@load ./readers/benchmark diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro new file mode 100644 index 0000000000..3293201cea --- /dev/null +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -0,0 +1,8 @@ +##! Interface for the ascii input reader. + +module InputBenchmark; + +export { + ## multiplication factor for each second + const factor = 1 &redef; +} diff --git a/src/input.bif b/src/input.bif index 1157b7b62b..e4ecf4d020 100644 --- a/src/input.bif +++ b/src/input.bif @@ -45,3 +45,6 @@ const unset_field: string; module InputRaw; const record_separator: string; + +module InputBenchmark; +const factor: count; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index ce79ecfd39..f0b4f8e7e9 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -58,7 +58,12 @@ public: name(name), num_vals(num_vals), val(val) {} virtual bool Process() { - return input_mgr->SendEvent(name, num_vals, val); + bool success = input_mgr->SendEvent(name, num_vals, val); + + if ( !success ) + reporter->Error("SendEvent for event %s failed", name.c_str()); + + return true; // we do not want to die if sendEvent fails because the event did not return. } private: diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index 07ee7eb9bc..de77ba1afa 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -21,6 +21,7 @@ using threading::Field; Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) { + multiplication_factor = int(BifConst::InputBenchmark::factor); } Benchmark::~Benchmark() @@ -198,13 +199,25 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { bool Benchmark::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - + num_lines = num_lines*multiplication_factor; + switch ( mode ) { case MANUAL: // yay, we do nothing :) break; case REREAD: case STREAM: + if ( multiplication_factor != 1 ) { + // we have to document at what time we changed the factor to what value. + Value** v = new Value*[2]; + v[0] = new Value(TYPE_COUNT, true); + v[0]->val.uint_val = num_lines; + v[1] = new Value(TYPE_TIME, true); + v[1]->val.double_val = CurrTime(); + + SendEvent("lines_changed", 2, v); + } + Update(); // call update and not DoUpdate, because update actually checks disabled. break; default: diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index e8de4ac773..e0d3f124af 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -38,6 +38,8 @@ private: int mode; int num_lines; + int multiplication_factor; + string RandomString(const int len); }; From 3405cbdfbd5eb9c00dc4f782e9cc875da3c26d8f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 26 Mar 2012 19:53:01 -0700 Subject: [PATCH 104/149] Introducing - the check if a thread queue might have data. Without locks. Who needs those anyways. --- src/threading/Manager.cc | 6 ++++++ src/threading/MsgThread.h | 6 ++++++ src/threading/Queue.h | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index db86caa26f..6a539861be 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -82,6 +82,12 @@ double Manager::NextTimestamp(double* network_time) // If we had something to process last time (or out heartbeat // is due), we want to check for more asap. return timer_mgr->Time(); + + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + { + if ( (*i)->MightHaveOut() ) + return timer_mgr->Time(); + } return -1.0; } diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 28c7690dfa..4220230a71 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -261,6 +261,12 @@ private: */ bool HasOut() { return queue_out.Ready(); } + /** + * Returns true if there might be at least one message pending for the main + * thread. + */ + bool MightHaveOut() { return queue_out.MaybeReady(); } + Queue queue_in; Queue queue_out; diff --git a/src/threading/Queue.h b/src/threading/Queue.h index a25f897d23..64d6e7cd93 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -53,6 +53,11 @@ public: */ bool Ready(); + /** + * Returns true if the next Get() operation might succeed. + */ + bool MaybeReady() { return ( ( read_ptr - write_ptr) != 0 ); } + /** * Returns the number of queued items not yet retrieved. */ From 579a10d060241f99275e72505230eb642ee2cc47 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 27 Mar 2012 09:18:01 -0700 Subject: [PATCH 105/149] make benchmark reader more configureable --- .../frameworks/input/readers/benchmark.bro | 8 ++++- src/input.bif | 4 ++- src/input/readers/Benchmark.cc | 30 +++++++++++++++---- src/input/readers/Benchmark.h | 5 +++- 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro index 3293201cea..c6a6e88fca 100644 --- a/scripts/base/frameworks/input/readers/benchmark.bro +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -4,5 +4,11 @@ module InputBenchmark; export { ## multiplication factor for each second - const factor = 1 &redef; + const factor = 1.0 &redef; + + ## spread factor between lines + const spread = 0 &redef; + + ## spreading where usleep = 1000000 / autospread * num_lines + const autospread = 0.0 &redef; } diff --git a/src/input.bif b/src/input.bif index e4ecf4d020..059a7ec8bf 100644 --- a/src/input.bif +++ b/src/input.bif @@ -47,4 +47,6 @@ module InputRaw; const record_separator: string; module InputBenchmark; -const factor: count; +const factor: double; +const spread: count; +const autospread: double; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index de77ba1afa..d8de8c2538 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -21,7 +21,11 @@ using threading::Field; Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) { - multiplication_factor = int(BifConst::InputBenchmark::factor); + multiplication_factor = double(BifConst::InputBenchmark::factor); + autospread = double(BifConst::InputBenchmark::autospread); + spread = int(BifConst::InputBenchmark::spread); + autospread_time = 0; + } Benchmark::~Benchmark() @@ -40,6 +44,9 @@ bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Fiel num_fields = arg_num_fields; fields = arg_fields; num_lines = atoi(path.c_str()); + + if ( autospread != 0.0 ) + autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); @@ -85,15 +92,21 @@ bool Benchmark::DoUpdate() { if ( mode == STREAM ) { // do not do tracking, spread out elements over the second that we have... Put(field); - usleep(900000/num_lines); } else { SendEntry(field); } + + if ( spread != 0 ) + usleep(spread); + + if ( autospread_time != 0 ) { + usleep( autospread_time ); + } } - //if ( mode != STREAM ) { // well, does not really make sense in the streaming sense - but I like getting the event. + if ( mode != STREAM ) { EndCurrentSend(); - //} + } return true; } @@ -199,7 +212,7 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { bool Benchmark::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - num_lines = num_lines*multiplication_factor; + num_lines = (int) ( (double) num_lines*multiplication_factor); switch ( mode ) { case MANUAL: @@ -217,8 +230,15 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) SendEvent("lines_changed", 2, v); } + + if ( autospread != 0.0 ) { + autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); + // because executing this in every loop is apparently too expensive. + } Update(); // call update and not DoUpdate, because update actually checks disabled. + + SendEvent("HeartbeatDone", 0, 0); break; default: assert(false); diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index e0d3f124af..f0bd0c752d 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -38,7 +38,10 @@ private: int mode; int num_lines; - int multiplication_factor; + double multiplication_factor; + int spread; + double autospread; + int autospread_time; string RandomString(const int len); From ed5374b6d7b4d4601d8e8ee11f547692ebd3fffc Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 28 Mar 2012 09:35:45 -0700 Subject: [PATCH 106/149] and another option for the benchmark reader (constant addition of lines) --- scripts/base/frameworks/input/readers/benchmark.bro | 3 +++ src/input.bif | 1 + src/input/readers/Benchmark.cc | 13 +++++++++++++ src/input/readers/Benchmark.h | 1 + 4 files changed, 18 insertions(+) diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro index c6a6e88fca..0f3553b117 100644 --- a/scripts/base/frameworks/input/readers/benchmark.bro +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -11,4 +11,7 @@ export { ## spreading where usleep = 1000000 / autospread * num_lines const autospread = 0.0 &redef; + + ## addition factor for each heartbeat + const addfactor = 0 &redef; } diff --git a/src/input.bif b/src/input.bif index 059a7ec8bf..798759ab66 100644 --- a/src/input.bif +++ b/src/input.bif @@ -50,3 +50,4 @@ module InputBenchmark; const factor: double; const spread: count; const autospread: double; +const addfactor: count; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index d8de8c2538..a17c8a7ff6 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -24,6 +24,7 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) multiplication_factor = double(BifConst::InputBenchmark::factor); autospread = double(BifConst::InputBenchmark::autospread); spread = int(BifConst::InputBenchmark::spread); + add = int(BifConst::InputBenchmark::addfactor); autospread_time = 0; } @@ -213,6 +214,7 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); num_lines = (int) ( (double) num_lines*multiplication_factor); + num_lines += add; switch ( mode ) { case MANUAL: @@ -230,6 +232,17 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) SendEvent("lines_changed", 2, v); } + + if ( add != 0 ) { + // we have to document at what time we changed the factor to what value. + Value** v = new Value*[2]; + v[0] = new Value(TYPE_COUNT, true); + v[0]->val.uint_val = num_lines; + v[1] = new Value(TYPE_TIME, true); + v[1]->val.double_val = CurrTime(); + + SendEvent("lines_changed", 2, v); + } if ( autospread != 0.0 ) { autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index f0bd0c752d..182adcd1af 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -42,6 +42,7 @@ private: int spread; double autospread; int autospread_time; + int add; string RandomString(const int len); From 719540414f6bcf80bd1efee9d6dcfce0b2ba7e7e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 28 Mar 2012 09:41:00 -0700 Subject: [PATCH 107/149] repair general stupidity --- src/input/readers/Benchmark.cc | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index a17c8a7ff6..a914f1a2e8 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -222,7 +222,7 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) break; case REREAD: case STREAM: - if ( multiplication_factor != 1 ) { + if ( multiplication_factor != 1 || add != 0 ) { // we have to document at what time we changed the factor to what value. Value** v = new Value*[2]; v[0] = new Value(TYPE_COUNT, true); @@ -233,17 +233,6 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) SendEvent("lines_changed", 2, v); } - if ( add != 0 ) { - // we have to document at what time we changed the factor to what value. - Value** v = new Value*[2]; - v[0] = new Value(TYPE_COUNT, true); - v[0]->val.uint_val = num_lines; - v[1] = new Value(TYPE_TIME, true); - v[1]->val.double_val = CurrTime(); - - SendEvent("lines_changed", 2, v); - } - if ( autospread != 0.0 ) { autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); // because executing this in every loop is apparently too expensive. From b47620e501944acddde0277fd2e9c7ab4de6cfec Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 30 Mar 2012 09:18:44 -0700 Subject: [PATCH 108/149] add a couple more configuration options --- .../frameworks/input/readers/benchmark.bro | 6 +++++ src/input.bif | 2 ++ src/input/readers/Benchmark.cc | 23 +++++++++++++++---- src/input/readers/Benchmark.h | 3 +++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro index 0f3553b117..b5adc70861 100644 --- a/scripts/base/frameworks/input/readers/benchmark.bro +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -14,4 +14,10 @@ export { ## addition factor for each heartbeat const addfactor = 0 &redef; + + ## stop spreading at x lines per heartbeat + const stopspreadat = 0 &redef; + + ## 1 -> enable timed spreading + const timedspread = 0 &redef; } diff --git a/src/input.bif b/src/input.bif index 798759ab66..63cbb2796d 100644 --- a/src/input.bif +++ b/src/input.bif @@ -51,3 +51,5 @@ const factor: double; const spread: count; const autospread: double; const addfactor: count; +const stopspreadat: count; +const timedspread: count; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index a914f1a2e8..391fdd7435 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -26,6 +26,8 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) spread = int(BifConst::InputBenchmark::spread); add = int(BifConst::InputBenchmark::addfactor); autospread_time = 0; + stopspreadat = int(BifConst::InputBenchmark::stopspreadat); + timedspread = int(BifConst::InputBenchmark::timedspread); } @@ -54,6 +56,7 @@ bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Fiel return false; } + heartbeatstarttime = CurrTime(); DoUpdate(); return true; @@ -97,12 +100,23 @@ bool Benchmark::DoUpdate() { SendEntry(field); } - if ( spread != 0 ) - usleep(spread); + if ( stopspreadat == 0 || num_lines < stopspreadat ) { + if ( spread != 0 ) + usleep(spread); - if ( autospread_time != 0 ) { - usleep( autospread_time ); + if ( autospread_time != 0 ) + usleep( autospread_time ); } + + if ( timedspread == 1 ) { + double diff; + do { + diff = CurrTime() - heartbeatstarttime; + //printf("%d %f\n", i, diff); + } while ( diff < i/(num_lines + (num_lines * 0.15) ) ); + //} while ( diff < 0.8); + } + } if ( mode != STREAM ) { @@ -215,6 +229,7 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) ReaderBackend::DoHeartbeat(network_time, current_time); num_lines = (int) ( (double) num_lines*multiplication_factor); num_lines += add; + heartbeatstarttime = CurrTime(); switch ( mode ) { case MANUAL: diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index 182adcd1af..e5dca66889 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -43,6 +43,9 @@ private: double autospread; int autospread_time; int add; + int stopspreadat; + double heartbeatstarttime; + int timedspread; string RandomString(const int len); From 1170a877693f01d03963227c5d3a2f1aeeec53e1 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 28 Mar 2012 15:37:32 -0700 Subject: [PATCH 109/149] make benchmark reader hartbeat inverval aware fix small memleak on tablereader destruction make timespread better configureable --- .../frameworks/input/readers/benchmark.bro | 2 +- src/input.bif | 2 +- src/input/Manager.cc | 11 ++++++---- src/input/readers/Benchmark.cc | 21 +++++++++++++++---- src/input/readers/Benchmark.h | 2 +- src/threading/Manager.h | 8 ++++++- 6 files changed, 34 insertions(+), 12 deletions(-) diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro index b5adc70861..fe44914271 100644 --- a/scripts/base/frameworks/input/readers/benchmark.bro +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -19,5 +19,5 @@ export { const stopspreadat = 0 &redef; ## 1 -> enable timed spreading - const timedspread = 0 &redef; + const timedspread = 0.0 &redef; } diff --git a/src/input.bif b/src/input.bif index 63cbb2796d..0749ac0287 100644 --- a/src/input.bif +++ b/src/input.bif @@ -52,4 +52,4 @@ const spread: count; const autospread: double; const addfactor: count; const stopspreadat: count; -const timedspread: count; +const timedspread: double; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index f8ad493e11..218a9209ee 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -133,11 +133,15 @@ Manager::TableFilter::~TableFilter() { if ( rtype ) // can be 0 for sets Unref(rtype); - if ( currDict != 0 ) + if ( currDict != 0 ) { + currDict->Clear(); delete currDict; + } - if ( lastDict != 0 ) + if ( lastDict != 0 ) { + lastDict->Clear();; delete lastDict; + } } struct ReaderDefinition { @@ -898,6 +902,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { } //i->tab->Assign(idxval, valval); + assert(idxval); HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); @@ -1067,8 +1072,6 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) { } int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const *vals) { - bool updated = false; - assert(i); assert(i->filter_type == EVENT_FILTER); diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index 391fdd7435..118b57f616 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -13,6 +13,8 @@ #include #include +#include "../../threading/Manager.h" + using namespace input::reader; using threading::Value; using threading::Field; @@ -27,7 +29,7 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) add = int(BifConst::InputBenchmark::addfactor); autospread_time = 0; stopspreadat = int(BifConst::InputBenchmark::stopspreadat); - timedspread = int(BifConst::InputBenchmark::timedspread); + timedspread = double(BifConst::InputBenchmark::timedspread); } @@ -87,7 +89,8 @@ double Benchmark::CurrTime() { // read the entire file and send appropriate thingies back to InputMgr bool Benchmark::DoUpdate() { - for ( int i = 0; i < num_lines; i++ ) { + int linestosend = num_lines * threading::Manager::HEART_BEAT_INTERVAL; + for ( int i = 0; i < linestosend; i++ ) { Value** field = new Value*[num_fields]; for (unsigned int j = 0; j < num_fields; j++ ) { field[j] = EntryToVal(fields[j]->type, fields[j]->subtype); @@ -108,12 +111,13 @@ bool Benchmark::DoUpdate() { usleep( autospread_time ); } - if ( timedspread == 1 ) { + if ( timedspread != 0.0 ) { double diff; do { diff = CurrTime() - heartbeatstarttime; //printf("%d %f\n", i, diff); - } while ( diff < i/(num_lines + (num_lines * 0.15) ) ); + //} while ( diff < i/threading::Manager::HEART_BEAT_INTERVAL*(num_lines + (num_lines * timedspread) ) ); + } while ( diff/threading::Manager::HEART_BEAT_INTERVAL < i/(linestosend + (linestosend * timedspread) ) ); //} while ( diff < 0.8); } @@ -226,6 +230,15 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { bool Benchmark::DoHeartbeat(double network_time, double current_time) { + /* + * This does not work the way I envisioned it, because the queueing is the problem. + printf("%f\n", CurrTime() - current_time); + if ( CurrTime() - current_time > 0.25 ) { + // event has hung for a time. refuse. + SendEvent("EndBenchmark", 0, 0); + return true; + } */ + ReaderBackend::DoHeartbeat(network_time, current_time); num_lines = (int) ( (double) num_lines*multiplication_factor); num_lines += add; diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index e5dca66889..ca248586da 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -45,7 +45,7 @@ private: int add; int stopspreadat; double heartbeatstarttime; - int timedspread; + double timedspread; string RandomString(const int len); diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 7d9ba766d4..d5d78b288a 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -9,6 +9,10 @@ #include "BasicThread.h" #include "MsgThread.h" +namespace input { namespace reader { + class Benchmark; +}} + namespace threading { /** @@ -80,6 +84,7 @@ public: protected: friend class BasicThread; friend class MsgThread; + friend class input::reader::Benchmark; // needs heartbeat /** * Registers a new basic thread with the manager. This is @@ -118,9 +123,10 @@ protected: * Part of the IOSource interface. */ virtual const char* Tag() { return "threading::Manager"; } + + static const int HEART_BEAT_INTERVAL = 10; private: - static const int HEART_BEAT_INTERVAL = 1; typedef std::list all_thread_list; all_thread_list all_threads; From 7a71a74994a1271384e6fdc5b6c54e82e1a6761c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 28 Mar 2012 16:31:11 -0700 Subject: [PATCH 110/149] fix largest leak in manager. --- src/input/Manager.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 218a9209ee..ed59900608 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1663,10 +1663,11 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { position += CopyValue(data, position, val); } - hash_t key = HashKey::HashBytes(data, length); + HashKey *key = new HashKey(data, length); + delete data; assert(position == length); - return new HashKey(data, length, key, true); + return key; } // convert threading value to Bro value From b7bbda724404273e38865b02b8e6c4dc767cdc51 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 28 Mar 2012 23:18:40 -0700 Subject: [PATCH 111/149] fix a couple more leaks. But - still leaking quite a lot with tables. --- src/input/Manager.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index ed59900608..009fdb0bbb 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -25,9 +25,15 @@ using threading::Field; struct InputHash { hash_t valhash; - HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. + HashKey* idxkey; + ~InputHash(); }; +InputHash::~InputHash() { + if ( idxkey ) + delete idxkey; +} + declare(PDict, InputHash); class Manager::Filter { @@ -821,6 +827,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { // ok, exact duplicate filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); + delete idxhash; return filter->num_val_fields + filter->num_idx_fields; } else { assert( filter->num_val_fields > 0 ); @@ -855,7 +862,6 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { //Val* predidx = ListValToRecordVal(idxval->AsListVal(), filter->itype, &startpos); predidx = ValueToRecordVal(vals, filter->itype, &startpos); //ValueToRecordVal(vals, filter->itype, &startpos); - Ref(valval); if ( updated ) { ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); @@ -865,7 +871,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { bool result; if ( filter->num_val_fields > 0 ) { // we have values - result = CallPred(filter->pred, 3, ev, predidx->Ref(), valval); + result = CallPred(filter->pred, 3, ev, predidx->Ref(), valval->Ref()); } else { // no values result = CallPred(filter->pred, 2, ev, predidx->Ref()); @@ -876,10 +882,12 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... delete(filter->currDict->RemoveEntry(idxhash)); + delete idxhash; return filter->num_val_fields + filter->num_idx_fields; } else { // keep old one filter->currDict->Insert(idxhash, h); + delete idxhash; return filter->num_val_fields + filter->num_idx_fields; } } @@ -916,8 +924,10 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { if ( filter->event && updated ) Ref(oldval); // otherwise it is no longer accessible after the assignment filter->tab->Assign(idxval, k, valval); + Unref(idxval); // asssign does not consume idxval. filter->currDict->Insert(idxhash, ih); + delete idxhash; if ( filter->event ) { EnumVal* ev; @@ -931,12 +941,11 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - Ref(valval); if ( filter->num_val_fields == 0 ) { Ref(filter->description); SendEvent(filter->event, 3, filter->description->Ref(), ev, predidx); } else { - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval->Ref()); } } } From 8e526a7f835f514cf864c6bda172485e35b0ac60 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 29 Mar 2012 01:09:11 -0700 Subject: [PATCH 112/149] fix memory leak for tables... nearly completely. There is still a tiny where I have not yet found where the delete could be missing. For big table imports the memory footprint is significant nevertheless -- with tables of > 200000 entries, memory consumption can apparently reach in excess of 1.5Gb - and on a first glance this seems legitimate. (The reason for this is probably that we use several hash tables to keep the performance impact small). --- src/input/Manager.cc | 45 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 009fdb0bbb..a1a3410f5e 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -34,6 +34,11 @@ InputHash::~InputHash() { delete idxkey; } +static void input_hash_delete_func(void* val) { + InputHash* h = (InputHash*) val; + delete h; +} + declare(PDict, InputHash); class Manager::Filter { @@ -170,6 +175,14 @@ Manager::Manager() { } +Manager::~Manager() { + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { + delete s->second; + delete s->first; + } + +} + ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) { ReaderDefinition* ir = input_readers; @@ -527,7 +540,9 @@ bool Manager::CreateTableStream(RecordVal* fval) { filter->itype = idx->AsRecordType(); filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter->currDict = new PDict(InputHash); + filter->currDict->SetDeleteFunc(input_hash_delete_func); filter->lastDict = new PDict(InputHash); + filter->lastDict->SetDeleteFunc(input_hash_delete_func); filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault @@ -820,20 +835,20 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { } } - InputHash *h = filter->lastDict->Lookup(idxhash); + InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before if ( filter->num_val_fields == 0 || h->valhash == valhash ) { - // ok, exact duplicate + // ok, exact duplicate, move entry to new dicrionary and do nothing else. filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); delete idxhash; return filter->num_val_fields + filter->num_idx_fields; } else { assert( filter->num_val_fields > 0 ); - // updated + // entry was updated in some way filter->lastDict->Remove(idxhash); - delete(h); + // keep h for predicates updated = true; } @@ -881,8 +896,10 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { Unref(predidx); if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... - delete(filter->currDict->RemoveEntry(idxhash)); + // (but why should it be in there? assert this). + assert ( filter->currDict->RemoveEntry(idxhash) == 0 ); delete idxhash; + delete h; return filter->num_val_fields + filter->num_idx_fields; } else { // keep old one @@ -893,6 +910,12 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { } } + + // now we don't need h anymore - if we are here, the entry is updated and a new h is created. + if ( h ) { + delete h; + h = 0; + } Val* idxval; @@ -1014,6 +1037,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { Unref(predidx); Unref(ev); filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); + delete lastDictIdxKey; continue; } } @@ -1030,8 +1054,9 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { if ( ev ) Unref(ev); - filter->tab->Delete(ih->idxkey); - filter->lastDict->Remove(lastDictIdxKey); // deletex in next line + Unref(filter->tab->Delete(ih->idxkey)); + filter->lastDict->Remove(lastDictIdxKey); // delete in next line + delete lastDictIdxKey; delete(ih); } @@ -1040,6 +1065,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { filter->lastDict = filter->currDict; filter->currDict = new PDict(InputHash); + filter->currDict->SetDeleteFunc(input_hash_delete_func); #ifdef DEBUG DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s, queueing update_finished event", @@ -1284,9 +1310,12 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { // only if filter = true -> no filtering if ( filterresult ) { - success = ( filter->tab->Delete(idxval) != 0 ); + Val* retptr = filter->tab->Delete(idxval); + success = ( retptr != 0 ); if ( !success ) { reporter->Error("Internal error while deleting values from input table"); + } else { + Unref(retptr); } } } else if ( i->filter_type == EVENT_FILTER ) { From 384fc730d472a69d921b18fa57a6db559a8faedd Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 1 Apr 2012 17:13:51 -0700 Subject: [PATCH 113/149] fix heart_beat_interval -- initialization in constructor does not work anymore (probably due to change in init ordering?) --- src/threading/Manager.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index ec7ab34d14..6c14fd65ca 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -60,6 +60,12 @@ void Manager::KillThreads() void Manager::AddThread(BasicThread* thread) { + if ( heart_beat_interval == 0 ) { + // sometimes initialization does not seem to work from constructor + heart_beat_interval = double(BifConst::Threading::heart_beat_interval); + DBG_LOG(DBG_THREADING, "Heart beat interval set to %f", heart_beat_interval); + } + DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str()); all_threads.push_back(thread); idle = false; From 25affe2c826c8ba93069685d6b64f68612c971e3 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 3 Apr 2012 00:52:41 +0200 Subject: [PATCH 114/149] fix missing get call for heart beat in benchmark reader. --- src/input/readers/Benchmark.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index f0cebd2dc1..9bba7d7831 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -30,6 +30,7 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) autospread_time = 0; stopspreadat = int(BifConst::InputBenchmark::stopspreadat); timedspread = double(BifConst::InputBenchmark::timedspread); + heart_beat_interval = double(BifConst::Threading::heart_beat_interval); } From a5cc98bb5d189dcc9ad04516b6a583bae4ea1508 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 10 Apr 2012 13:57:09 -0700 Subject: [PATCH 115/149] fix memory leak in tables and vectors that are read into tables --- src/input/Manager.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index a1a3410f5e..102fd78d6f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1795,8 +1795,12 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); for ( int i = 0; i < val->val.set_val.size; i++ ) { - t->Assign(ValueToVal( val->val.set_val.vals[i], type ), 0); + Val* assignval = ValueToVal( val->val.set_val.vals[i], type ); + t->Assign(assignval, 0); + Unref(assignval); // idex is not consumed by assign. } + + Unref(s); return t; break; } @@ -1809,6 +1813,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { for ( int i = 0; i < val->val.vector_val.size; i++ ) { v->Assign(i, ValueToVal( val->val.set_val.vals[i], type ), 0); } + Unref(vt); return v; } From 1967f6f81caec93c1d85c851e3229a9197fe5cf1 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 15 Apr 2012 15:11:39 -0700 Subject: [PATCH 116/149] rename a couple of structures and make the names in manager fit the api more. This should it make easier for other people to understand what is going on without having knowledge of an "internal api * means * in external api" mapping. --- src/input/Manager.cc | 98 ++++++++++++++++++++++++-------------------- src/input/Manager.h | 24 +++++------ 2 files changed, 65 insertions(+), 57 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 102fd78d6f..8c8d6d8ba3 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -23,6 +23,11 @@ using namespace input; using threading::Value; using threading::Field; +/** + * InputHashes are used as Dictionaries to store the value and index hashes for all lines currently stored in a table. + * Index hash is stored as HashKey*, because it is thrown into other Bro functions that need the complex structure of it. + * For everything we do (with values), we just take the hash_t value and compare it directly with == + */ struct InputHash { hash_t valhash; HashKey* idxkey; @@ -41,7 +46,10 @@ static void input_hash_delete_func(void* val) { declare(PDict, InputHash); -class Manager::Filter { +/** + * Base stuff that every stream can do + */ +class Manager::Stream { public: string name; string source; @@ -49,25 +57,25 @@ public: int mode; - FilterType filter_type; // to distinguish between event and table filters + StreamType filter_type; // to distinguish between event and table filters EnumVal* type; ReaderFrontend* reader; RecordVal* description; - Filter(); - virtual ~Filter(); + Stream(); + virtual ~Stream(); }; -Manager::Filter::Filter() { +Manager::Stream::Stream() { type = 0; reader = 0; description = 0; removed = false; } -Manager::Filter::~Filter() { +Manager::Stream::~Stream() { if ( type ) Unref(type); if ( description ) @@ -77,7 +85,7 @@ Manager::Filter::~Filter() { delete(reader); } -class Manager::TableFilter: public Manager::Filter { +class Manager::TableStream: public Manager::Stream { public: unsigned int num_idx_fields; @@ -96,11 +104,11 @@ public: EventHandlerPtr event; - TableFilter(); - ~TableFilter(); + TableStream(); + ~TableStream(); }; -class Manager::EventFilter: public Manager::Filter { +class Manager::EventStream: public Manager::Stream { public: EventHandlerPtr event; @@ -108,11 +116,11 @@ public: unsigned int num_fields; bool want_record; - EventFilter(); - ~EventFilter(); + EventStream(); + ~EventStream(); }; -Manager::TableFilter::TableFilter() : Manager::Filter::Filter() { +Manager::TableStream::TableStream() : Manager::Stream::Stream() { filter_type = TABLE_FILTER; tab = 0; @@ -125,18 +133,18 @@ Manager::TableFilter::TableFilter() : Manager::Filter::Filter() { pred = 0; } -Manager::EventFilter::EventFilter() : Manager::Filter::Filter() { +Manager::EventStream::EventStream() : Manager::Stream::Stream() { fields = 0; filter_type = EVENT_FILTER; } -Manager::EventFilter::~EventFilter() { +Manager::EventStream::~EventStream() { if ( fields ) { Unref(fields); } } -Manager::TableFilter::~TableFilter() { +Manager::TableStream::~TableStream() { if ( tab ) Unref(tab); if ( itype ) @@ -176,7 +184,7 @@ Manager::Manager() } Manager::~Manager() { - for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { delete s->second; delete s->first; } @@ -233,7 +241,7 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) } // create a new input reader object to be used at whomevers leisure lateron. -bool Manager::CreateStream(Filter* info, RecordVal* description) +bool Manager::CreateStream(Stream* info, RecordVal* description) { ReaderDefinition* ir = input_readers; @@ -249,7 +257,7 @@ bool Manager::CreateStream(Filter* info, RecordVal* description) Unref(name_val); { - Filter *i = FindFilter(name); + Stream *i = FindStream(name); if ( i != 0 ) { reporter->Error("Trying create already existing input stream %s", name.c_str()); return false; @@ -296,7 +304,7 @@ bool Manager::CreateEventStream(RecordVal* fval) { return false; } - EventFilter* filter = new EventFilter(); + EventStream* filter = new EventStream(); { bool res = CreateStream(filter, fval); if ( res == false ) { @@ -412,7 +420,7 @@ bool Manager::CreateTableStream(RecordVal* fval) { return false; } - TableFilter* filter = new TableFilter(); + TableStream* filter = new TableStream(); { bool res = CreateStream(filter, fval); if ( res == false ) { @@ -620,7 +628,7 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) bool Manager::RemoveStream(const string &name) { - Filter *i = FindFilter(name); + Stream *i = FindStream(name); if ( i == 0 ) { return false; // not found @@ -644,7 +652,7 @@ bool Manager::RemoveStream(const string &name) { } bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->Error("Stream not found in RemoveStreamContinuation"); @@ -712,7 +720,7 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co bool Manager::ForceUpdate(const string &name) { - Filter *i = FindFilter(name); + Stream *i = FindStream(name); if ( i == 0 ) { reporter->Error("Stream %s not found", name.c_str()); return false; @@ -786,7 +794,7 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader in SendEntry"); return; @@ -797,7 +805,7 @@ void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { readFields = SendEntryTable(i, vals); } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - readFields = SendEventFilterEvent(i, type, vals); + readFields = SendEventStreamEvent(i, type, vals); } else { assert(false); } @@ -808,13 +816,13 @@ void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { delete [] vals; } -int Manager::SendEntryTable(Filter* i, const Value* const *vals) { +int Manager::SendEntryTable(Stream* i, const Value* const *vals) { bool updated = false; assert(i); assert(i->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i; + TableStream* filter = (TableStream*) i; HashKey* idxhash = HashValues(filter->num_idx_fields, vals); @@ -979,7 +987,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { void Manager::EndCurrentSend(ReaderFrontend* reader) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader in EndCurrentSend"); return; @@ -996,7 +1004,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { } assert(i->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i; + TableStream* filter = (TableStream*) i; // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = filter->lastDict->InitForIteration(); @@ -1083,7 +1091,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { } void Manager::Put(ReaderFrontend* reader, Value* *vals) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader in Put"); return; @@ -1094,7 +1102,7 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) { readFields = PutTable(i, vals); } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - readFields = SendEventFilterEvent(i, type, vals); + readFields = SendEventStreamEvent(i, type, vals); } else { assert(false); } @@ -1106,11 +1114,11 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) { } -int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const *vals) { +int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) { assert(i); assert(i->filter_type == EVENT_FILTER); - EventFilter* filter = (EventFilter*) i; + EventStream* filter = (EventStream*) i; Val *val; list out_vals; @@ -1143,11 +1151,11 @@ int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const * } -int Manager::PutTable(Filter* i, const Value* const *vals) { +int Manager::PutTable(Stream* i, const Value* const *vals) { assert(i); assert(i->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i; + TableStream* filter = (TableStream*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; @@ -1244,7 +1252,7 @@ int Manager::PutTable(Filter* i, const Value* const *vals) { // Todo:: perhaps throw some kind of clear-event? void Manager::Clear(ReaderFrontend* reader) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader in Clear"); return; @@ -1256,14 +1264,14 @@ void Manager::Clear(ReaderFrontend* reader) { #endif assert(i->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i; + TableStream* filter = (TableStream*) i; filter->tab->RemoveAll(); } // put interface: delete old entry from table. bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader in Delete"); return false; @@ -1273,7 +1281,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { int readVals = 0; if ( i->filter_type == TABLE_FILTER ) { - TableFilter* filter = (TableFilter*) i; + TableStream* filter = (TableStream*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); assert(idxval != 0); readVals = filter->num_idx_fields + filter->num_val_fields; @@ -1320,7 +1328,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { } } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - readVals = SendEventFilterEvent(i, type, vals); + readVals = SendEventStreamEvent(i, type, vals); success = true; } else { assert(false); @@ -1840,9 +1848,9 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { return NULL; } -Manager::Filter* Manager::FindFilter(const string &name) +Manager::Stream* Manager::FindStream(const string &name) { - for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { if ( (*s).second->name == name ) { @@ -1853,9 +1861,9 @@ Manager::Filter* Manager::FindFilter(const string &name) return 0; } -Manager::Filter* Manager::FindFilter(ReaderFrontend* reader) +Manager::Stream* Manager::FindStream(ReaderFrontend* reader) { - map::iterator s = readers.find(reader); + map::iterator s = readers.find(reader); if ( s != readers.end() ) { return s->second; } diff --git a/src/input/Manager.h b/src/input/Manager.h index c6dd40bd95..8f09828988 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -107,25 +107,25 @@ protected: // Functions are called from the ReaderBackend to notify the manager, that a filter has been removed // or a stream has been closed. // Used to prevent race conditions where data for a specific filter is still in the queue when the - // RemoveFilter directive is executed by the main thread. + // RemoveStream directive is executed by the main thread. // This makes sure all data that has ben queued for a filter is still received. bool RemoveStreamContinuation(ReaderFrontend* reader); private: - class Filter; - class TableFilter; - class EventFilter; + class Stream; + class TableStream; + class EventStream; - bool CreateStream(Filter*, RecordVal* description); + bool CreateStream(Stream*, RecordVal* description); // SendEntry implementation for Tablefilter - int SendEntryTable(Filter* i, const threading::Value* const *vals); + int SendEntryTable(Stream* i, const threading::Value* const *vals); // Put implementation for Tablefilter - int PutTable(Filter* i, const threading::Value* const *vals); + int PutTable(Stream* i, const threading::Value* const *vals); // SendEntry and Put implementation for Eventfilter - int SendEventFilterEvent(Filter* i, EnumVal* type, const threading::Value* const *vals); + int SendEventStreamEvent(Stream* i, EnumVal* type, const threading::Value* const *vals); // Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types // from the log framework @@ -163,12 +163,12 @@ private: // Converts a Bro ListVal to a RecordVal given the record type RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); - Filter* FindFilter(const string &name); - Filter* FindFilter(ReaderFrontend* reader); + Stream* FindStream(const string &name); + Stream* FindStream(ReaderFrontend* reader); - enum FilterType { TABLE_FILTER, EVENT_FILTER }; + enum StreamType { TABLE_FILTER, EVENT_FILTER }; - map readers; + map readers; }; From 48e05621c082983c8eb103499a4151cd320eedbd Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 16 Apr 2012 14:49:24 -0700 Subject: [PATCH 117/149] update description to current interface. But this needs to get longer... --- doc/input.rst | 128 +++++++++++++++++++++----------------------------- 1 file changed, 54 insertions(+), 74 deletions(-) diff --git a/doc/input.rst b/doc/input.rst index e201af9fed..d9fe8aa6b8 100644 --- a/doc/input.rst +++ b/doc/input.rst @@ -20,11 +20,9 @@ very similar to the abstracts used in the logging framework: Input Streams An input stream corresponds to a single input source (usually a textfile). It defined the information necessary - to find the source (e.g. the filename) - - Filters - Each input stream has a set of filters attached to it, that - determine exaclty what kind of information is read. + to find the source (e.g. the filename), the reader that it used + to get data from it (see below). + It also defines exactly what data is read from the input source. There are two different kind of streams, event streams and table streams. By default, event streams generate an event for each line read @@ -41,28 +39,37 @@ very similar to the abstracts used in the logging framework: one event per line. -Basics -====== +Event Streams +============= For examples, please look at the unit tests in ``testing/btest/scripts/base/frameworks/input/``. -A very basic example to open an input stream is: +Event Streams are streams that generate an event for each line in of the input source. + +For example, a simple stream retrieving the fields ``i`` and ``b`` from an inputSource +could be defined as follows: .. code:: bro + + type Val: record { + i: int; + b: bool; + }; + + event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool) { + # work with event data + } - module Foo; - - export { - # Create an ID for our new stream - redef enum Input::ID += { INPUT }; + event bro_init { + Input::add_event([$source="input.log", $name="input", $fields=Val, $ev=line]); } - event bro_init() { - Input::create_stream(FOO::INPUT, [$source="input.log"]); - } +The fields that can be set for an event stream are: -The fields that can be set when creating a stream are: + ``want_record`` + Boolean value, that defines if the event wants to receive the fields inside of + a single record value, or individually (default). ``source`` A mandatory string identifying the source of the data. @@ -81,49 +88,9 @@ The fields that can be set when creating a stream are: ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new data is added to the file. - ``autostart`` - If set to yes, the first update operation is triggered automatically after the first filter has been added to the stream. - This has to be set to false if several filters are added to the input source. - In this case Input::force_update has to be called manually once after all filters have been added. - -Filters -======= - -Each filter defines the data fields that it wants to receive from the respective -input file. Depending on the type of filter, events or a table are created from -the data in the source file. - -Event Filters -------------- - -Event filters are filters that generate an event for each line in of the input source. - -For example, a simple filter retrieving the fields ``i`` and ``b`` from an inputSource -could be defined as follows: - -.. code:: bro - - type Val: record { - i: int; - b: bool; - }; - - event line(tpe: Input::Event, i: int, b: bool) { - # work with event data - } - - event bro_init { - # Input stream definition, etc - ... - - Input::add_eventfilter(Foo::INPUT, [$name="input", $fields=Val, $ev=line]); - } - -The fields that can be set for an event filter are: - ``name`` - A mandatory name for the filter that can later be used - to manipulate it further. + A mandatory name for the stream that can later be used + to remove it. ``fields`` Name of a record type containing the fields, which should be retrieved from @@ -138,16 +105,14 @@ The fields that can be set for an event filter are: been ``CHANGED`` or ``DELETED``. Singe the ascii reader cannot track this information for event filters, the value is always ``NEW`` at the moment. - ``want_record`` - Boolean value, that defines if the event wants to receive the fields inside of - a single record value, or individually (default). -Table Filters -------------- -Table filters are the second, more complex type of filter. +Table Streams +============= -Table filters store the information they read from an input source in a bro table. For example, +Table streams are the second, more complex type of input streams. + +Table streams store the information they read from an input source in a bro table. For example, when reading a file that contains ip addresses and connection attemt information one could use an approach similar to this: @@ -164,18 +129,33 @@ an approach similar to this: global conn_attempts: table[addr] of count = table(); event bro_init { - # Input stream definitions, etc. - ... - - Input::add_tablefilter(Foo::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=conn_attempts]); - - # read the file after all filters have been set (only needed if autostart is set to false) - Input::force_update(Foo::INPUT); + Input::add_table([$source="input.txt", $name="input", $idx=Idx, $val=Val, $destination=conn_attempts]); } The table conn_attempts will then contain the information about connection attemps. -The possible fields that can be set for an table filter are: +The possible fields that can be set for an table stream are: + + ``want_record`` + Boolean value, that defines if the event wants to receive the fields inside of + a single record value, or individually (default). + + ``source`` + A mandatory string identifying the source of the data. + For the ASCII reader this is the filename. + + ``reader`` + The reader used for this stream. Default is ``READER_ASCII``. + + ``mode`` + The mode in which the stream is opened. Possible values are ``MANUAL``, ``REREAD`` and ``STREAM``. + Default is ``MANUAL``. + ``MANUAL`` means, that the files is not updated after it has been read. Changes to the file will not + be reflected in the data bro knows. + ``REREAD`` means that the whole file is read again each time a change is found. This should be used for + files that are mapped to a table where individual lines can change. + ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new + data is added to the file. ``name`` A mandatory name for the filter that can later be used From 1e66fe905a948fcded4b0ba13c11e907831c835e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 9 May 2012 15:08:36 -0500 Subject: [PATCH 118/149] Add support to Bro for connecting with peers over IPv6. - Communication::listen_ipv6 needs to be redef'd to true in order for IPv6 listening sockets to be opened. - Added Communication::listen_retry option as an interval at which to retry binding to socket addresses that were already in use. - Added some explicit baselines to check in the istate.events and istate.events-ssl tests -- the SSL test was incorrectly passing because it compared two empty files. (The files being empty because "http/base" was given as an argument to Bro which it couldn't handle because that script doesn't exist anymore). --- .../base/frameworks/communication/main.bro | 13 +- scripts/base/utils/addrs.bro | 15 + .../frameworks/communication/listen.bro | 3 +- src/IPAddr.h | 22 +- src/RemoteSerializer.cc | 362 +++++++++++------- src/RemoteSerializer.h | 27 +- src/bro.bif | 9 +- .../Baseline/istate.bro-ipv6/recv..stdout | 1 + .../Baseline/istate.bro-ipv6/send..stdout | 2 + .../Baseline/istate.events-ssl/events.rec.log | 33 ++ .../Baseline/istate.events-ssl/events.snd.log | 33 ++ .../istate.events-ssl/receiver.http.log | 2 +- .../istate.events-ssl/sender.http.log | 2 +- .../Baseline/istate.events/events.rec.log | 33 ++ .../Baseline/istate.events/events.snd.log | 33 ++ .../Baseline/istate.events/receiver.http.log | 2 +- .../Baseline/istate.events/sender.http.log | 2 +- testing/btest/istate/bro-ipv6.bro | 52 +++ testing/btest/istate/events-ssl.bro | 6 +- testing/btest/istate/events.bro | 2 + 20 files changed, 480 insertions(+), 174 deletions(-) create mode 100644 testing/btest/Baseline/istate.bro-ipv6/recv..stdout create mode 100644 testing/btest/Baseline/istate.bro-ipv6/send..stdout create mode 100644 testing/btest/Baseline/istate.events-ssl/events.rec.log create mode 100644 testing/btest/Baseline/istate.events-ssl/events.snd.log create mode 100644 testing/btest/Baseline/istate.events/events.rec.log create mode 100644 testing/btest/Baseline/istate.events/events.snd.log create mode 100644 testing/btest/istate/bro-ipv6.bro diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro index 04772f57aa..26ec9f41b8 100644 --- a/scripts/base/frameworks/communication/main.bro +++ b/scripts/base/frameworks/communication/main.bro @@ -2,6 +2,7 @@ ##! and/or transfer events. @load base/frameworks/packet-filter +@load base/utils/addrs module Communication; @@ -10,7 +11,7 @@ export { ## The communication logging stream identifier. redef enum Log::ID += { LOG }; - ## Which interface to listen on (0.0.0.0 for any interface). + ## Which interface to listen on (``0.0.0.0`` or ``[::]`` are wildcards). const listen_interface = 0.0.0.0 &redef; ## Which port to listen on. @@ -19,6 +20,14 @@ export { ## This defines if a listening socket should use SSL. const listen_ssl = F &redef; + ## Defines if a listening socket can bind to IPv6 addresses. + const listen_ipv6 = F &redef; + + ## Defines the interval at which to retry binding to + ## :bro:id:`listen_interface` on :bro:id:`listen_port` if it's already in + ## use. + const listen_retry = 30 secs &redef; + ## Default compression level. Compression level is 0-9, with 0 = no ## compression. global compression_level = 0 &redef; @@ -160,7 +169,7 @@ event remote_log(level: count, src: count, msg: string) # This is a core generated event. event remote_log_peer(p: event_peer, level: count, src: count, msg: string) { - local rmsg = fmt("[#%d/%s:%d] %s", p$id, p$host, p$p, msg); + local rmsg = fmt("[#%d/%s:%d] %s", p$id, addr_to_uri(p$host), p$p, msg); do_script_log_common(level, src, rmsg); } diff --git a/scripts/base/utils/addrs.bro b/scripts/base/utils/addrs.bro index 415b9adfa9..08efd5281a 100644 --- a/scripts/base/utils/addrs.bro +++ b/scripts/base/utils/addrs.bro @@ -98,3 +98,18 @@ function find_ip_addresses(input: string): string_array } return output; } + +## Returns the string representation of an IP address suitable for inclusion +## in a URI. For IPv4, this does no special formatting, but for IPv6, the +## address is included in square brackets. +## +## a: the address to make suitable for URI inclusion. +## +## Returns: the string representation of *a* suitable for URI inclusion. +function addr_to_uri(a: addr): string + { + if ( is_v4_addr(a) ) + return fmt("%s", a); + else + return fmt("[%s]", a); + } diff --git a/scripts/policy/frameworks/communication/listen.bro b/scripts/policy/frameworks/communication/listen.bro index e366e5b4ff..609e8c91d6 100644 --- a/scripts/policy/frameworks/communication/listen.bro +++ b/scripts/policy/frameworks/communication/listen.bro @@ -8,5 +8,6 @@ module Communication; event bro_init() &priority=-10 { enable_communication(); - listen(listen_interface, listen_port, listen_ssl); + listen(listen_interface, listen_port, listen_ssl, listen_ipv6, + listen_retry); } diff --git a/src/IPAddr.h b/src/IPAddr.h index 8e1921e07b..447669d422 100644 --- a/src/IPAddr.h +++ b/src/IPAddr.h @@ -188,11 +188,16 @@ public: * IPv4 to IPv6 address mapping to return a full 16 bytes. * * @param bytes The pointer to a memory location in which the - * raw bytes of the address are to be copied in network byte-order. + * raw bytes of the address are to be copied. + * + * @param order The byte-order in which the returned raw bytes are copied. + * The default is network order. */ - void CopyIPv6(uint32_t* bytes) const + void CopyIPv6(uint32_t* bytes, ByteOrder order = Network) const { memcpy(bytes, in6.s6_addr, sizeof(in6.s6_addr)); + if ( order == Host ) + for ( unsigned int i = 0; i < 4; ++i ) bytes[i] = ntohl(bytes[i]); } /** @@ -280,6 +285,19 @@ public: */ string AsString() const; + /** + * Returns a string representation of the address suitable for inclusion + * in an URI. For IPv4 addresses, this is the same as AsString(), but + * IPv6 addresses are encased in square brackets. + */ + string AsURIString() const + { + if ( GetFamily() == IPv4 ) + return AsString(); + else + return string("[") + AsString() + "]"; + } + /** * Returns a host-order, plain hex string representation of the address. */ diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 61be8a9e8f..3abec00f59 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -147,6 +147,7 @@ #include #include +#include #include #include #include @@ -195,7 +196,7 @@ extern "C" { // Gets incremented each time there's an incompatible change // to the communication internals. -static const unsigned short PROTOCOL_VERSION = 0x07; +static const unsigned short PROTOCOL_VERSION = 0x08; static const char MSG_NONE = 0x00; static const char MSG_VERSION = 0x01; @@ -458,17 +459,6 @@ static inline char* fmt_uint32s(int nargs, va_list ap) } #endif - -static inline const char* ip2a(uint32 ip) - { - static char buffer[32]; - struct in_addr addr; - - addr.s_addr = htonl(ip); - - return bro_inet_ntop(AF_INET, &addr, buffer, 32); - } - static pid_t child_pid = 0; // Return true if message type is sent by a peer (rather than the child @@ -683,24 +673,20 @@ RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip, if ( ! initialized ) reporter->InternalError("remote serializer not initialized"); - if ( ip.GetFamily() == IPv6 ) - Error("inter-Bro communication not supported over IPv6"); - - const uint32* bytes; - ip.GetBytes(&bytes); - uint32 ip4 = ntohl(*bytes); - if ( ! child_pid ) Fork(); - Peer* p = AddPeer(ip4, port); + Peer* p = AddPeer(ip, port); p->orig = true; if ( our_class ) p->our_class = our_class; - if ( ! SendToChild(MSG_CONNECT_TO, p, 5, p->id, - ip4, port, uint32(retry), use_ssl) ) + uint32 bytes[4]; + ip.CopyIPv6(bytes, IPAddr::Host); + + if ( ! SendToChild(MSG_CONNECT_TO, p, 8, p->id, bytes[0], bytes[1], + bytes[2], bytes[3], port, uint32(retry), use_ssl) ) { RemovePeer(p); return false; @@ -1232,7 +1218,8 @@ bool RemoteSerializer::SendCapabilities(Peer* peer) return caps ? SendToChild(MSG_CAPS, peer, 3, caps, 0, 0) : true; } -bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl) +bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl, + bool ipv6, double retry) { if ( ! using_communication ) return true; @@ -1240,14 +1227,15 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl) if ( ! initialized ) reporter->InternalError("remote serializer not initialized"); - if ( ip.GetFamily() == IPv6 ) - Error("inter-Bro communication not supported over IPv6"); + if ( ! ipv6 && ip.GetFamily() == IPv6 && + ip != IPAddr("0.0.0.0") && ip != IPAddr("::") ) + reporter->FatalError("Attempt to listen on address %s, but IPv6 communication disabled", ip.AsString().c_str()); - const uint32* bytes; - ip.GetBytes(&bytes); - uint32 ip4 = ntohl(*bytes); + uint32 bytes[4]; + ip.CopyIPv6(bytes, IPAddr::Host); - if ( ! SendToChild(MSG_LISTEN, 0, 3, ip4, port, expect_ssl) ) + if ( ! SendToChild(MSG_LISTEN, 0, 8, bytes[0], bytes[1], bytes[2], bytes[3], + port, expect_ssl, ipv6, (uint32) retry) ) return false; listening = true; @@ -1784,7 +1772,7 @@ RecordVal* RemoteSerializer::MakePeerVal(Peer* peer) RecordVal* v = new RecordVal(::peer); v->Assign(0, new Val(uint32(peer->id), TYPE_COUNT)); // Sic! Network order for AddrVal, host order for PortVal. - v->Assign(1, new AddrVal(htonl(peer->ip))); + v->Assign(1, new AddrVal(peer->ip)); v->Assign(2, new PortVal(peer->port, TRANSPORT_TCP)); v->Assign(3, new Val(false, TYPE_BOOL)); v->Assign(4, new StringVal("")); // set when received @@ -1793,8 +1781,8 @@ RecordVal* RemoteSerializer::MakePeerVal(Peer* peer) return v; } -RemoteSerializer::Peer* RemoteSerializer::AddPeer(uint32 ip, uint16 port, - PeerID id) +RemoteSerializer::Peer* RemoteSerializer::AddPeer(const IPAddr& ip, uint16 port, + PeerID id) { Peer* peer = new Peer; peer->id = id != PEER_NONE ? id : id_counter++; @@ -1960,8 +1948,8 @@ bool RemoteSerializer::ProcessConnected() { // IP and port follow. uint32* args = (uint32*) current_args->data; - uint32 host = ntohl(args[0]); // ### Fix: only works for IPv4 - uint16 port = (uint16) ntohl(args[1]); + IPAddr host = IPAddr(IPv6, args, IPAddr::Network); + uint16 port = (uint16) ntohl(args[4]); if ( ! current_peer ) { @@ -2980,7 +2968,8 @@ void RemoteSerializer::Log(LogLevel level, const char* msg, Peer* peer, if ( peer ) len += snprintf(buffer + len, sizeof(buffer) - len, "[#%d/%s:%d] ", - int(peer->id), ip2a(peer->ip), peer->port); + int(peer->id), peer->ip.AsURIString().c_str(), + peer->port); len += safe_snprintf(buffer + len, sizeof(buffer) - len, "%s", msg); @@ -3266,8 +3255,10 @@ SocketComm::SocketComm() terminating = false; killing = false; - listen_fd_clear = -1; - listen_fd_ssl = -1; + listen_port = 0; + listen_ssl = false; + enable_ipv6 = false; + bind_retry_interval = 0; listen_next_try = 0; // We don't want to use the signal handlers of our parent. @@ -3290,8 +3281,7 @@ SocketComm::~SocketComm() delete peers[i]->io; delete io; - close(listen_fd_clear); - close(listen_fd_ssl); + CloseListenFDs(); } static unsigned int first_rtime = 0; @@ -3340,20 +3330,13 @@ void SocketComm::Run() } if ( listen_next_try && time(0) > listen_next_try ) - Listen(listen_if, listen_port, listen_ssl); + Listen(); - if ( listen_fd_clear >= 0 ) + for ( size_t i = 0; i < listen_fds.size(); ++i ) { - FD_SET(listen_fd_clear, &fd_read); - if ( listen_fd_clear > max_fd ) - max_fd = listen_fd_clear; - } - - if ( listen_fd_ssl >= 0 ) - { - FD_SET(listen_fd_ssl, &fd_read); - if ( listen_fd_ssl > max_fd ) - max_fd = listen_fd_ssl; + FD_SET(listen_fds[i], &fd_read); + if ( listen_fds[i] > max_fd ) + max_fd = listen_fds[i]; } if ( io->IsFillingUp() && ! shutting_conns_down ) @@ -3442,12 +3425,9 @@ void SocketComm::Run() } } - if ( listen_fd_clear >= 0 && - FD_ISSET(listen_fd_clear, &fd_read) ) - AcceptConnection(listen_fd_clear); - - if ( listen_fd_ssl >= 0 && FD_ISSET(listen_fd_ssl, &fd_read) ) - AcceptConnection(listen_fd_ssl); + for ( size_t i = 0; i < listen_fds.size(); ++i ) + if ( FD_ISSET(listen_fds[i], &fd_read) ) + AcceptConnection(listen_fds[i]); // Hack to display CPU usage of the child, triggered via // SIGPROF. @@ -3571,13 +3551,8 @@ bool SocketComm::DoParentMessage() case MSG_LISTEN_STOP: { - if ( listen_fd_ssl >= 0 ) - close(listen_fd_ssl); + CloseListenFDs(); - if ( listen_fd_clear >= 0 ) - close(listen_fd_clear); - - listen_fd_clear = listen_fd_ssl = -1; Log("stopped listening"); return true; @@ -3721,10 +3696,10 @@ bool SocketComm::ProcessConnectTo() Peer* peer = new Peer; peer->id = ntohl(args[0]); - peer->ip = ntohl(args[1]); - peer->port = ntohl(args[2]); - peer->retry = ntohl(args[3]); - peer->ssl = ntohl(args[4]); + peer->ip = IPAddr(IPv6, &args[1], IPAddr::Network); + peer->port = ntohl(args[5]); + peer->retry = ntohl(args[6]); + peer->ssl = ntohl(args[7]); return Connect(peer); } @@ -3734,11 +3709,13 @@ bool SocketComm::ProcessListen() assert(parent_args); uint32* args = (uint32*) parent_args->data; - uint32 addr = ntohl(args[0]); - uint16 port = uint16(ntohl(args[1])); - uint32 ssl = ntohl(args[2]); + listen_if = IPAddr(IPv6, args, IPAddr::Network); + listen_port = uint16(ntohl(args[4])); + listen_ssl = ntohl(args[5]) != 0; + enable_ipv6 = ntohl(args[6]) != 0; + bind_retry_interval = ntohl(args[7]); - return Listen(addr, port, ssl); + return Listen(); } bool SocketComm::ProcessParentCompress() @@ -3900,29 +3877,53 @@ bool SocketComm::ProcessPeerCompress(Peer* peer) bool SocketComm::Connect(Peer* peer) { - struct sockaddr_in server; + int status; + addrinfo hints, *res, *res0; + bzero(&hints, sizeof(hints)); - int sockfd = socket(PF_INET, SOCK_STREAM, 0); - if ( sockfd < 0 ) + hints.ai_family = PF_UNSPEC; + hints.ai_protocol = IPPROTO_TCP; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_NUMERICHOST; + + char port_str[16]; + modp_uitoa10(peer->port, port_str); + + // TODO: better to accept string arguments from the user to pass into + // getaddrinfo? This might make it easier to explicitly connect to + // non-global IPv6 addresses with a scope zone identifier (RFC 4007). + status = getaddrinfo(peer->ip.AsString().c_str(), port_str, &hints, &res0); + if ( status != 0 ) { - Error(fmt("can't create socket, %s", strerror(errno))); + Error(fmt("getaddrinfo error: %s", gai_strerror(status))); return false; } - bzero(&server, sizeof(server)); - server.sin_family = AF_INET; - server.sin_port = htons(peer->port); - server.sin_addr.s_addr = htonl(peer->ip); - - bool connected = true; - - if ( connect(sockfd, (sockaddr*) &server, sizeof(server)) < 0 ) + int sockfd = -1; + for ( res = res0; res; res = res->ai_next ) { - Error(fmt("connect failed: %s", strerror(errno)), peer); - close(sockfd); - connected = false; + sockfd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if ( sockfd < 0 ) + { + Error(fmt("can't create connect socket, %s", strerror(errno))); + continue; + } + + if ( connect(sockfd, res->ai_addr, res->ai_addrlen) < 0 ) + { + Error(fmt("connect failed: %s", strerror(errno)), peer); + close(sockfd); + sockfd = -1; + continue; + } + + break; } + freeaddrinfo(res0); + + bool connected = sockfd != -1; + if ( ! (connected || peer->retry) ) { CloseConnection(peer, false); @@ -3947,9 +3948,7 @@ bool SocketComm::Connect(Peer* peer) if ( connected ) { if ( peer->ssl ) - { peer->io = new ChunkedIOSSL(sockfd, false); - } else peer->io = new ChunkedIOFd(sockfd, "child->peer"); @@ -3964,7 +3963,12 @@ bool SocketComm::Connect(Peer* peer) if ( connected ) { Log("connected", peer); - if ( ! SendToParent(MSG_CONNECTED, peer, 2, peer->ip, peer->port) ) + + uint32 bytes[4]; + peer->ip.CopyIPv6(bytes, IPAddr::Host); + + if ( ! SendToParent(MSG_CONNECTED, peer, 5, bytes[0], bytes[1], + bytes[2], bytes[3], peer->port) ) return false; } @@ -4001,86 +4005,139 @@ bool SocketComm::CloseConnection(Peer* peer, bool reconnect) return true; } -bool SocketComm::Listen(uint32 ip, uint16 port, bool expect_ssl) +bool SocketComm::Listen() { - int* listen_fd = expect_ssl ? &listen_fd_ssl : &listen_fd_clear; + int status, on = 1; + addrinfo hints, *res, *res0; + bzero(&hints, sizeof(hints)); - if ( *listen_fd >= 0 ) - close(*listen_fd); - - struct sockaddr_in server; - - *listen_fd = socket(PF_INET, SOCK_STREAM, 0); - if ( *listen_fd < 0 ) + if ( enable_ipv6 ) { - Error(fmt("can't create listen socket, %s", - strerror(errno))); + if ( listen_if == IPAddr("0.0.0.0") || listen_if == IPAddr("::") ) + hints.ai_family = PF_UNSPEC; + else + hints.ai_family = listen_if.GetFamily() == IPv4 ? PF_INET : PF_INET6; + } + else + hints.ai_family = PF_INET; + + hints.ai_protocol = IPPROTO_TCP; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG | AI_NUMERICHOST; + + char port_str[16]; + modp_uitoa10(listen_port, port_str); + + const char* addr_str = 0; + if ( listen_if != IPAddr("0.0.0.0") && listen_if != IPAddr("::") ) + addr_str = listen_if.AsString().c_str(); + + CloseListenFDs(); + + // TODO: better to accept string arguments from the user to pass into + // getaddrinfo? This might make it easier to explicitly bind to a + // non-global IPv6 address with a scope zone identifier (RFC 4007). + if ( (status = getaddrinfo(addr_str, port_str, &hints, &res0)) != 0 ) + { + Error(fmt("getaddrinfo error: %s", gai_strerror(status))); return false; } - // Set SO_REUSEADDR. - int turn_on = 1; - if ( setsockopt(*listen_fd, SOL_SOCKET, SO_REUSEADDR, - &turn_on, sizeof(turn_on)) < 0 ) + for ( res = res0; res; res = res->ai_next ) { - Error(fmt("can't set SO_REUSEADDR, %s", - strerror(errno))); - return false; - } - - bzero(&server, sizeof(server)); - server.sin_family = AF_INET; - server.sin_port = htons(port); - server.sin_addr.s_addr = htonl(ip); - - if ( bind(*listen_fd, (sockaddr*) &server, sizeof(server)) < 0 ) - { - Error(fmt("can't bind to port %d, %s", port, strerror(errno))); - close(*listen_fd); - *listen_fd = -1; - - if ( errno == EADDRINUSE ) + if ( res->ai_family != AF_INET && res->ai_family != AF_INET6 ) { - listen_if = ip; - listen_port = port; - listen_ssl = expect_ssl; - // FIXME: Make this timeout configurable. - listen_next_try = time(0) + 30; + Error(fmt("can't create listen socket: unknown address family, %d", + res->ai_family)); + continue; } - return false; + + IPAddr a = res->ai_family == AF_INET ? + IPAddr(((sockaddr_in*)res->ai_addr)->sin_addr) : + IPAddr(((sockaddr_in6*)res->ai_addr)->sin6_addr); + + int fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if ( fd < 0 ) + { + Error(fmt("can't create listen socket, %s", strerror(errno))); + continue; + } + + if ( setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) < 0 ) + Error(fmt("can't set SO_REUSEADDR, %s", strerror(errno))); + + // For IPv6 listening sockets, we don't want do dual binding to also + // get IPv4-mapped addresses because that's not as portable. e.g. + // many BSDs don't allow that. + if ( res->ai_family == AF_INET6 && + setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof(on)) < 0 ) + Error(fmt("can't set IPV6_V6ONLY, %s", strerror(errno))); + + if ( bind(fd, res->ai_addr, res->ai_addrlen) < 0 ) + { + Error(fmt("can't bind to %s:%s, %s", a.AsURIString().c_str(), + port_str, strerror(errno))); + close(fd); + + if ( errno == EADDRINUSE ) + { + // Abandon completely this attempt to set up listening sockets, + // try again later. + CloseListenFDs(); + listen_next_try = time(0) + bind_retry_interval; + return false; + } + continue; + } + + if ( listen(fd, 50) < 0 ) + { + Error(fmt("can't listen on %s:%s, %s", a.AsURIString().c_str(), + port_str, strerror(errno))); + close(fd); + continue; + } + + listen_fds.push_back(fd); + Log(fmt("listening on %s:%s (%s)", a.AsURIString().c_str(), port_str, + listen_ssl ? "ssl" : "clear")); } - if ( listen(*listen_fd, 50) < 0 ) - { - Error(fmt("can't listen, %s", strerror(errno))); - return false; - } + freeaddrinfo(res0); listen_next_try = 0; - Log(fmt("listening on %s:%d (%s)", - ip2a(ip), port, expect_ssl ? "ssl" : "clear")); - return true; + return listen_fds.size() > 0; } bool SocketComm::AcceptConnection(int fd) { - sockaddr_in client; + sockaddr_storage client; socklen_t len = sizeof(client); int clientfd = accept(fd, (sockaddr*) &client, &len); if ( clientfd < 0 ) { - Error(fmt("accept failed, %s %d", - strerror(errno), errno)); + Error(fmt("accept failed, %s %d", strerror(errno), errno)); + return false; + } + + if ( client.ss_family != AF_INET && client.ss_family != AF_INET6 ) + { + Error(fmt("accept fail, unknown address family %d", client.ss_family)); + close(clientfd); return false; } Peer* peer = new Peer; peer->id = id_counter++; - peer->ip = ntohl(client.sin_addr.s_addr); - peer->port = ntohs(client.sin_port); + peer->ip = client.ss_family == AF_INET ? + IPAddr(((sockaddr_in*)&client)->sin_addr) : + IPAddr(((sockaddr_in6*)&client)->sin6_addr); + peer->port = client.ss_family == AF_INET ? + ntohs(((sockaddr_in*)&client)->sin_port) : + ntohs(((sockaddr_in6*)&client)->sin6_port); peer->connected = true; - peer->ssl = (fd == listen_fd_ssl); + peer->ssl = listen_ssl; peer->compressor = false; if ( peer->ssl ) @@ -4090,8 +4147,7 @@ bool SocketComm::AcceptConnection(int fd) if ( ! peer->io->Init() ) { - Error(fmt("can't init peer io: %s", - peer->io->Error()), false); + Error(fmt("can't init peer io: %s", peer->io->Error()), false); return false; } @@ -4099,7 +4155,11 @@ bool SocketComm::AcceptConnection(int fd) Log(fmt("accepted %s connection", peer->ssl ? "SSL" : "clear"), peer); - if ( ! SendToParent(MSG_CONNECTED, peer, 2, peer->ip, peer->port) ) + uint32 bytes[4]; + peer->ip.CopyIPv6(bytes, IPAddr::Host); + + if ( ! SendToParent(MSG_CONNECTED, peer, 5, bytes[0], bytes[1], bytes[2], + bytes[3], peer->port) ) return false; return true; @@ -4117,12 +4177,19 @@ const char* SocketComm::MakeLogString(const char* msg, Peer* peer) if ( peer ) len = snprintf(buffer, BUFSIZE, "[#%d/%s:%d] ", int(peer->id), - ip2a(peer->ip), peer->port); + peer->ip.AsURIString().c_str(), peer->port); len += safe_snprintf(buffer + len, BUFSIZE - len, "%s", msg); return buffer; } +void SocketComm::CloseListenFDs() + { + for ( size_t i = 0; i < listen_fds.size(); ++i ) + close(listen_fds[i]); + listen_fds.clear(); + } + void SocketComm::Error(const char* msg, bool kill_me) { if ( kill_me ) @@ -4165,7 +4232,7 @@ void SocketComm::Log(const char* msg, Peer* peer) void SocketComm::InternalError(const char* msg) { - fprintf(stderr, "interal error in child: %s\n", msg); + fprintf(stderr, "internal error in child: %s\n", msg); Kill(); } @@ -4180,8 +4247,7 @@ void SocketComm::Kill() LogProf(); Log("terminating"); - close(listen_fd_clear); - close(listen_fd_ssl); + CloseListenFDs(); kill(getpid(), SIGTERM); diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index 05d25ca525..f6f94f53d3 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -10,8 +10,7 @@ #include "Stats.h" #include "File.h" -// All IP arguments are in host byte-order. -// FIXME: Change this to network byte order +#include class IncrementalSendTimer; @@ -63,7 +62,8 @@ public: bool CompleteHandshake(PeerID peer); // Start to listen. - bool Listen(const IPAddr& ip, uint16 port, bool expect_ssl); + bool Listen(const IPAddr& ip, uint16 port, bool expect_ssl, bool ipv6, + double retry); // Stop it. bool StopListening(); @@ -179,9 +179,7 @@ protected: struct Peer { PeerID id; // Unique ID (non-zero) per peer. - // ### Fix: currently, we only work for IPv4. - // addr_type ip; - uint32 ip; + IPAddr ip; uint16 port; handler_list handlers; @@ -277,7 +275,7 @@ protected: bool ProcessLogWrite(); bool ProcessRequestLogs(); - Peer* AddPeer(uint32 ip, uint16 port, PeerID id = PEER_NONE); + Peer* AddPeer(const IPAddr& ip, uint16 port, PeerID id = PEER_NONE); Peer* LookupPeer(PeerID id, bool only_if_connected); void RemovePeer(Peer* peer); bool IsConnectedPeer(PeerID id); @@ -412,7 +410,6 @@ protected: { id = 0; io = 0; - ip = 0; port = 0; state = 0; connected = false; @@ -424,7 +421,7 @@ protected: RemoteSerializer::PeerID id; ChunkedIO* io; - uint32 ip; + IPAddr ip; uint16 port; char state; bool connected; @@ -437,7 +434,7 @@ protected: bool compressor; }; - bool Listen(uint32 ip, uint16 port, bool expect_ssl); + bool Listen(); bool AcceptConnection(int listen_fd); bool Connect(Peer* peer); bool CloseConnection(Peer* peer, bool reconnect); @@ -482,6 +479,9 @@ protected: bool ForwardChunkToPeer(); const char* MakeLogString(const char* msg, Peer *peer); + // Closes all file descriptors associated with listening sockets. + void CloseListenFDs(); + // Peers we are communicating with: declare(PList, Peer); typedef PList(Peer) peer_list; @@ -498,14 +498,15 @@ protected: char parent_msgtype; ChunkedIO::Chunk* parent_args; - int listen_fd_clear; - int listen_fd_ssl; + vector listen_fds; // If the port we're trying to bind to is already in use, we will retry // it regularly. - uint32 listen_if; // Fix: only supports IPv4 + IPAddr listen_if; uint16 listen_port; bool listen_ssl; + bool enable_ipv6; // allow IPv6 listen sockets + uint32 bind_retry_interval; time_t listen_next_try; bool shutting_conns_down; bool terminating; diff --git a/src/bro.bif b/src/bro.bif index 15740a83c7..3f4215dc13 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -5402,12 +5402,17 @@ function set_compression_level%(p: event_peer, level: count%) : bool ## ## ssl: If true, Bro uses SSL to encrypt the session. ## +## ipv6: If true, enable listening on IPv6 addresses. +## +## retry_interval: If address *ip* is found to be already in use, this is +## the interval at which to automatically retry binding. +## ## Returns: True on success. ## ## .. bro:see:: connect disconnect -function listen%(ip: addr, p: port, ssl: bool %) : bool +function listen%(ip: addr, p: port, ssl: bool, ipv6: bool, retry_interval: interval%) : bool %{ - return new Val(remote_serializer->Listen(ip->AsAddr(), p->Port(), ssl), TYPE_BOOL); + return new Val(remote_serializer->Listen(ip->AsAddr(), p->Port(), ssl, ipv6, retry_interval), TYPE_BOOL); %} ## Checks whether the last raised event came from a remote peer. diff --git a/testing/btest/Baseline/istate.bro-ipv6/recv..stdout b/testing/btest/Baseline/istate.bro-ipv6/recv..stdout new file mode 100644 index 0000000000..673af68234 --- /dev/null +++ b/testing/btest/Baseline/istate.bro-ipv6/recv..stdout @@ -0,0 +1 @@ +handshake done with peer: ::1 diff --git a/testing/btest/Baseline/istate.bro-ipv6/send..stdout b/testing/btest/Baseline/istate.bro-ipv6/send..stdout new file mode 100644 index 0000000000..fbc855464d --- /dev/null +++ b/testing/btest/Baseline/istate.bro-ipv6/send..stdout @@ -0,0 +1,2 @@ +handshake done with peer: ::1 +my_event: hello world diff --git a/testing/btest/Baseline/istate.events-ssl/events.rec.log b/testing/btest/Baseline/istate.events-ssl/events.rec.log new file mode 100644 index 0000000000..04993fb84a --- /dev/null +++ b/testing/btest/Baseline/istate.events-ssl/events.rec.log @@ -0,0 +1,33 @@ +http_request +http_begin_entity +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_end_entity +http_message_done +http_signature_found +http_reply +http_begin_entity +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_end_entity +http_message_done diff --git a/testing/btest/Baseline/istate.events-ssl/events.snd.log b/testing/btest/Baseline/istate.events-ssl/events.snd.log new file mode 100644 index 0000000000..04993fb84a --- /dev/null +++ b/testing/btest/Baseline/istate.events-ssl/events.snd.log @@ -0,0 +1,33 @@ +http_request +http_begin_entity +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_end_entity +http_message_done +http_signature_found +http_reply +http_begin_entity +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_end_entity +http_message_done diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index 1601f8ad3c..5a7912d23d 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -5,4 +5,4 @@ #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324314406.995958 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - +1336588614.060989 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index 1601f8ad3c..5a7912d23d 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -5,4 +5,4 @@ #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324314406.995958 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - +1336588614.060989 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/events.rec.log b/testing/btest/Baseline/istate.events/events.rec.log new file mode 100644 index 0000000000..04993fb84a --- /dev/null +++ b/testing/btest/Baseline/istate.events/events.rec.log @@ -0,0 +1,33 @@ +http_request +http_begin_entity +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_end_entity +http_message_done +http_signature_found +http_reply +http_begin_entity +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_end_entity +http_message_done diff --git a/testing/btest/Baseline/istate.events/events.snd.log b/testing/btest/Baseline/istate.events/events.snd.log new file mode 100644 index 0000000000..04993fb84a --- /dev/null +++ b/testing/btest/Baseline/istate.events/events.snd.log @@ -0,0 +1,33 @@ +http_request +http_begin_entity +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_end_entity +http_message_done +http_signature_found +http_reply +http_begin_entity +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_header +http_all_headers +http_content_type +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_entity_data +http_end_entity +http_message_done diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index 25a7f289c0..55a0189cec 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -5,4 +5,4 @@ #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324314415.616486 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - +1336587178.164598 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index 25a7f289c0..55a0189cec 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -5,4 +5,4 @@ #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324314415.616486 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - +1336587178.164598 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/istate/bro-ipv6.bro b/testing/btest/istate/bro-ipv6.bro new file mode 100644 index 0000000000..6230018636 --- /dev/null +++ b/testing/btest/istate/bro-ipv6.bro @@ -0,0 +1,52 @@ +# @TEST-GROUP: comm +# +# @TEST-REQUIRES: ifconfig | grep -q "inet6 ::1" +# +# @TEST-EXEC: btest-bg-run recv bro -b ../recv.bro +# @TEST-EXEC: btest-bg-run send bro -b ../send.bro +# @TEST-EXEC: btest-bg-wait -k 20 +# +# @TEST-EXEC: btest-diff recv/.stdout +# @TEST-EXEC: btest-diff send/.stdout + +@TEST-START-FILE send.bro + +@load base/frameworks/communication + +redef Communication::nodes += { + ["foo"] = [$host=[::1], $connect=T, $events=/my_event/] +}; + +global my_event: event(s: string); + +event remote_connection_handshake_done(p: event_peer) + { + print fmt("handshake done with peer: %s", p$host); + } + +event my_event(s: string) + { + print fmt("my_event: %s", s); + terminate(); + } + +@TEST-END-FILE + +############# + +@TEST-START-FILE recv.bro + +@load frameworks/communication/listen + +redef Communication::listen_ipv6=T; + +global my_event: event(s: string); + +event remote_connection_handshake_done(p: event_peer) + { + print fmt("handshake done with peer: %s", p$host); + event my_event("hello world"); + terminate(); + } + +@TEST-END-FILE diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro index 25aa2dc8fb..c86087df81 100644 --- a/testing/btest/istate/events-ssl.bro +++ b/testing/btest/istate/events-ssl.bro @@ -8,8 +8,10 @@ # @TEST-EXEC: btest-diff receiver/http.log # @TEST-EXEC: cmp sender/http.log receiver/http.log # -# @TEST-EXEC: bro -x sender/events.bst http/base | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.snd.log -# @TEST-EXEC: bro -x receiver/events.bst http/base | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.rec.log +# @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.snd.log +# @TEST-EXEC: bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.rec.log +# @TEST-EXEC: btest-diff events.rec.log +# @TEST-EXEC: btest-diff events.snd.log # @TEST-EXEC: cmp events.rec.log events.snd.log # # We don't compare the transmitted event paramerters anymore. With the dynamic diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro index 81a1d765db..6d8227c810 100644 --- a/testing/btest/istate/events.bro +++ b/testing/btest/istate/events.bro @@ -10,6 +10,8 @@ # # @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.snd.log # @TEST-EXEC: bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' >events.rec.log +# @TEST-EXEC: btest-diff events.rec.log +# @TEST-EXEC: btest-diff events.snd.log # @TEST-EXEC: cmp events.rec.log events.snd.log # # We don't compare the transmitted event paramerters anymore. With the dynamic From 8bb62eaaa2df13adf1de74081ca4f8b1dfc66423 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 9 May 2012 16:09:16 -0500 Subject: [PATCH 119/149] Undo communication protocol version bump. Looks like it wasn't necessary because no message between remote peers needed to be changed to support IPv6, just messages between Bro parent and child processes were changed. --- src/RemoteSerializer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 3abec00f59..0383977de1 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -196,7 +196,7 @@ extern "C" { // Gets incremented each time there's an incompatible change // to the communication internals. -static const unsigned short PROTOCOL_VERSION = 0x08; +static const unsigned short PROTOCOL_VERSION = 0x07; static const char MSG_NONE = 0x00; static const char MSG_VERSION = 0x01; From 2338a322882e966c9f1ce673ae3f180b153f73bd Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 10 May 2012 10:47:39 -0500 Subject: [PATCH 120/149] Remove AI_ADDRCONFIG getaddrinfo hints flag for listening sockets. Because, according to RFC 3493, that will cause getaddrinfo to overlook the ::1 loopback if there's not some other interface with a global IPv6 address. The rationale being that the flag helps prevent unnecessary AAAA lookups, but since I set AI_NUMERICHOST, lookups aren't going to happen anyway. Also update the IPv6 Bro communication test to get it to work more reliably. --- src/RemoteSerializer.cc | 2 +- .../recv..stdout | 0 .../send..stdout | 0 .../btest/istate/{bro-ipv6.bro => bro-ipv6-socket.bro} | 8 ++++++-- 4 files changed, 7 insertions(+), 3 deletions(-) rename testing/btest/Baseline/{istate.bro-ipv6 => istate.bro-ipv6-socket}/recv..stdout (100%) rename testing/btest/Baseline/{istate.bro-ipv6 => istate.bro-ipv6-socket}/send..stdout (100%) rename testing/btest/istate/{bro-ipv6.bro => bro-ipv6-socket.bro} (88%) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 0383977de1..9123e99ef4 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -4023,7 +4023,7 @@ bool SocketComm::Listen() hints.ai_protocol = IPPROTO_TCP; hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG | AI_NUMERICHOST; + hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST; char port_str[16]; modp_uitoa10(listen_port, port_str); diff --git a/testing/btest/Baseline/istate.bro-ipv6/recv..stdout b/testing/btest/Baseline/istate.bro-ipv6-socket/recv..stdout similarity index 100% rename from testing/btest/Baseline/istate.bro-ipv6/recv..stdout rename to testing/btest/Baseline/istate.bro-ipv6-socket/recv..stdout diff --git a/testing/btest/Baseline/istate.bro-ipv6/send..stdout b/testing/btest/Baseline/istate.bro-ipv6-socket/send..stdout similarity index 100% rename from testing/btest/Baseline/istate.bro-ipv6/send..stdout rename to testing/btest/Baseline/istate.bro-ipv6-socket/send..stdout diff --git a/testing/btest/istate/bro-ipv6.bro b/testing/btest/istate/bro-ipv6-socket.bro similarity index 88% rename from testing/btest/istate/bro-ipv6.bro rename to testing/btest/istate/bro-ipv6-socket.bro index 6230018636..ae77a42c54 100644 --- a/testing/btest/istate/bro-ipv6.bro +++ b/testing/btest/istate/bro-ipv6-socket.bro @@ -1,11 +1,11 @@ # @TEST-GROUP: comm # -# @TEST-REQUIRES: ifconfig | grep -q "inet6 ::1" +# @TEST-REQUIRES: ifconfig | grep -q -E "inet6 ::1|inet6 addr: ::1" # # @TEST-EXEC: btest-bg-run recv bro -b ../recv.bro # @TEST-EXEC: btest-bg-run send bro -b ../send.bro # @TEST-EXEC: btest-bg-wait -k 20 -# +# # @TEST-EXEC: btest-diff recv/.stdout # @TEST-EXEC: btest-diff send/.stdout @@ -46,6 +46,10 @@ event remote_connection_handshake_done(p: event_peer) { print fmt("handshake done with peer: %s", p$host); event my_event("hello world"); + } + +event remote_connection_closed(p: event_peer) + { terminate(); } From d3ea3127822e2c51edbba02a88dc49a591ef9d11 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 11 May 2012 17:16:57 -0500 Subject: [PATCH 121/149] Add unit tests for Broccoli SSL and Broccoli IPv6 connectivity. --- .../istate.broccoli-ipv6-socket/bro..stdout | 9 +++ .../broccoli..stdout | 6 ++ .../Baseline/istate.broccoli-ssl/bro..stdout | 9 +++ .../istate.broccoli-ssl/broccoli..stdout | 6 ++ testing/btest/istate/broccoli-ipv6-socket.bro | 10 +++ testing/btest/istate/broccoli-ipv6.bro | 8 +-- testing/btest/istate/broccoli-ssl.bro | 68 +++++++++++++++++++ 7 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 testing/btest/Baseline/istate.broccoli-ipv6-socket/bro..stdout create mode 100644 testing/btest/Baseline/istate.broccoli-ipv6-socket/broccoli..stdout create mode 100644 testing/btest/Baseline/istate.broccoli-ssl/bro..stdout create mode 100644 testing/btest/Baseline/istate.broccoli-ssl/broccoli..stdout create mode 100644 testing/btest/istate/broccoli-ipv6-socket.bro create mode 100644 testing/btest/istate/broccoli-ssl.bro diff --git a/testing/btest/Baseline/istate.broccoli-ipv6-socket/bro..stdout b/testing/btest/Baseline/istate.broccoli-ipv6-socket/bro..stdout new file mode 100644 index 0000000000..0a7bac52c5 --- /dev/null +++ b/testing/btest/Baseline/istate.broccoli-ipv6-socket/bro..stdout @@ -0,0 +1,9 @@ +handshake done with peer +bro_addr(1.2.3.4) +bro_subnet(10.0.0.0/16) +bro_addr(2607:f8b0:4009:802::1014) +bro_subnet(2607:f8b0::/32) +broccoli_addr(1.2.3.4) +broccoli_subnet(10.0.0.0/16) +broccoli_addr(2607:f8b0:4009:802::1014) +broccoli_subnet(2607:f8b0::/32) diff --git a/testing/btest/Baseline/istate.broccoli-ipv6-socket/broccoli..stdout b/testing/btest/Baseline/istate.broccoli-ipv6-socket/broccoli..stdout new file mode 100644 index 0000000000..dba9318891 --- /dev/null +++ b/testing/btest/Baseline/istate.broccoli-ipv6-socket/broccoli..stdout @@ -0,0 +1,6 @@ +Connected to Bro instance at: ::1:47757 +Received bro_addr(1.2.3.4) +Received bro_subnet(10.0.0.0/16) +Received bro_addr(2607:f8b0:4009:802::1014) +Received bro_subnet(2607:f8b0::/32) +Terminating diff --git a/testing/btest/Baseline/istate.broccoli-ssl/bro..stdout b/testing/btest/Baseline/istate.broccoli-ssl/bro..stdout new file mode 100644 index 0000000000..0a7bac52c5 --- /dev/null +++ b/testing/btest/Baseline/istate.broccoli-ssl/bro..stdout @@ -0,0 +1,9 @@ +handshake done with peer +bro_addr(1.2.3.4) +bro_subnet(10.0.0.0/16) +bro_addr(2607:f8b0:4009:802::1014) +bro_subnet(2607:f8b0::/32) +broccoli_addr(1.2.3.4) +broccoli_subnet(10.0.0.0/16) +broccoli_addr(2607:f8b0:4009:802::1014) +broccoli_subnet(2607:f8b0::/32) diff --git a/testing/btest/Baseline/istate.broccoli-ssl/broccoli..stdout b/testing/btest/Baseline/istate.broccoli-ssl/broccoli..stdout new file mode 100644 index 0000000000..481778c98a --- /dev/null +++ b/testing/btest/Baseline/istate.broccoli-ssl/broccoli..stdout @@ -0,0 +1,6 @@ +Connected to Bro instance at: localhost:47757 +Received bro_addr(1.2.3.4) +Received bro_subnet(10.0.0.0/16) +Received bro_addr(2607:f8b0:4009:802::1014) +Received bro_subnet(2607:f8b0::/32) +Terminating diff --git a/testing/btest/istate/broccoli-ipv6-socket.bro b/testing/btest/istate/broccoli-ipv6-socket.bro new file mode 100644 index 0000000000..e36ac9e9f7 --- /dev/null +++ b/testing/btest/istate/broccoli-ipv6-socket.bro @@ -0,0 +1,10 @@ +# @TEST-GROUP: comm +# +# @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib +# @TEST-REQUIRES: ifconfig | grep -q -E "inet6 ::1|inet6 addr: ::1" +# +# @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro "Communication::listen_ipv6=T" +# @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broccoli-v6addrs -6 ::1 +# @TEST-EXEC: btest-bg-wait -k 20 +# @TEST-EXEC: btest-diff bro/.stdout +# @TEST-EXEC: btest-diff broccoli/.stdout diff --git a/testing/btest/istate/broccoli-ipv6.bro b/testing/btest/istate/broccoli-ipv6.bro index b7ab5bdb05..415c8bb2d2 100644 --- a/testing/btest/istate/broccoli-ipv6.bro +++ b/testing/btest/istate/broccoli-ipv6.bro @@ -2,14 +2,8 @@ # # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib # -# @TEST-EXEC: btest-bg-run bro bro %INPUT $DIST/aux/broccoli/test/broccoli-v6addrs.bro +# @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro # @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broccoli-v6addrs # @TEST-EXEC: btest-bg-wait -k 20 # @TEST-EXEC: btest-diff bro/.stdout # @TEST-EXEC: btest-diff broccoli/.stdout - -event remote_connection_closed(p: event_peer) - { - terminate(); - } - diff --git a/testing/btest/istate/broccoli-ssl.bro b/testing/btest/istate/broccoli-ssl.bro new file mode 100644 index 0000000000..61401c483a --- /dev/null +++ b/testing/btest/istate/broccoli-ssl.bro @@ -0,0 +1,68 @@ +# @TEST-GROUP: comm +# +# @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib +# +# @TEST-EXEC: chmod 600 broccoli.conf +# @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro "Communication::listen_ssl=T" "ssl_ca_certificate=../ca_cert.pem" "ssl_private_key=../bro.pem" +# @TEST-EXEC: btest-bg-run broccoli BROCCOLI_CONFIG_FILE=../broccoli.conf $BUILD/aux/broccoli/test/broccoli-v6addrs +# @TEST-EXEC: btest-bg-wait -k 20 +# @TEST-EXEC: btest-diff bro/.stdout +# @TEST-EXEC: btest-diff broccoli/.stdout + +@TEST-START-FILE broccoli.conf +/broccoli/use_ssl yes +/broccoli/ca_cert ../ca_cert.pem +/broccoli/host_cert ../bro.pem +/broccoli/host_key ../bro.pem +@TEST-END-FILE + +@TEST-START-FILE bro.pem +-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQD17FE8UVaO224Y8UL2bH1okCYxr5dVytTQ93uE5J9caGADzPZe +qYPuvtPt9ivhBtf2L9odK7unQU60v6RsO3bb9bQktQbEdh0FEjnso2UHe/nLreYn +VyLCEp9Sh1OFQnMhJNYuzNwVzWOqH/TYNy3ODueZTS4YBsRyEkpEfgeoaQIDAQAB +AoGAJ/S1Xi94+Mz+Hl9UmeUWmx6QlhIJbI7/9NPA5d6fZcwvjW6HuOmh3fBzTn5o +sq8B96Xesk6gtpQNzaA1fsBKlzDSpGRDVg2odN9vIT3jd0Dub2F47JHdFCqtMUIV +rCsO+fpGtavv1zJ/rzlJz7rx4cRP+/Gwd5YlH0q5cFuHhAECQQD9q328Ye4A7o2e +cLOhzuWUZszqdIY7ZTgDtk06F57VrjLVERrZjrtAwbs77m+ybw4pDKKU7H5inhQQ +03PU40ARAkEA+C6cCM6E4hRwuR+QyIqpNC4CzgPaKlF+VONZLYYvHEwFvx2/EPtX +zOZdE4HdJwnXBYx7+AGFeq8uHhrN2Tq62QJBAMory2JAinejqKsGF6R2SPMlm1ug +0vqziRksShBqkuSqmUjHASczYnoR7S+usMb9S8PblhgrA++FHWjrnf2lwIECQQCj ++/AfpY2J8GWW/HNm/q/UiX5S75qskZI+tsXK3bmtIdI+OIJxzxFxktj3NbyRud+4 +i92xvhebO7rmK2HOYg7pAkEA2wrwY1E237twoYXuUInv9F9kShKLQs19nup/dfmF +xfoVqYjJwidzPfgngowJZij7SoTaIBKv/fKp5Tq6xW3AEg== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICZDCCAc2gAwIBAgIJAKoxR9yFGsk8MA0GCSqGSIb3DQEBBQUAMCsxKTAnBgNV +BAMTIEJybyBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MCAXDTExMDYxNTIx +MjgxNVoYDzIxMTEwNTIyMjEyODE1WjArMSkwJwYDVQQDEyBCcm8gUm9vdCBDZXJ0 +aWZpY2F0aW9uIEF1dGhvcml0eTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA +9exRPFFWjttuGPFC9mx9aJAmMa+XVcrU0Pd7hOSfXGhgA8z2XqmD7r7T7fYr4QbX +9i/aHSu7p0FOtL+kbDt22/W0JLUGxHYdBRI57KNlB3v5y63mJ1ciwhKfUodThUJz +ISTWLszcFc1jqh/02Dctzg7nmU0uGAbEchJKRH4HqGkCAwEAAaOBjTCBijAdBgNV +HQ4EFgQU2vIsKYuGhHP8c7GeJLfWAjbKCFgwWwYDVR0jBFQwUoAU2vIsKYuGhHP8 +c7GeJLfWAjbKCFihL6QtMCsxKTAnBgNVBAMTIEJybyBSb290IENlcnRpZmljYXRp +b24gQXV0aG9yaXR5ggkAqjFH3IUayTwwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B +AQUFAAOBgQAF2oceL61dA7WxA9lxcxsA/Fccr7+J6sO+pLXoZtx5tpknEuIUebkm +UfMGAiyYIenHi8u0Sia8KrIfuCDc2dG3DYmfX7/faCEbtSx8KtNQFIs3aXr1zhsw +3sX9fLS0gp/qHoPMuhbhlvTlMFSE/Mih3KDsZEGcifzI6ooLF0YP5A== +-----END CERTIFICATE----- +@TEST-END-FILE + +@TEST-START-FILE ca_cert.pem +-----BEGIN CERTIFICATE----- +MIICZDCCAc2gAwIBAgIJAKoxR9yFGsk8MA0GCSqGSIb3DQEBBQUAMCsxKTAnBgNV +BAMTIEJybyBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MCAXDTExMDYxNTIx +MjgxNVoYDzIxMTEwNTIyMjEyODE1WjArMSkwJwYDVQQDEyBCcm8gUm9vdCBDZXJ0 +aWZpY2F0aW9uIEF1dGhvcml0eTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA +9exRPFFWjttuGPFC9mx9aJAmMa+XVcrU0Pd7hOSfXGhgA8z2XqmD7r7T7fYr4QbX +9i/aHSu7p0FOtL+kbDt22/W0JLUGxHYdBRI57KNlB3v5y63mJ1ciwhKfUodThUJz +ISTWLszcFc1jqh/02Dctzg7nmU0uGAbEchJKRH4HqGkCAwEAAaOBjTCBijAdBgNV +HQ4EFgQU2vIsKYuGhHP8c7GeJLfWAjbKCFgwWwYDVR0jBFQwUoAU2vIsKYuGhHP8 +c7GeJLfWAjbKCFihL6QtMCsxKTAnBgNVBAMTIEJybyBSb290IENlcnRpZmljYXRp +b24gQXV0aG9yaXR5ggkAqjFH3IUayTwwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B +AQUFAAOBgQAF2oceL61dA7WxA9lxcxsA/Fccr7+J6sO+pLXoZtx5tpknEuIUebkm +UfMGAiyYIenHi8u0Sia8KrIfuCDc2dG3DYmfX7/faCEbtSx8KtNQFIs3aXr1zhsw +3sX9fLS0gp/qHoPMuhbhlvTlMFSE/Mih3KDsZEGcifzI6ooLF0YP5A== +-----END CERTIFICATE----- +@TEST-END-FILE From 74f3a32321010928cf380abec3df30640382b289 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 17 May 2012 12:59:20 -0500 Subject: [PATCH 122/149] Enable Bro to communicate with peers over non-global IPv6 addresses. This usually requires specifying an additional zone identifier (see RFC 4007). The connect() and listen() BIFs have been changed to accept this zone identifier as an argument. --- scripts/base/frameworks/cluster/main.bro | 3 + .../frameworks/cluster/setup-connections.bro | 25 ++- .../base/frameworks/communication/main.bro | 16 +- scripts/base/frameworks/control/main.bro | 4 + .../frameworks/communication/listen.bro | 2 +- .../policy/frameworks/control/controller.bro | 4 +- src/RemoteSerializer.cc | 197 +++++++++++++----- src/RemoteSerializer.h | 18 +- src/bro.bif | 19 +- src/util.cc | 2 + 10 files changed, 215 insertions(+), 75 deletions(-) diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 1e89e9b2a7..766dea912f 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -77,6 +77,9 @@ export { node_type: NodeType; ## The IP address of the cluster node. ip: addr; + ## If the *ip* field is a non-global IPv6 address, this field + ## can specify a particular :rfc:`4007` ``zone_id``. + zone_id: string &default=""; ## The port to which the this local node can connect when ## establishing communication. p: port; diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index b5a0d25e1f..3d89e39f30 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -19,23 +19,26 @@ event bro_init() &priority=9 # Connections from the control node for runtime control and update events. # Every node in a cluster is eligible for control from this host. if ( n$node_type == CONTROL ) - Communication::nodes["control"] = [$host=n$ip, $connect=F, - $class="control", $events=control_events]; + Communication::nodes["control"] = [$host=n$ip, $zone_id=n$zone_id, + $connect=F, $class="control", + $events=control_events]; if ( me$node_type == MANAGER ) { if ( n$node_type == WORKER && n$manager == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, + [$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i, $events=worker2manager_events, $request_logs=T]; if ( n$node_type == PROXY && n$manager == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, + [$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i, $events=proxy2manager_events, $request_logs=T]; if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i ) - Communication::nodes["time-machine"] = [$host=nodes[i]$ip, $p=nodes[i]$p, + Communication::nodes["time-machine"] = [$host=nodes[i]$ip, + $zone_id=nodes[i]$zone_id, + $p=nodes[i]$p, $connect=T, $retry=1min, $events=tm2manager_events]; } @@ -44,7 +47,8 @@ event bro_init() &priority=9 { if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events]; + [$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i, + $sync=T, $auth=T, $events=worker2proxy_events]; # accepts connections from the previous one. # (This is not ideal for setups with many proxies) @@ -53,16 +57,18 @@ event bro_init() &priority=9 { if ( n?$proxy ) Communication::nodes[i] - = [$host=n$ip, $p=n$p, + = [$host=n$ip, $zone_id=n$zone_id, $p=n$p, $connect=T, $auth=F, $sync=T, $retry=1mins]; else if ( me?$proxy && me$proxy == i ) Communication::nodes[me$proxy] - = [$host=nodes[i]$ip, $connect=F, $auth=T, $sync=T]; + = [$host=nodes[i]$ip, $zone_id=nodes[i]$zone_id, + $connect=F, $auth=T, $sync=T]; } # Finally the manager, to send it status updates. if ( n$node_type == MANAGER && me$manager == i ) Communication::nodes["manager"] = [$host=nodes[i]$ip, + $zone_id=nodes[i]$zone_id, $p=nodes[i]$p, $connect=T, $retry=1mins, $class=node, @@ -72,6 +78,7 @@ event bro_init() &priority=9 { if ( n$node_type == MANAGER && me$manager == i ) Communication::nodes["manager"] = [$host=nodes[i]$ip, + $zone_id=nodes[i]$zone_id, $p=nodes[i]$p, $connect=T, $retry=1mins, $class=node, @@ -79,6 +86,7 @@ event bro_init() &priority=9 if ( n$node_type == PROXY && me$proxy == i ) Communication::nodes["proxy"] = [$host=nodes[i]$ip, + $zone_id=nodes[i]$zone_id, $p=nodes[i]$p, $connect=T, $retry=1mins, $sync=T, $class=node, @@ -87,6 +95,7 @@ event bro_init() &priority=9 if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i ) Communication::nodes["time-machine"] = [$host=nodes[i]$ip, + $zone_id=nodes[i]$zone_id, $p=nodes[i]$p, $connect=T, $retry=1min, diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro index 26ec9f41b8..b9b15bfd22 100644 --- a/scripts/base/frameworks/communication/main.bro +++ b/scripts/base/frameworks/communication/main.bro @@ -23,9 +23,14 @@ export { ## Defines if a listening socket can bind to IPv6 addresses. const listen_ipv6 = F &redef; + ## If :bro:id:`Communication::listen_interface` is a non-global + ## IPv6 address and requires a specific :rfc:`4007` ``zone_id``, + ## it can be specified here. + const listen_ipv6_zone_id = "" &redef; + ## Defines the interval at which to retry binding to - ## :bro:id:`listen_interface` on :bro:id:`listen_port` if it's already in - ## use. + ## :bro:id:`Communication::listen_interface` on + ## :bro:id:`Communication::listen_port` if it's already in use. const listen_retry = 30 secs &redef; ## Default compression level. Compression level is 0-9, with 0 = no @@ -60,6 +65,10 @@ export { type Node: record { ## Remote address. host: addr; + + ## If the *host* field is a non-global IPv6 address, this field + ## can specify a particular :rfc:`4007` ``zone_id``. + zone_id: string &optional; ## Port of the remote Bro communication endpoint if we are initiating ## the connection based on the :bro:id:`connect` field. @@ -187,7 +196,8 @@ function connect_peer(peer: string) p = node$p; local class = node?$class ? node$class : ""; - local id = connect(node$host, p, class, node$retry, node$ssl); + local zone_id = node?$zone_id ? node$zone_id : ""; + local id = connect(node$host, zone_id, p, class, node$retry, node$ssl); if ( id == PEER_ID_NONE ) Log::write(Communication::LOG, [$ts = network_time(), diff --git a/scripts/base/frameworks/control/main.bro b/scripts/base/frameworks/control/main.bro index 4fe8872801..63e5f639a0 100644 --- a/scripts/base/frameworks/control/main.bro +++ b/scripts/base/frameworks/control/main.bro @@ -11,6 +11,10 @@ export { ## The port of the host that will be controlled. const host_port = 0/tcp &redef; + ## If :bro:id:`Control::host` is a non-global IPv6 address and + ## requires a specific :rfc:`4007` ``zone_id``, it can be set here. + const zone_id = "" &redef; + ## The command that is being done. It's typically set on the ## command line. const cmd = "" &redef; diff --git a/scripts/policy/frameworks/communication/listen.bro b/scripts/policy/frameworks/communication/listen.bro index 609e8c91d6..111bc64a23 100644 --- a/scripts/policy/frameworks/communication/listen.bro +++ b/scripts/policy/frameworks/communication/listen.bro @@ -9,5 +9,5 @@ event bro_init() &priority=-10 { enable_communication(); listen(listen_interface, listen_port, listen_ssl, listen_ipv6, - listen_retry); + listen_ipv6_zone_id, listen_retry); } diff --git a/scripts/policy/frameworks/control/controller.bro b/scripts/policy/frameworks/control/controller.bro index 39647095db..22b19bf973 100644 --- a/scripts/policy/frameworks/control/controller.bro +++ b/scripts/policy/frameworks/control/controller.bro @@ -25,8 +25,8 @@ event bro_init() &priority=5 # Establish the communication configuration and only request response # messages. - Communication::nodes["control"] = [$host=host, $p=host_port, - $sync=F, $connect=T, + Communication::nodes["control"] = [$host=host, $zone_id=zone_id, + $p=host_port, $sync=F, $connect=T, $class="control", $events=Control::controllee_events]; } diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 9123e99ef4..b73494204c 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -173,6 +173,9 @@ #include #include +#include +#include +#include #include "RemoteSerializer.h" #include "Func.h" @@ -323,6 +326,16 @@ static const char* msgToStr(int msg) } } +static vector tokenize(const string& s, char delim) + { + vector tokens; + stringstream ss(s); + string token; + while ( std::getline(ss, token, delim) ) + tokens.push_back(token); + return tokens; + } + // Start of every message between two processes. We do the low-level work // ourselves to make this 64-bit safe. (The actual layout is an artifact of // an earlier design that depended on how a 32-bit GCC lays out its structs ...) @@ -665,7 +678,8 @@ void RemoteSerializer::Fork() } RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip, - uint16 port, const char* our_class, double retry, bool use_ssl) + const string& zone_id, uint16 port, const char* our_class, double retry, + bool use_ssl) { if ( ! using_communication ) return true; @@ -682,11 +696,13 @@ RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip, if ( our_class ) p->our_class = our_class; - uint32 bytes[4]; - ip.CopyIPv6(bytes, IPAddr::Host); + const size_t BUFSIZE = 1024; + char* data = new char[BUFSIZE]; + snprintf(data, BUFSIZE, "%"PRIu64",%s,%s,%"PRIu16",%"PRIu32",%d", p->id, + ip.AsString().c_str(), zone_id.c_str(), port, uint32(retry), + use_ssl); - if ( ! SendToChild(MSG_CONNECT_TO, p, 8, p->id, bytes[0], bytes[1], - bytes[2], bytes[3], port, uint32(retry), use_ssl) ) + if ( ! SendToChild(MSG_CONNECT_TO, p, data) ) { RemovePeer(p); return false; @@ -1219,7 +1235,7 @@ bool RemoteSerializer::SendCapabilities(Peer* peer) } bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl, - bool ipv6, double retry) + bool ipv6, const string& zone_id, double retry) { if ( ! using_communication ) return true; @@ -1229,13 +1245,16 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl, if ( ! ipv6 && ip.GetFamily() == IPv6 && ip != IPAddr("0.0.0.0") && ip != IPAddr("::") ) - reporter->FatalError("Attempt to listen on address %s, but IPv6 communication disabled", ip.AsString().c_str()); + reporter->FatalError("Attempt to listen on address %s, but IPv6 " + "communication disabled", ip.AsString().c_str()); - uint32 bytes[4]; - ip.CopyIPv6(bytes, IPAddr::Host); + const size_t BUFSIZE = 1024; + char* data = new char[BUFSIZE]; + snprintf(data, BUFSIZE, "%s,%"PRIu16",%d,%d,%s,%"PRIu32, + ip.AsString().c_str(), port, expect_ssl, ipv6, zone_id.c_str(), + (uint32) retry); - if ( ! SendToChild(MSG_LISTEN, 0, 8, bytes[0], bytes[1], bytes[2], bytes[3], - port, expect_ssl, ipv6, (uint32) retry) ) + if ( ! SendToChild(MSG_LISTEN, 0, data) ) return false; listening = true; @@ -1947,9 +1966,22 @@ bool RemoteSerializer::EnterPhaseRunning(Peer* peer) bool RemoteSerializer::ProcessConnected() { // IP and port follow. - uint32* args = (uint32*) current_args->data; - IPAddr host = IPAddr(IPv6, args, IPAddr::Network); - uint16 port = (uint16) ntohl(args[4]); + vector args = tokenize(current_args->data, ','); + + if ( args.size() != 2 ) + { + InternalCommError("ProcessConnected() bad number of arguments"); + return false; + } + + IPAddr host = IPAddr(args[0]); + uint16 port; + + if ( ! atoi_n(args[1].size(), args[1].c_str(), 0, 10, port) ) + { + InternalCommError("ProcessConnected() bad peer port string"); + return false; + } if ( ! current_peer ) { @@ -3692,14 +3724,43 @@ bool SocketComm::ForwardChunkToPeer() bool SocketComm::ProcessConnectTo() { assert(parent_args); - uint32* args = (uint32*) parent_args->data; + vector args = tokenize(parent_args->data, ','); + + if ( args.size() != 6 ) + { + Error(fmt("ProcessConnectTo() bad number of arguments")); + return false; + } Peer* peer = new Peer; - peer->id = ntohl(args[0]); - peer->ip = IPAddr(IPv6, &args[1], IPAddr::Network); - peer->port = ntohl(args[5]); - peer->retry = ntohl(args[6]); - peer->ssl = ntohl(args[7]); + + if ( ! atoi_n(args[0].size(), args[0].c_str(), 0, 10, peer->id) ) + { + Error(fmt("ProccessConnectTo() bad peer id string")); + delete peer; + return false; + } + + peer->ip = IPAddr(args[1]); + peer->zone_id = args[2]; + + if ( ! atoi_n(args[3].size(), args[3].c_str(), 0, 10, peer->port) ) + { + Error(fmt("ProcessConnectTo() bad peer port string")); + delete peer; + return false; + } + + if ( ! atoi_n(args[4].size(), args[4].c_str(), 0, 10, peer->retry) ) + { + Error(fmt("ProcessConnectTo() bad peer retry string")); + delete peer; + return false; + } + + peer->ssl = false; + if ( args[5] != "0" ) + peer->ssl = true; return Connect(peer); } @@ -3707,13 +3768,37 @@ bool SocketComm::ProcessConnectTo() bool SocketComm::ProcessListen() { assert(parent_args); - uint32* args = (uint32*) parent_args->data; + vector args = tokenize(parent_args->data, ','); - listen_if = IPAddr(IPv6, args, IPAddr::Network); - listen_port = uint16(ntohl(args[4])); - listen_ssl = ntohl(args[5]) != 0; - enable_ipv6 = ntohl(args[6]) != 0; - bind_retry_interval = ntohl(args[7]); + if ( args.size() != 6 ) + { + Error(fmt("ProcessListen() bad number of arguments")); + return false; + } + + listen_if = args[0]; + + if ( ! atoi_n(args[1].size(), args[1].c_str(), 0, 10, listen_port) ) + { + Error(fmt("ProcessListen() bad peer port string")); + return false; + } + + listen_ssl = false; + if ( args[2] != "0" ) + listen_ssl = true; + + enable_ipv6 = false; + if ( args[3] != "0" ) + enable_ipv6 = true; + + listen_zone_id = args[4]; + + if ( ! atoi_n(args[5].size(), args[5].c_str(), 0, 10, bind_retry_interval) ) + { + Error(fmt("ProcessListen() bad peer port string")); + return false; + } return Listen(); } @@ -3889,10 +3974,11 @@ bool SocketComm::Connect(Peer* peer) char port_str[16]; modp_uitoa10(peer->port, port_str); - // TODO: better to accept string arguments from the user to pass into - // getaddrinfo? This might make it easier to explicitly connect to - // non-global IPv6 addresses with a scope zone identifier (RFC 4007). - status = getaddrinfo(peer->ip.AsString().c_str(), port_str, &hints, &res0); + string gaihostname(peer->ip.AsString()); + if ( peer->zone_id != "" ) + gaihostname.append("%").append(peer->zone_id); + + status = getaddrinfo(gaihostname.c_str(), port_str, &hints, &res0); if ( status != 0 ) { Error(fmt("getaddrinfo error: %s", gai_strerror(status))); @@ -3964,11 +4050,12 @@ bool SocketComm::Connect(Peer* peer) { Log("connected", peer); - uint32 bytes[4]; - peer->ip.CopyIPv6(bytes, IPAddr::Host); + const size_t BUFSIZE = 1024; + char* data = new char[BUFSIZE]; + snprintf(data, BUFSIZE, "%s,%"PRIu32, peer->ip.AsString().c_str(), + peer->port); - if ( ! SendToParent(MSG_CONNECTED, peer, 5, bytes[0], bytes[1], - bytes[2], bytes[3], peer->port) ) + if ( ! SendToParent(MSG_CONNECTED, peer, data) ) return false; } @@ -4011,12 +4098,14 @@ bool SocketComm::Listen() addrinfo hints, *res, *res0; bzero(&hints, sizeof(hints)); + IPAddr listen_ip(listen_if); + if ( enable_ipv6 ) { - if ( listen_if == IPAddr("0.0.0.0") || listen_if == IPAddr("::") ) + if ( listen_ip == IPAddr("0.0.0.0") || listen_ip == IPAddr("::") ) hints.ai_family = PF_UNSPEC; else - hints.ai_family = listen_if.GetFamily() == IPv4 ? PF_INET : PF_INET6; + hints.ai_family = listen_ip.GetFamily() == IPv4 ? PF_INET : PF_INET6; } else hints.ai_family = PF_INET; @@ -4028,15 +4117,15 @@ bool SocketComm::Listen() char port_str[16]; modp_uitoa10(listen_port, port_str); + string scoped_addr(listen_if); + if ( listen_zone_id != "" ) + scoped_addr.append("%").append(listen_zone_id); const char* addr_str = 0; - if ( listen_if != IPAddr("0.0.0.0") && listen_if != IPAddr("::") ) - addr_str = listen_if.AsString().c_str(); + if ( listen_ip != IPAddr("0.0.0.0") && listen_ip != IPAddr("::") ) + addr_str = scoped_addr.c_str(); CloseListenFDs(); - // TODO: better to accept string arguments from the user to pass into - // getaddrinfo? This might make it easier to explicitly bind to a - // non-global IPv6 address with a scope zone identifier (RFC 4007). if ( (status = getaddrinfo(addr_str, port_str, &hints, &res0)) != 0 ) { Error(fmt("getaddrinfo error: %s", gai_strerror(status))); @@ -4056,6 +4145,10 @@ bool SocketComm::Listen() IPAddr(((sockaddr_in*)res->ai_addr)->sin_addr) : IPAddr(((sockaddr_in6*)res->ai_addr)->sin6_addr); + string l_addr_str(a.AsURIString()); + if ( listen_zone_id != "") + l_addr_str.append("%").append(listen_zone_id); + int fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); if ( fd < 0 ) { @@ -4075,7 +4168,7 @@ bool SocketComm::Listen() if ( bind(fd, res->ai_addr, res->ai_addrlen) < 0 ) { - Error(fmt("can't bind to %s:%s, %s", a.AsURIString().c_str(), + Error(fmt("can't bind to %s:%s, %s", l_addr_str.c_str(), port_str, strerror(errno))); close(fd); @@ -4092,14 +4185,14 @@ bool SocketComm::Listen() if ( listen(fd, 50) < 0 ) { - Error(fmt("can't listen on %s:%s, %s", a.AsURIString().c_str(), + Error(fmt("can't listen on %s:%s, %s", l_addr_str.c_str(), port_str, strerror(errno))); close(fd); continue; } listen_fds.push_back(fd); - Log(fmt("listening on %s:%s (%s)", a.AsURIString().c_str(), port_str, + Log(fmt("listening on %s:%s (%s)", l_addr_str.c_str(), port_str, listen_ssl ? "ssl" : "clear")); } @@ -4155,11 +4248,12 @@ bool SocketComm::AcceptConnection(int fd) Log(fmt("accepted %s connection", peer->ssl ? "SSL" : "clear"), peer); - uint32 bytes[4]; - peer->ip.CopyIPv6(bytes, IPAddr::Host); + const size_t BUFSIZE = 1024; + char* data = new char[BUFSIZE]; + snprintf(data, BUFSIZE, "%s,%"PRIu32, peer->ip.AsString().c_str(), + peer->port); - if ( ! SendToParent(MSG_CONNECTED, peer, 5, bytes[0], bytes[1], bytes[2], - bytes[3], peer->port) ) + if ( ! SendToParent(MSG_CONNECTED, peer, data) ) return false; return true; @@ -4176,8 +4270,13 @@ const char* SocketComm::MakeLogString(const char* msg, Peer* peer) int len = 0; if ( peer ) + { + string scoped_addr(peer->ip.AsURIString()); + if ( peer->zone_id != "" ) + scoped_addr.append("%").append(peer->zone_id); len = snprintf(buffer, BUFSIZE, "[#%d/%s:%d] ", int(peer->id), - peer->ip.AsURIString().c_str(), peer->port); + scoped_addr.c_str(), peer->port); + } len += safe_snprintf(buffer + len, BUFSIZE - len, "%s", msg); return buffer; diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index f6f94f53d3..4ebf15e68d 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -11,6 +11,7 @@ #include "File.h" #include +#include class IncrementalSendTimer; @@ -34,7 +35,8 @@ public: static const PeerID PEER_NONE = SOURCE_LOCAL; // Connect to host (returns PEER_NONE on error). - PeerID Connect(const IPAddr& ip, uint16 port, const char* our_class, double retry, bool use_ssl); + PeerID Connect(const IPAddr& ip, const string& zone_id, uint16 port, + const char* our_class, double retry, bool use_ssl); // Close connection to host. bool CloseConnection(PeerID peer); @@ -63,7 +65,7 @@ public: // Start to listen. bool Listen(const IPAddr& ip, uint16 port, bool expect_ssl, bool ipv6, - double retry); + const string& zone_id, double retry); // Stop it. bool StopListening(); @@ -422,6 +424,7 @@ protected: RemoteSerializer::PeerID id; ChunkedIO* io; IPAddr ip; + string zone_id; uint16 port; char state; bool connected; @@ -502,12 +505,13 @@ protected: // If the port we're trying to bind to is already in use, we will retry // it regularly. - IPAddr listen_if; + string listen_if; + string listen_zone_id; // RFC 4007 IPv6 zone_id uint16 listen_port; - bool listen_ssl; - bool enable_ipv6; // allow IPv6 listen sockets - uint32 bind_retry_interval; - time_t listen_next_try; + bool listen_ssl; // use SSL for IO + bool enable_ipv6; // allow IPv6 listen sockets + uint32 bind_retry_interval; // retry interval for already-in-use sockets + time_t listen_next_try; // time at which to try another bind bool shutting_conns_down; bool terminating; bool killing; diff --git a/src/bro.bif b/src/bro.bif index 3f4215dc13..f1e451bb03 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -5267,6 +5267,10 @@ function capture_state_updates%(filename: string%) : bool ## ## ip: The IP address of the remote peer. ## +## zone_id: If *ip* is a non-global IPv6 address, a particular :rfc:`4007` +## ``zone_id`` can given here. An empty string, ``""``, means +## not to add any ``zone_id``. +## ## port: The port of the remote peer. ## ## our_class: If an non-empty string, the remote (listening) peer checks it @@ -5290,10 +5294,11 @@ function capture_state_updates%(filename: string%) : bool ## set_compression_level ## send_state ## send_id -function connect%(ip: addr, p: port, our_class: string, retry: interval, ssl: bool%) : count +function connect%(ip: addr, zone_id: string, p: port, our_class: string, retry: interval, ssl: bool%) : count %{ - return new Val(uint32(remote_serializer->Connect(ip->AsAddr(), p->Port(), - our_class->CheckString(), retry, ssl)), + return new Val(uint32(remote_serializer->Connect(ip->AsAddr(), + zone_id->CheckString(), p->Port(), our_class->CheckString(), + retry, ssl)), TYPE_COUNT); %} @@ -5404,15 +5409,19 @@ function set_compression_level%(p: event_peer, level: count%) : bool ## ## ipv6: If true, enable listening on IPv6 addresses. ## +## zone_id: If *ip* is a non-global IPv6 address, a particular :rfc:`4007` +## ``zone_id`` can given here. An empty string, ``""``, means +## not to add any ``zone_id``. +## ## retry_interval: If address *ip* is found to be already in use, this is ## the interval at which to automatically retry binding. ## ## Returns: True on success. ## ## .. bro:see:: connect disconnect -function listen%(ip: addr, p: port, ssl: bool, ipv6: bool, retry_interval: interval%) : bool +function listen%(ip: addr, p: port, ssl: bool, ipv6: bool, zone_id: string, retry_interval: interval%) : bool %{ - return new Val(remote_serializer->Listen(ip->AsAddr(), p->Port(), ssl, ipv6, retry_interval), TYPE_BOOL); + return new Val(remote_serializer->Listen(ip->AsAddr(), p->Port(), ssl, ipv6, zone_id->CheckString(), retry_interval), TYPE_BOOL); %} ## Checks whether the last raised event came from a remote peer. diff --git a/src/util.cc b/src/util.cc index 90143923f1..798be400d1 100644 --- a/src/util.cc +++ b/src/util.cc @@ -376,6 +376,8 @@ template int atoi_n(int len, const char* s, const char** end, int base, // Instantiate the ones we need. template int atoi_n(int len, const char* s, const char** end, int base, int& result); +template int atoi_n(int len, const char* s, const char** end, int base, uint16_t& result); +template int atoi_n(int len, const char* s, const char** end, int base, uint32_t& result); template int atoi_n(int len, const char* s, const char** end, int base, int64_t& result); template int atoi_n(int len, const char* s, const char** end, int base, uint64_t& result); From be65ddca375e906bd6d409d50fbe894c759bb32d Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 17 May 2012 16:03:17 -0500 Subject: [PATCH 123/149] Correct various errors in the BIF documentation --- src/bro.bif | 210 ++++++++++++++++++++++++------------------------ src/strings.bif | 50 ++++++------ 2 files changed, 131 insertions(+), 129 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 15740a83c7..212a27044f 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -963,7 +963,7 @@ function sha256_hash_finish%(index: any%): string ## Generates a random number. ## -## max: The maximum value the random number. +## max: The maximum value of the random number. ## ## Returns: a random positive integer in the interval *[0, max)*. ## @@ -1020,7 +1020,7 @@ extern "C" { ## data: The data to find the MIME type for. ## ## return_mime: If true, the function returns a short MIME type string (e.g., -## ``text/plain`` instead of a more elaborate textual description. +## ``text/plain`` instead of a more elaborate textual description). ## ## Returns: The MIME type of *data*. function identify_data%(data: string, return_mime: bool%): string @@ -1241,8 +1241,6 @@ function unique_id_from%(pool: int, prefix: string%) : string ## Removes all elements from a set or table. ## ## v: The set or table -## -## Returns: The cleared set/table or 0 if *v* is not a set/table type. function clear_table%(v: any%): any %{ if ( v->Type()->Tag() == TYPE_TABLE ) @@ -1290,7 +1288,7 @@ function same_object%(o1: any, o2: any%): bool return new Val(o1 == o2, TYPE_BOOL); %} -## Returns the number bytes that a value occupies in memory. +## Returns the number of bytes that a value occupies in memory. ## ## v: The value ## @@ -1306,7 +1304,7 @@ function val_size%(v: any%): count ## ## newsize: The new size of *aggr*. ## -## Returns: The old size of *aggr* and 0 if *aggr* is not a :bro:type:`vector`. +## Returns: The old size of *aggr*, or 0 if *aggr* is not a :bro:type:`vector`. function resize%(aggr: any, newsize: count%) : count %{ if ( aggr->Type()->Tag() != TYPE_VECTOR ) @@ -1423,7 +1421,7 @@ bool indirect_int_sort_function(int a, int b) %%} ## Sorts a vector in place. The second argument is a comparison function that -## takes two arguments: if the vector type is \verb|vector of T|, then the +## takes two arguments: if the vector type is ``vector of T``, then the ## comparison function must be ``function(a: T, b: T): bool``, which returns ## ``a < b`` for some type-specific notion of the less-than operator. ## @@ -1599,7 +1597,7 @@ function cat%(...%): string ## given argument. If any of the variable arguments is an empty string it is ## replaced by a given default string instead. ## -## sep: The separator to place betwen each argument. +## sep: The separator to place between each argument. ## ## def: The default string to use when an argument is the empty string. ## @@ -1657,7 +1655,7 @@ function cat_sep%(sep: string, def: string, ...%): string ## ## - ``[DT]``: ISO timestamp with microsecond precision ## -## - ``d``: Signed/Unsigned integer (using C-style ``%lld|``/``%llu`` +## - ``d``: Signed/Unsigned integer (using C-style ``%lld``/``%llu`` ## for ``int``/``count``) ## ## - ``x``: Unsigned hexadecimal (using C-style ``%llx``); @@ -1782,7 +1780,7 @@ function log10%(d: double%): double # =========================================================================== ## Determines whether *c* has been received externally. For example, -## Broccoli or the Time Machine can send packets to Bro via a mechanism that +## Broccoli or the Time Machine can send packets to Bro via a mechanism that is ## one step lower than sending events. This function checks whether the packets ## of a connection stem from one of these external *packet sources*. ## @@ -1796,7 +1794,7 @@ function is_external_connection%(c: connection%) : bool ## Returns the ID of the analyzer which raised the current event. ## -## Returns: The ID of the analyzer which raised hte current event, or 0 if +## Returns: The ID of the analyzer which raised the current event, or 0 if ## none. function current_analyzer%(%) : count %{ @@ -2053,7 +2051,7 @@ function get_gap_summary%(%): gap_info %} ## Generates a table of the size of all global variables. The table index is -## the variable name and the value the variable size in bytes. +## the variable name and the value is the variable size in bytes. ## ## Returns: A table that maps variable names to their sizes. ## @@ -2138,7 +2136,7 @@ function lookup_ID%(id: string%) : any return i->ID_Val()->Ref(); %} -## Generates meta data about a record fields. The returned information +## Generates metadata about a record's fields. The returned information ## includes the field name, whether it is logged, its value (if it has one), ## and its default value (if specified). ## @@ -2269,11 +2267,11 @@ function dump_rule_stats%(f: file%): bool return new Val(1, TYPE_BOOL); %} -## Checks wheter Bro is terminating. +## Checks if Bro is terminating. ## ## Returns: True if Bro is in the process of shutting down. ## -## .. bro:see: terminate +## .. bro:see:: terminate function bro_is_terminating%(%): bool %{ return new Val(terminating, TYPE_BOOL); @@ -2354,7 +2352,7 @@ function routing0_data_to_addrs%(s: string%): addr_vec return rval; %} -## Converts a :bro:type:`addr` to a :bro:type:`index_vec`. +## Converts an :bro:type:`addr` to an :bro:type:`index_vec`. ## ## a: The address to convert into a vector of counts. ## @@ -2374,7 +2372,7 @@ function addr_to_counts%(a: addr%): index_vec return rval; %} -## Converts a :bro:type:`index_vec` to a :bro:type:`addr`. +## Converts an :bro:type:`index_vec` to an :bro:type:`addr`. ## ## v: The vector containing host-order IP address representation, ## one element for IPv4 addresses, four elements for IPv6 addresses. @@ -2404,7 +2402,7 @@ function counts_to_addr%(v: index_vec%): addr } %} -## Converts a :bro:type:`string` to a :bro:type:`int`. +## Converts a :bro:type:`string` to an :bro:type:`int`. ## ## str: The :bro:type:`string` to convert. ## @@ -2434,7 +2432,7 @@ function to_int%(str: string%): int ## ## n: The :bro:type:`int` to convert. ## -## Returns: The :bro:type:`int` *n* as unsigned integer or 0 if *n* < 0. +## Returns: The :bro:type:`int` *n* as unsigned integer, or 0 if *n* < 0. function int_to_count%(n: int%): count %{ if ( n < 0 ) @@ -2449,7 +2447,7 @@ function int_to_count%(n: int%): count ## ## d: The :bro:type:`double` to convert. ## -## Returns: The :bro:type:`double` *d* as unsigned integer or 0 if *d* < 0.0. +## Returns: The :bro:type:`double` *d* as unsigned integer, or 0 if *d* < 0.0. ## ## .. bro:see:: double_to_time function double_to_count%(d: double%): count @@ -2464,8 +2462,8 @@ function double_to_count%(d: double%): count ## ## str: The :bro:type:`string` to convert. ## -## Returns: The :bro:type:`string` *str* as unsigned integer or if in invalid -## format. +## Returns: The :bro:type:`string` *str* as unsigned integer, or 0 if *str* has +## an invalid format. ## ## .. bro:see:: to_addr to_int to_port to_subnet function to_count%(str: string%): count @@ -2498,7 +2496,7 @@ function interval_to_double%(i: interval%): double ## Converts a :bro:type:`time` value to a :bro:type:`double`. ## -## t: The :bro:type:`interval` to convert. +## t: The :bro:type:`time` to convert. ## ## Returns: The :bro:type:`time` value *t* as :bro:type:`double`. ## @@ -2508,11 +2506,11 @@ function time_to_double%(t: time%): double return new Val(t, TYPE_DOUBLE); %} -## Converts a :bro:type:`time` value to a :bro:type:`double`. +## Converts a :bro:type:`double` value to a :bro:type:`time`. ## -## t: The :bro:type:`interval` to convert. +## d: The :bro:type:`double` to convert. ## -## Returns: The :bro:type:`time` value *t* as :bro:type:`double`. +## Returns: The :bro:type:`double` value *d* as :bro:type:`time`. ## ## .. bro:see:: time_to_double double_to_count function double_to_time%(d: double%): time @@ -2550,7 +2548,7 @@ function port_to_count%(p: port%): count ## ## proto: The transport protocol. ## -## Returns: The :bro:type:`count` *c* as :bro:type:`port`. +## Returns: The :bro:type:`count` *num* as :bro:type:`port`. ## ## .. bro:see:: port_to_count function count_to_port%(num: count, proto: transport_proto%): port @@ -2562,7 +2560,7 @@ function count_to_port%(num: count, proto: transport_proto%): port ## ## ip: The :bro:type:`string` to convert. ## -## Returns: The :bro:type:`string` *ip* as :bro:type:`addr` or the unspecified +## Returns: The :bro:type:`string` *ip* as :bro:type:`addr`, or the unspecified ## address ``::`` if the input string does not parse correctly. ## ## .. bro:see:: to_count to_int to_port count_to_v4_addr raw_bytes_to_v4_addr @@ -2579,7 +2577,7 @@ function to_addr%(ip: string%): addr ## ## sn: The subnet to convert. ## -## Returns: The *sn* string as a :bro:type:`subnet` or the unspecified subnet +## Returns: The *sn* string as a :bro:type:`subnet`, or the unspecified subnet ## ``::/0`` if the input string does not parse correctly. ## ## .. bro:see:: to_count to_int to_port count_to_v4_addr raw_bytes_to_v4_addr @@ -2616,7 +2614,7 @@ function count_to_v4_addr%(ip: count%): addr ## ## b: The raw bytes (:bro:type:`string`) to convert. ## -## Returns: The byte :bro:type:`string` *ip* as :bro:type:`addr`. +## Returns: The byte :bro:type:`string` *b* as :bro:type:`addr`. ## ## .. bro:see:: raw_bytes_to_v4_addr to_addr to_subnet function raw_bytes_to_v4_addr%(b: string%): addr @@ -2635,7 +2633,7 @@ function raw_bytes_to_v4_addr%(b: string%): addr return new AddrVal(htonl(a)); %} -## Converts a :bro:type:`string` to an :bro:type:`port`. +## Converts a :bro:type:`string` to a :bro:type:`port`. ## ## s: The :bro:type:`string` to convert. ## @@ -2885,7 +2883,7 @@ function parse_ftp_port%(s: string%): ftp_port %} ## Converts a string representation of the FTP EPRT command to an ``ftp_port``. -## (see `RFC 2428 `_). +## See `RFC 2428 `_. ## The format is ``EPRT``, ## where ```` is a delimiter in the ASCII range 33-126 (usually ``|``). ## @@ -2976,7 +2974,7 @@ function fmt_ftp_port%(a: addr, p: port%): string ## Decode a NetBIOS name. See http://support.microsoft.com/kb/194203. ## -## name: The encoded NetBIOS name, e.g., ``"FEEIEFCAEOEFFEECEJEPFDCAEOEBENEF:``. +## name: The encoded NetBIOS name, e.g., ``"FEEIEFCAEOEFFEECEJEPFDCAEOEBENEF"``. ## ## Returns: The decoded NetBIOS name, e.g., ``"THE NETBIOS NAME"``. ## @@ -3009,7 +3007,7 @@ function decode_netbios_name%(name: string%): string return new StringVal(i, result); %} -## Converts a NetBIOS name type to its corresonding numeric value. +## Converts a NetBIOS name type to its corresponding numeric value. ## See http://support.microsoft.com/kb/163409. ## ## name: The NetBIOS name type. @@ -3029,7 +3027,7 @@ function decode_netbios_name_type%(name: string%): count ## ## bytestring: The string of bytes. ## -## Returns: The hexadecimal reprsentation of *bytestring*. +## Returns: The hexadecimal representation of *bytestring*. ## ## .. bro:see:: hexdump function bytestring_to_hexstr%(bytestring: string%): string @@ -3069,7 +3067,7 @@ function decode_base64%(s: string%): string ## s: The Base64-encoded string. ## ## a: The custom alphabet. The empty string indicates the default alphabet. The -## lengh of *a* must bt 64. For example, a custom alphabet could be +## length of *a* must be 64. For example, a custom alphabet could be ## ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``. ## ## Returns: The decoded version of *s*. @@ -3138,7 +3136,7 @@ function uuid_to_string%(uuid: string%): string ## ## p2: The second pattern. ## -## Returns: The compiled pattern of the concatentation of *p1* and *p2*. +## Returns: The compiled pattern of the concatenation of *p1* and *p2*. ## ## .. bro:see:: convert_for_pattern string_to_pattern ## @@ -3277,7 +3275,7 @@ function strftime%(fmt: string, d: time%) : string ## a: The address to mask. ## ## top_bits_to_keep: The number of top bits to keep in *a*; must be greater -## than 0 and less than 33. +## than 0 and less than 33 for IPv4, or 129 for IPv6. ## ## Returns: The address *a* masked down to *top_bits_to_keep* bits. ## @@ -3341,7 +3339,7 @@ function is_udp_port%(p: port%): bool ## ## p: The :bro:type:`port` to check. ## -## Returns: True iff *p* is a ICMP port. +## Returns: True iff *p* is an ICMP port. ## ## .. bro:see:: is_tcp_port is_udp_port function is_icmp_port%(p: port%): bool @@ -3383,7 +3381,7 @@ EnumVal* map_conn_type(TransportProto tp) ## ## cid: The connection identifier. ## -## Returns: The transport protocol of the connection identified by *id*. +## Returns: The transport protocol of the connection identified by *cid*. ## ## .. bro:see:: get_port_transport_proto ## get_orig_seq get_resp_seq @@ -3497,7 +3495,7 @@ const char* conn_id_string(Val* c) ## ## c: The HTTP connection. ## -## is_orig: If true, the client data is skipped and the server data otherwise. +## is_orig: If true, the client data is skipped, and the server data otherwise. ## ## .. bro:see:: skip_smtp_data function skip_http_entity_data%(c: connection, is_orig: bool%): any @@ -3572,7 +3570,7 @@ function dump_current_packet%(file_name: string%) : bool ## Returns the currently processed PCAP packet. ## -## Returns: The currently processed packet, which is as a record +## Returns: The currently processed packet, which is a record ## containing the timestamp, ``snaplen``, and packet data. ## ## .. bro:see:: dump_current_packet dump_packet send_current_packet @@ -3730,7 +3728,7 @@ function lookup_addr%(host: addr%) : string ## ## host: The hostname to lookup. ## -## Returns: A set of DNS A records associated with *host*. +## Returns: A set of DNS A and AAAA records associated with *host*. ## ## .. bro:see:: lookup_addr function lookup_hostname%(host: string%) : addr_set @@ -3897,6 +3895,7 @@ function lookup_location%(a: addr%) : geo_location %} ## Performs an AS lookup of an IP address. +## Requires Bro to be built with ``libgeoip``. ## ## a: The IP address to lookup. ## @@ -4096,7 +4095,7 @@ function x509_err2str%(err_num: count%): string ## Converts UNIX file permissions given by a mode to an ASCII string. ## -## mode: The permisssions, e.g., 644 or 755. +## mode: The permissions (an octal number like 0644 converted to decimal). ## ## Returns: A string representation of *mode* in the format ## ``rw[xsS]rw[xsS]rw[xtT]``. @@ -4273,7 +4272,7 @@ function analyzer_name%(aid: count%) : string ## ## cid: The connection ID. ## -## Returns: False if *id* does not point to an active connection and true +## Returns: False if *cid* does not point to an active connection, and true ## otherwise. ## ## .. note:: @@ -4295,10 +4294,10 @@ function skip_further_processing%(cid: conn_id%): bool ## ## cid: The connection identifier. ## -## do_record: True to enable packet contens and false to disable for the +## do_record: True to enable packet contents, and false to disable for the ## connection identified by *cid*. ## -## Returns: False if *id* does not point to an active connection and true +## Returns: False if *cid* does not point to an active connection, and true ## otherwise. ## ## .. bro:see:: skip_further_processing @@ -4309,7 +4308,7 @@ function skip_further_processing%(cid: conn_id%): bool ## connection, which is controlled separately by ## :bro:id:`skip_further_processing`. ## -## .. bro:see: get_contents_file set_contents_file +## .. bro:see:: get_contents_file set_contents_file function set_record_packets%(cid: conn_id, do_record: bool%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -4326,7 +4325,7 @@ function set_record_packets%(cid: conn_id, do_record: bool%): bool ## cid: The connection ID. ## ## direction: Controls what sides of the connection to record. The argument can -## take one the four values: +## take one of the four values: ## ## - ``CONTENTS_NONE``: Stop recording the connection's content. ## - ``CONTENTS_ORIG``: Record the data sent by the connection @@ -4340,7 +4339,7 @@ function set_record_packets%(cid: conn_id, do_record: bool%): bool ## ## f: The file handle of the file to write the contents to. ## -## Returns: Returns false if *id* does not point to an active connection and +## Returns: Returns false if *cid* does not point to an active connection, and ## true otherwise. ## ## .. note:: @@ -4351,7 +4350,7 @@ function set_record_packets%(cid: conn_id, do_record: bool%): bool ## missing data; this can happen, e.g., due to an ## :bro:id:`ack_above_hole` event. ## -## .. bro:see: get_contents_file set_record_packets +## .. bro:see:: get_contents_file set_record_packets function set_contents_file%(cid: conn_id, direction: count, f: file%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -4366,15 +4365,15 @@ function set_contents_file%(cid: conn_id, direction: count, f: file%): bool ## ## cid: The connection ID. ## -## direction: Controls what sides of the connection to record. SEe +## direction: Controls what sides of the connection to record. See ## :bro:id:`set_contents_file` for possible values. ## -## Returns: The :bro:type:`file` handle for the contentents file of the +## Returns: The :bro:type:`file` handle for the contents file of the ## connection identified by *cid*. If the connection exists -## but no contents file for *direction*, the function generates a -## error and returns a file handle to ``stderr``. +## but there is no contents file for *direction*, then the function +## generates an error and returns a file handle to ``stderr``. ## -## .. bro:see: set_contents_file set_record_packets +## .. bro:see:: set_contents_file set_record_packets function get_contents_file%(cid: conn_id, direction: count%): file %{ Connection* c = sessions->FindConnection(cid); @@ -4425,7 +4424,7 @@ function set_inactivity_timeout%(cid: conn_id, t: interval%): interval ## ## - ``LOGIN_STATE_AUTHENTICATE``: The connection is in its ## initial authentication dialog. -## - ``OGIN_STATE_LOGGED_IN``: The analyzer believes the user has +## - ``LOGIN_STATE_LOGGED_IN``: The analyzer believes the user has ## successfully authenticated. ## - ``LOGIN_STATE_SKIP``: The analyzer has skipped any further ## processing of the connection. @@ -4433,7 +4432,7 @@ function set_inactivity_timeout%(cid: conn_id, t: interval%): interval ## does not correctly know the state of the connection, and/or ## the username associated with it. ## -## .. bro:see: set_login_state +## .. bro:see:: set_login_state function get_login_state%(cid: conn_id%): count %{ Connection* c = sessions->FindConnection(cid); @@ -4456,9 +4455,9 @@ function get_login_state%(cid: conn_id%): count ## :bro:id:`get_login_state` for possible values. ## ## Returns: Returns false if *cid* is not an active connection -## or does not tagged as login analyzer, and true otherwise. +## or is not tagged as a login analyzer, and true otherwise. ## -## .. bro:see: get_login_state +## .. bro:see:: get_login_state function set_login_state%(cid: conn_id, new_state: count%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -4590,9 +4589,9 @@ function disable_event_group%(group: string%) : any ## ## f: The path to the file. ## -## Returns: A :bro:type:`file` handle for subsequent operations. +## Returns: A :bro:type:`file` handle for subsequent operations. ## -## .. bro:see;: active_file open_for_append close write_file +## .. bro:see:: active_file open_for_append close write_file ## get_file_name set_buf flush_all mkdir enable_raw_output function open%(f: string%): file %{ @@ -4609,9 +4608,9 @@ function open%(f: string%): file ## ## f: The path to the file. ## -## Returns: A :bro:type:`file` handle for subsequent operations. +## Returns: A :bro:type:`file` handle for subsequent operations. ## -## .. bro:see;: active_file open close write_file +## .. bro:see:: active_file open close write_file ## get_file_name set_buf flush_all mkdir enable_raw_output function open_for_append%(f: string%): file %{ @@ -4619,13 +4618,12 @@ function open_for_append%(f: string%): file %} ## Closes an open file and flushes any buffered content. -## exists, this function appends to it (as opposed to :bro:id:`open`). ## ## f: A :bro:type:`file` handle to an open file. ## -## Returns: True on success. +## Returns: True on success. ## -## .. bro:see;: active_file open open_for_append write_file +## .. bro:see:: active_file open open_for_append write_file ## get_file_name set_buf flush_all mkdir enable_raw_output function close%(f: file%): bool %{ @@ -4638,9 +4636,9 @@ function close%(f: file%): bool ## ## data: The data to write to *f*. ## -## Returns: True on success. +## Returns: True on success. ## -## .. bro:see;: active_file open open_for_append close +## .. bro:see:: active_file open open_for_append close ## get_file_name set_buf flush_all mkdir enable_raw_output function write_file%(f: file, data: string%): bool %{ @@ -4656,11 +4654,11 @@ function write_file%(f: file, data: string%): bool ## f: A :bro:type:`file` handle to an open file. ## ## buffered: When true, *f* is fully buffered, i.e., bytes are saved in a -## buffered until the block size has been reached. When +## buffer until the block size has been reached. When ## false, *f* is line buffered, i.e., bytes are saved up until a ## newline occurs. ## -## .. bro:see;: active_file open open_for_append close +## .. bro:see:: active_file open open_for_append close ## get_file_name write_file flush_all mkdir enable_raw_output function set_buf%(f: file, buffered: bool%): any %{ @@ -4670,9 +4668,9 @@ function set_buf%(f: file, buffered: bool%): any ## Flushes all open files to disk. ## -## Returns: True on success. +## Returns: True on success. ## -## .. bro:see;: active_file open open_for_append close +## .. bro:see:: active_file open open_for_append close ## get_file_name write_file set_buf mkdir enable_raw_output function flush_all%(%): bool %{ @@ -4683,10 +4681,10 @@ function flush_all%(%): bool ## ## f: The directory name. ## -## Returns: Returns true if the operation succeeds and false if the +## Returns: Returns true if the operation succeeds, or false if the ## creation fails or if *f* exists already. ## -## .. bro:see;: active_file open_for_append close write_file +## .. bro:see:: active_file open_for_append close write_file ## get_file_name set_buf flush_all enable_raw_output function mkdir%(f: string%): bool %{ @@ -4731,7 +4729,7 @@ function get_file_name%(f: file%): string ## ## f: An open file handle. ## -## Returns: Rotations statistics which include the original file name, the name +## Returns: Rotation statistics which include the original file name, the name ## after the rotation, and the time when *f* was opened/closed. ## ## .. bro:see:: rotate_file_by_name calc_next_rotate @@ -4755,7 +4753,7 @@ function rotate_file%(f: file%): rotate_info ## ## f: The name of the file to rotate ## -## Returns: Rotations statistics which include the original file name, the name +## Returns: Rotation statistics which include the original file name, the name ## after the rotation, and the time when *f* was opened/closed. ## ## .. bro:see:: rotate_file calc_next_rotate @@ -4851,7 +4849,7 @@ function disable_print_hook%(f: file%): any return 0; %} -## Prevents escaping of non-ASCII character when writing to a file. +## Prevents escaping of non-ASCII characters when writing to a file. ## This function is equivalent to :bro:attr:`&disable_print_hook`. ## ## f: The file to disable raw output for. @@ -5213,9 +5211,9 @@ function checkpoint_state%(%) : bool return new Val(persistence_serializer->WriteState(true), TYPE_BOOL); %} -## Reads persistent state from the \texttt{.state} directory and populates the -## in-memory data structures accordingly. This function is the dual to -## :bro:id:`checkpoint_state`. +## Reads persistent state and populates the in-memory data structures +## accordingly. Persistent state is read from the ``.state`` directory. +## This function is the dual to :bro:id:`checkpoint_state`. ## ## Returns: True on success. ## @@ -5267,16 +5265,16 @@ function capture_state_updates%(filename: string%) : bool ## ## ip: The IP address of the remote peer. ## -## port: The port of the remote peer. +## p: The port of the remote peer. ## -## our_class: If an non-empty string, the remote (listening) peer checks it +## our_class: If a non-empty string, then the remote (listening) peer checks it ## against its class name in its peer table and terminates the ## connection if they don't match. ## ## retry: If the connection fails, try to reconnect with the peer after this ## time interval. ## -## ssl: If true, uses SSL to encrypt the session. +## ssl: If true, use SSL to encrypt the session. ## ## Returns: A locally unique ID of the new peer. ## @@ -5299,7 +5297,7 @@ function connect%(ip: addr, p: port, our_class: string, retry: interval, ssl: bo ## Terminate the connection with a peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## Returns: True on success. ## @@ -5313,7 +5311,7 @@ function disconnect%(p: event_peer%) : bool ## Subscribes to all events from a remote peer whose names match a given ## pattern. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## handlers: The pattern describing the events to request from peer *p*. ## @@ -5331,7 +5329,7 @@ function request_remote_events%(p: event_peer, handlers: pattern%) : bool ## Requests synchronization of IDs with a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## auth: If true, the local instance considers its current state authoritative ## and sends it to *p* right after the handshake. @@ -5349,7 +5347,7 @@ function request_remote_sync%(p: event_peer, auth: bool%) : bool ## Requests logs from a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## Returns: True on success. ## @@ -5361,9 +5359,11 @@ function request_remote_logs%(p: event_peer%) : bool return new Val(remote_serializer->RequestLogs(id), TYPE_BOOL); %} -## Sets a boolean flag whether Bro accepts state from a remote peer. +## Sets a boolean flag indicating whether Bro accepts state from a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. +## +## accept: True if Bro accepts state from peer *p*, or false otherwise. ## ## Returns: True on success. ## @@ -5379,7 +5379,7 @@ function set_accept_state%(p: event_peer, accept: bool%) : bool ## Sets the compression level of the session with a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## level: Allowed values are in the range *[0, 9]*, where 0 is the default and ## means no compression. @@ -5394,11 +5394,11 @@ function set_compression_level%(p: event_peer, level: count%) : bool TYPE_BOOL); %} -## Listens on address a given IP address and port for remote connections. +## Listens on a given IP address and port for remote connections. ## ## ip: The IP address to bind to. ## -## p: The TCP port to listen to. +## p: The TCP port to listen on. ## ## ssl: If true, Bro uses SSL to encrypt the session. ## @@ -5420,7 +5420,7 @@ function is_remote_event%(%) : bool ## Sends all persistent state to a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## Returns: True on success. ## @@ -5431,10 +5431,10 @@ function send_state%(p: event_peer%) : bool return new Val(persistence_serializer->SendState(id, true), TYPE_BOOL); %} -## Sends a global identifier to a remote peer, which them might install it +## Sends a global identifier to a remote peer, which then might install it ## locally. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## id: The identifier to send. ## @@ -5468,7 +5468,7 @@ function terminate_communication%(%) : bool ## Signals a remote peer that the local Bro instance finished the initial ## handshake. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## Returns: True on success. function complete_handshake%(p: event_peer%) : bool @@ -5481,7 +5481,7 @@ function complete_handshake%(p: event_peer%) : bool ## for :bro:id:`remote_pong`, this function can be used to measure latency ## between two peers. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## seq: A sequence number (also included by :bro:id:`remote_pong`). ## @@ -5496,7 +5496,7 @@ function send_ping%(p: event_peer, seq: count%) : bool ## Sends the currently processed packet to a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## Returns: True if sending the packet succeeds. ## @@ -5522,7 +5522,7 @@ function send_current_packet%(p: event_peer%) : bool ## Returns the peer who generated the last event. ## -## Returns: The ID of the peer who genereated the last event. +## Returns: The ID of the peer who generated the last event. ## ## .. bro:see:: get_local_event_peer function get_event_peer%(%) : event_peer @@ -5565,7 +5565,7 @@ function get_local_event_peer%(%) : event_peer ## Sends a capture filter to a remote peer. ## -## p: The peer ID return from :bro:id:`connect`. +## p: The peer ID returned from :bro:id:`connect`. ## ## s: The capture filter. ## @@ -5582,7 +5582,7 @@ function send_capture_filter%(p: event_peer, s: string%) : bool ## distributed trace processing with communication enabled ## (*pseudo-realtime* mode). ## -## .. bro:see: continue_processing suspend_state_updates resume_state_updates +## .. bro:see:: continue_processing suspend_state_updates resume_state_updates function suspend_processing%(%) : any %{ net_suspend_processing(); @@ -5591,7 +5591,7 @@ function suspend_processing%(%) : any ## Resumes Bro's packet processing. ## -## .. bro:see: suspend_processing suspend_state_updates resume_state_updates +## .. bro:see:: suspend_processing suspend_state_updates resume_state_updates function continue_processing%(%) : any %{ net_continue_processing(); @@ -5600,7 +5600,7 @@ function continue_processing%(%) : any ## Stops propagating :bro:attr:`&synchronized` accesses. ## -## .. bro:see: suspend_processing continue_processing resume_state_updates +## .. bro:see:: suspend_processing continue_processing resume_state_updates function suspend_state_updates%(%) : any %{ if ( remote_serializer ) @@ -5610,7 +5610,7 @@ function suspend_state_updates%(%) : any ## Resumes propagating :bro:attr:`&synchronized` accesses. ## -## .. bro:see: suspend_processing continue_processing suspend_state_updates +## .. bro:see:: suspend_processing continue_processing suspend_state_updates function resume_state_updates%(%) : any %{ if ( remote_serializer ) diff --git a/src/strings.bif b/src/strings.bif index ebe16529ea..27c11b4013 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -11,8 +11,8 @@ using namespace std; %%} -## Concates all arguments into a single string. The function takes a variable -## number of arguments of type string and stiches them together. +## Concatenates all arguments into a single string. The function takes a +## variable number of arguments of type string and stitches them together. ## ## Returns: The concatenation of all (string) arguments. ## @@ -157,9 +157,9 @@ function join_string_array%(sep: string, a: string_array%): string ## ## sep: The separator to place between each element. ## -## a: The :bro:type:`string_vec` (``vector of string``). +## vec: The :bro:type:`string_vec` (``vector of string``). ## -## Returns: The concatenation of all elements in *a*, with *sep* placed +## Returns: The concatenation of all elements in *vec*, with *sep* placed ## between each element. ## ## .. bro:see:: cat cat_sep string_cat cat_string_array cat_string_array_n @@ -219,7 +219,7 @@ function sort_string_array%(a: string_array%): string_array ## Returns an edited version of a string that applies a special ## "backspace character" (usually ``\x08`` for backspace or ``\x7f`` for DEL). -## For ## example, ``edit("hello there", "e")`` returns ``"llo t"``. +## For example, ``edit("hello there", "e")`` returns ``"llo t"``. ## ## arg_s: The string to edit. ## @@ -229,7 +229,7 @@ function sort_string_array%(a: string_array%): string_array ## the string. ## ## Returns: An edited version of *arg_s* where *arg_edit_char* triggers the -## deletetion of the last character. +## deletion of the last character. ## ## .. bro:see:: clean ## to_string_literal @@ -278,7 +278,7 @@ function byte_len%(s: string%): count return new Val(s->Len(), TYPE_COUNT); %} -## Get a substring of from a string, given a starting position length. +## Get a substring from a string, given a starting position and length. ## ## s: The string to obtain a substring from. ## @@ -486,10 +486,10 @@ function split%(str: string, re: pattern%): string_array return do_split(str, re, 0, 0, 0); %} -## Splits a string *once* into a a two-element array of strings according to a -## pattern. This function is the same as :bro:id:`split`, but * is only split -## once (if possible) at the earliest position and an array of two strings is -## returned. +## Splits a string *once* into a two-element array of strings according to a +## pattern. This function is the same as :bro:id:`split`, but *str* is only +## split once (if possible) at the earliest position and an array of two strings +## is returned. ## ## str: The string to split. ## @@ -518,7 +518,7 @@ function split1%(str: string, re: pattern%): string_array ## ## Returns: An array of strings where each two successive elements correspond ## to a substring in *str* of the part not matching *re* (odd-indexed) -## and thei part that matches *re* (even-indexed). +## and the part that matches *re* (even-indexed). ## ## .. bro:see:: split split1 split_n str_split function split_all%(str: string, re: pattern%): string_array @@ -568,7 +568,7 @@ function split_complete%(str: string, ## ## re: The pattern being replaced with *repl*. ## -## repl: The string that replacs *re*. +## repl: The string that replaces *re*. ## ## Returns: A copy of *str* with the first occurence of *re* replaced with ## *repl*. @@ -579,16 +579,16 @@ function sub%(str: string, re: pattern, repl: string%): string return do_sub(str, re, repl, 0); %} -## Substitutes a given replacement string for the all occurrences of a pattern +## Substitutes a given replacement string for all occurrences of a pattern ## in a given string. ## ## str: The string to perform the substitution in. ## ## re: The pattern being replaced with *repl*. ## -## repl: The string that replacs *re*. +## repl: The string that replaces *re*. ## -## Returns: A copy of *str* with all occurences of *re* replaced with *repl*. +## Returns: A copy of *str* with all occurrences of *re* replaced with *repl*. ## ## .. bro:see:: sub subst_string function gsub%(str: string, re: pattern, repl: string%): string @@ -597,7 +597,7 @@ function gsub%(str: string, re: pattern, repl: string%): string %} -## Lexicographically compares two string. +## Lexicographically compares two strings. ## ## s1: The first string. ## @@ -616,7 +616,7 @@ function strcmp%(s1: string, s2: string%): int ## ## little: The (smaller) string to find inside *big*. ## -## Returns: The location of *little* in *big* or 0 if *little* is not found in +## Returns: The location of *little* in *big*, or 0 if *little* is not found in ## *big*. ## ## .. bro:see:: find_all find_last @@ -685,7 +685,7 @@ function subst_string%(s: string, from: string, to: string%): string ## str: The string to convert to lowercase letters. ## ## Returns: A copy of the given string with the uppercase letters (as indicated -## by ``isascii`` and \verb|isupper|``) folded to lowercase +## by ``isascii`` and ``isupper``) folded to lowercase ## (via ``tolower``). ## ## .. bro:see:: to_upper is_ascii @@ -714,7 +714,7 @@ function to_lower%(str: string%): string ## str: The string to convert to uppercase letters. ## ## Returns: A copy of the given string with the lowercase letters (as indicated -## by ``isascii`` and \verb|islower|``) folded to uppercase +## by ``isascii`` and ``islower``) folded to uppercase ## (via ``toupper``). ## ## .. bro:see:: to_lower is_ascii @@ -744,7 +744,7 @@ function to_upper%(str: string%): string ## - ``NUL`` to ``\0`` ## - ``DEL`` to ``^?`` ## - values <= 26 to ``^[A-Z]`` -## - values not in *[32, 126]** to ``%XX`` +## - values not in *[32, 126]* to ``%XX`` ## ## If the string does not yet have a trailing NUL, one is added. ## @@ -765,7 +765,7 @@ function clean%(str: string%): string ## - ``NUL`` to ``\0`` ## - ``DEL`` to ``^?`` ## - values <= 26 to ``^[A-Z]`` -## - values not in *[32, 126]** to ``%XX`` +## - values not in *[32, 126]* to ``%XX`` ## ## str: The string to escape. ## @@ -831,14 +831,16 @@ function string_to_ascii_hex%(s: string%): string return new StringVal(new BroString(1, (u_char*) x, s->Len() * 2)); %} -## Uses the Smith Waterman algorithm to find similar/overlapping substrings. +## Uses the Smith-Waterman algorithm to find similar/overlapping substrings. ## See `Wikipedia `_. ## ## s1: The first string. ## ## s2: The second string. ## -## Returns: The result of the Smit Waterman algorithm calculation. +## params: Parameters for the Smith-Waterman algorithm. +## +## Returns: The result of the Smith-Waterman algorithm calculation. function str_smith_waterman%(s1: string, s2: string, params: sw_params%) : sw_substring_vec %{ SWParams sw_params(params->AsRecordVal()->Lookup(0)->AsCount(), From 5ab765b4b6643fa872889ef03f56604ba7748a2a Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Fri, 18 May 2012 11:23:09 -0500 Subject: [PATCH 124/149] Replace ip6_hdr_chain with ip6_ext_hdr in comments This fixes some warnings that were appearing when building the documentation. --- scripts/base/init-bare.bro | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 20ce7b8ff5..73f7d725d4 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -967,7 +967,7 @@ const IPPROTO_MOBILITY = 135; ##< IPv6 mobility header. ## Values extracted from an IPv6 extension header's (e.g. hop-by-hop or ## destination option headers) option field. ## -## .. bro:see:: ip6_hdr ip6_hdr_chain ip6_hopopts ip6_dstopts +## .. bro:see:: ip6_hdr ip6_ext_hdr ip6_hopopts ip6_dstopts type ip6_option: record { otype: count; ##< Option type. len: count; ##< Option data length. @@ -976,7 +976,7 @@ type ip6_option: record { ## Values extracted from an IPv6 Hop-by-Hop options extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain ip6_option +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr ip6_option type ip6_hopopts: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -989,7 +989,7 @@ type ip6_hopopts: record { ## Values extracted from an IPv6 Destination options extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain ip6_option +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr ip6_option type ip6_dstopts: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1002,7 +1002,7 @@ type ip6_dstopts: record { ## Values extracted from an IPv6 Routing extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr type ip6_routing: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1019,7 +1019,7 @@ type ip6_routing: record { ## Values extracted from an IPv6 Fragment extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr type ip6_fragment: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1038,7 +1038,7 @@ type ip6_fragment: record { ## Values extracted from an IPv6 Authentication extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr type ip6_ah: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1057,7 +1057,7 @@ type ip6_ah: record { ## Values extracted from an IPv6 ESP extension header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr type ip6_esp: record { ## Security Parameters Index. spi: count; @@ -1067,7 +1067,7 @@ type ip6_esp: record { ## Values extracted from an IPv6 Mobility Binding Refresh Request message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_brr: record { ## Reserved. rsv: count; @@ -1077,7 +1077,7 @@ type ip6_mobility_brr: record { ## Values extracted from an IPv6 Mobility Home Test Init message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_hoti: record { ## Reserved. rsv: count; @@ -1089,7 +1089,7 @@ type ip6_mobility_hoti: record { ## Values extracted from an IPv6 Mobility Care-of Test Init message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_coti: record { ## Reserved. rsv: count; @@ -1101,7 +1101,7 @@ type ip6_mobility_coti: record { ## Values extracted from an IPv6 Mobility Home Test message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_hot: record { ## Home Nonce Index. nonce_idx: count; @@ -1115,7 +1115,7 @@ type ip6_mobility_hot: record { ## Values extracted from an IPv6 Mobility Care-of Test message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_cot: record { ## Care-of Nonce Index. nonce_idx: count; @@ -1129,7 +1129,7 @@ type ip6_mobility_cot: record { ## Values extracted from an IPv6 Mobility Binding Update message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_bu: record { ## Sequence number. seq: count; @@ -1149,7 +1149,7 @@ type ip6_mobility_bu: record { ## Values extracted from an IPv6 Mobility Binding Acknowledgement message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_back: record { ## Status. status: count; @@ -1165,7 +1165,7 @@ type ip6_mobility_back: record { ## Values extracted from an IPv6 Mobility Binding Error message. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg type ip6_mobility_be: record { ## Status. status: count; @@ -1177,7 +1177,7 @@ type ip6_mobility_be: record { ## Values extracted from an IPv6 Mobility header's message data. ## -## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr type ip6_mobility_msg: record { ## The type of message from the header's MH Type field. id: count; @@ -1201,7 +1201,7 @@ type ip6_mobility_msg: record { ## Values extracted from an IPv6 Mobility header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain +## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr type ip6_mobility_hdr: record { ## Protocol number of the next header (RFC 1700 et seq., IANA assigned ## number), e.g. :bro:id:`IPPROTO_ICMP`. @@ -1244,7 +1244,7 @@ type ip6_ext_hdr: record { ## Values extracted from an IPv6 header. ## -## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr_chain ip6_hopopts ip6_dstopts +## .. bro:see:: pkt_hdr ip4_hdr ip6_ext_hdr ip6_hopopts ip6_dstopts ## ip6_routing ip6_fragment ip6_ah ip6_esp type ip6_hdr: record { class: count; ##< Traffic class. From 5312b21d7bd4e19f7fbd8dffa6e0f6277014fb01 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 22 May 2012 15:18:33 -0500 Subject: [PATCH 125/149] Improve availability of IPv6 flow label in connection records. Without this change, flow labeling of connections over IPv6 are only available in the per-packet types of events (e.g. new_packet) in which header fields can be inspected, but now minimal tracking of the most recent flow label is done internally and that's available per-connection for all events that use connection record arguments. Specifically, this adds a "flow_label" field to the "endpoint" record type, which is used for both the "orig" and "resp" fields of "connection" records. The new "connection_flow_label_changed" event also allows tracking of changes in flow labels: it's raised each time one direction of the connection starts using a different label. --- scripts/base/init-bare.bro | 7 +- src/Conn.cc | 49 +++++++++++- src/Conn.h | 7 +- src/IP.h | 6 ++ src/Sessions.cc | 10 ++- src/Sessions.h | 2 +- src/event.bif | 14 ++++ .../Baseline/core.ipv6-flow-labels/output | 74 +++++++++++++++++++ testing/btest/core/ipv6-flow-labels.test | 32 ++++++++ 9 files changed, 192 insertions(+), 9 deletions(-) create mode 100644 testing/btest/Baseline/core.ipv6-flow-labels/output create mode 100644 testing/btest/core/ipv6-flow-labels.test diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 20ce7b8ff5..dadeab734a 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -178,9 +178,9 @@ type endpoint_stats: record { ## use ``count``. That should be changed. type AnalyzerID: count; -## Statistics about an endpoint. +## Statistics about a :bro:type:`connection` endpoint. ## -## todo::Where is this used? +## .. bro:see:: connection type endpoint: record { size: count; ##< Logical size of data sent (for TCP: derived from sequence numbers). ## Endpoint state. For TCP connection, one of the constants: @@ -194,6 +194,9 @@ type endpoint: record { ## Number of IP-level bytes sent. Only set if :bro:id:`use_conn_size_analyzer` is ## true. num_bytes_ip: count &optional; + ## The current IPv6 flow label that the connection endpoint is using. + ## Always 0 if the connection is over IPv4. + flow_label: count; }; # A connection. This is Bro's basic connection type describing IP- and diff --git a/src/Conn.cc b/src/Conn.cc index acf17fab3a..3835097b6a 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -111,7 +111,8 @@ unsigned int Connection::external_connections = 0; IMPLEMENT_SERIAL(Connection, SER_CONNECTION); -Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id) +Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, + uint32 flow) { sessions = s; key = k; @@ -122,6 +123,10 @@ Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id) orig_port = id->src_port; resp_port = id->dst_port; proto = TRANSPORT_UNKNOWN; + orig_flow_label = flow; + resp_flow_label = 0; + saw_first_orig_packet = 1; + saw_first_resp_packet = 0; conn_val = 0; login_conn = 0; @@ -323,10 +328,12 @@ RecordVal* Connection::BuildConnVal() RecordVal *orig_endp = new RecordVal(endpoint); orig_endp->Assign(0, new Val(0, TYPE_COUNT)); orig_endp->Assign(1, new Val(0, TYPE_COUNT)); + orig_endp->Assign(4, new Val(orig_flow_label, TYPE_COUNT)); RecordVal *resp_endp = new RecordVal(endpoint); resp_endp->Assign(0, new Val(0, TYPE_COUNT)); resp_endp->Assign(1, new Val(0, TYPE_COUNT)); + resp_endp->Assign(4, new Val(resp_flow_label, TYPE_COUNT)); conn_val->Assign(0, id_val); conn_val->Assign(1, orig_endp); @@ -675,6 +682,14 @@ void Connection::FlipRoles() resp_port = orig_port; orig_port = tmp_port; + bool tmp_bool = saw_first_resp_packet; + saw_first_resp_packet = saw_first_orig_packet; + saw_first_orig_packet = tmp_bool; + + uint32 tmp_flow = resp_flow_label; + resp_flow_label = orig_flow_label; + orig_flow_label = tmp_flow; + Unref(conn_val); conn_val = 0; @@ -882,3 +897,35 @@ void Connection::SetRootAnalyzer(TransportLayerAnalyzer* analyzer, PIA* pia) root_analyzer = analyzer; primary_PIA = pia; } + +void Connection::CheckFlowLabel(bool is_orig, uint32 flow_label) + { + uint32& my_flow_label = is_orig ? orig_flow_label : resp_flow_label; + + if ( my_flow_label != flow_label ) + { + if ( conn_val ) + { + RecordVal *endp = conn_val->Lookup(is_orig ? 1 : 2)->AsRecordVal(); + endp->Assign(4, new Val(flow_label, TYPE_COUNT)); + } + + if ( connection_flow_label_changed && + (is_orig ? saw_first_orig_packet : saw_first_resp_packet) ) + { + val_list* vl = new val_list(4); + vl->append(BuildConnVal()); + vl->append(new Val(is_orig, TYPE_BOOL)); + vl->append(new Val(my_flow_label, TYPE_COUNT)); + vl->append(new Val(flow_label, TYPE_COUNT)); + ConnectionEvent(connection_flow_label_changed, 0, vl); + } + + my_flow_label = flow_label; + } + + if ( is_orig ) + saw_first_orig_packet = 1; + else + saw_first_resp_packet = 1; + } diff --git a/src/Conn.h b/src/Conn.h index b3eb9013d0..7404721968 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -50,7 +50,8 @@ class Analyzer; class Connection : public BroObj { public: - Connection(NetSessions* s, HashKey* k, double t, const ConnID* id); + Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, + uint32 flow); virtual ~Connection(); // Invoked when connection is about to be removed. Use Ref(this) @@ -241,6 +242,8 @@ public: void SetUID(uint64 arg_uid) { uid = arg_uid; } + void CheckFlowLabel(bool is_orig, uint32 flow_label); + protected: Connection() { persistent = 0; } @@ -271,6 +274,7 @@ protected: IPAddr resp_addr; uint32 orig_port, resp_port; // in network order TransportProto proto; + uint32 orig_flow_label, resp_flow_label; // most recent IPv6 flow labels double start_time, last_time; double inactivity_timeout; RecordVal* conn_val; @@ -286,6 +290,7 @@ protected: unsigned int record_packets:1, record_contents:1; unsigned int persistent:1; unsigned int record_current_packet:1, record_current_content:1; + unsigned int saw_first_orig_packet:1, saw_first_resp_packet:1; // Count number of connections. static unsigned int total_connections; diff --git a/src/IP.h b/src/IP.h index 502ae857c0..c3a74b4a01 100644 --- a/src/IP.h +++ b/src/IP.h @@ -524,6 +524,12 @@ public: int DF() const { return ip4 ? ((ntohs(ip4->ip_off) & 0x4000) != 0) : 0; } + /** + * Returns value of an IPv6 header's flow label field or 0 if it's IPv4. + */ + uint32 FlowLabel() const + { return ip4 ? 0 : (ntohl(ip6->ip6_flow) & 0x000fffff); } + /** * Returns number of IP headers in packet (includes IPv6 extension headers). */ diff --git a/src/Sessions.cc b/src/Sessions.cc index 7da1f088de..4419936fbd 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -602,7 +602,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, conn = (Connection*) d->Lookup(h); if ( ! conn ) { - conn = NewConn(h, t, &id, data, proto); + conn = NewConn(h, t, &id, data, proto, ip_hdr->FlowLabel()); if ( conn ) d->Insert(h, conn); } @@ -623,7 +623,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, conn->Event(connection_reused, 0); Remove(conn); - conn = NewConn(h, t, &id, data, proto); + conn = NewConn(h, t, &id, data, proto, ip_hdr->FlowLabel()); if ( conn ) d->Insert(h, conn); } @@ -644,6 +644,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, int is_orig = (id.src_addr == conn->OrigAddr()) && (id.src_port == conn->OrigPort()); + conn->CheckFlowLabel(is_orig, ip_hdr->FlowLabel()); + Val* pkt_hdr_val = 0; if ( ipv6_ext_headers && ip_hdr->NumHeaders() > 1 ) @@ -1002,7 +1004,7 @@ void NetSessions::GetStats(SessionStats& s) const } Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, - const u_char* data, int proto) + const u_char* data, int proto, uint32 flow_label) { // FIXME: This should be cleaned up a bit, it's too protocol-specific. // But I'm not yet sure what the right abstraction for these things is. @@ -1058,7 +1060,7 @@ Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, id = &flip_id; } - Connection* conn = new Connection(this, k, t, id); + Connection* conn = new Connection(this, k, t, id, flow_label); conn->SetTransport(tproto); dpm->BuildInitialAnalyzerTree(tproto, conn, data); diff --git a/src/Sessions.h b/src/Sessions.h index 06c6057dbf..d29ab0eeab 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -142,7 +142,7 @@ protected: friend class TimerMgrExpireTimer; Connection* NewConn(HashKey* k, double t, const ConnID* id, - const u_char* data, int proto); + const u_char* data, int proto, uint32 flow_label); // Check whether the tag of the current packet is consistent with // the given connection. Returns: diff --git a/src/event.bif b/src/event.bif index ded054dd53..80bb46e561 100644 --- a/src/event.bif +++ b/src/event.bif @@ -401,6 +401,20 @@ event connection_reused%(c: connection%); ## new_connection new_connection_contents partial_connection event connection_status_update%(c: connection%); +## Generated for a connection over IPv6 when one direction has changed +## the flow label that it's using. +## +## c: The connection. +## +## is_orig: True if the event is raised for the originator side. +## +## old_label: The old flow label that the endpoint was using. +## +## new_label: The new flow label that the endpoint is using. +## +## .. bro:see:: connection_established new_connection +event connection_flow_label_changed%(c: connection, is_orig: bool, old_label: count, new_label: count%); + ## Generated at the end of reassembled TCP connections. The TCP reassembler ## raised the event once for each endpoint of a connection when it finished ## reassembling the corresponding side of the communication. diff --git a/testing/btest/Baseline/core.ipv6-flow-labels/output b/testing/btest/Baseline/core.ipv6-flow-labels/output new file mode 100644 index 0000000000..9f7292d485 --- /dev/null +++ b/testing/btest/Baseline/core.ipv6-flow-labels/output @@ -0,0 +1,74 @@ +new_connection: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49185/tcp, resp_h=2001:470:4867:99::21, resp_p=21/tcp] + orig_flow 0 + resp_flow 0 +connection_established: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49185/tcp, resp_h=2001:470:4867:99::21, resp_p=21/tcp] + orig_flow 0 + resp_flow 0 +connection_flow_label_changed(resp): [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49185/tcp, resp_h=2001:470:4867:99::21, resp_p=21/tcp] + orig_flow 0 + resp_flow 7407 + old_label 0 + new_label 7407 +new_connection: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49186/tcp, resp_h=2001:470:4867:99::21, resp_p=57086/tcp] + orig_flow 0 + resp_flow 0 +connection_established: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49186/tcp, resp_h=2001:470:4867:99::21, resp_p=57086/tcp] + orig_flow 0 + resp_flow 0 +connection_flow_label_changed(resp): [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49186/tcp, resp_h=2001:470:4867:99::21, resp_p=57086/tcp] + orig_flow 0 + resp_flow 176012 + old_label 0 + new_label 176012 +new_connection: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49187/tcp, resp_h=2001:470:4867:99::21, resp_p=57087/tcp] + orig_flow 0 + resp_flow 0 +connection_established: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49187/tcp, resp_h=2001:470:4867:99::21, resp_p=57087/tcp] + orig_flow 0 + resp_flow 0 +connection_flow_label_changed(resp): [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49187/tcp, resp_h=2001:470:4867:99::21, resp_p=57087/tcp] + orig_flow 0 + resp_flow 390927 + old_label 0 + new_label 390927 +new_connection: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49188/tcp, resp_h=2001:470:4867:99::21, resp_p=57088/tcp] + orig_flow 0 + resp_flow 0 +connection_established: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49188/tcp, resp_h=2001:470:4867:99::21, resp_p=57088/tcp] + orig_flow 0 + resp_flow 0 +connection_flow_label_changed(resp): [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49188/tcp, resp_h=2001:470:4867:99::21, resp_p=57088/tcp] + orig_flow 0 + resp_flow 364705 + old_label 0 + new_label 364705 +connection_state_remove: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49186/tcp, resp_h=2001:470:4867:99::21, resp_p=57086/tcp] + orig_flow 0 + resp_flow 176012 +connection_state_remove: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49187/tcp, resp_h=2001:470:4867:99::21, resp_p=57087/tcp] + orig_flow 0 + resp_flow 390927 +connection_state_remove: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49188/tcp, resp_h=2001:470:4867:99::21, resp_p=57088/tcp] + orig_flow 0 + resp_flow 364705 +new_connection: [orig_h=2001:470:4867:99::21, orig_p=55785/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49189/tcp] + orig_flow 267377 + resp_flow 0 +connection_established: [orig_h=2001:470:4867:99::21, orig_p=55785/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49189/tcp] + orig_flow 267377 + resp_flow 126027 +new_connection: [orig_h=2001:470:4867:99::21, orig_p=55647/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49190/tcp] + orig_flow 355265 + resp_flow 0 +connection_established: [orig_h=2001:470:4867:99::21, orig_p=55647/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49190/tcp] + orig_flow 355265 + resp_flow 126028 +connection_state_remove: [orig_h=2001:470:4867:99::21, orig_p=55785/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49189/tcp] + orig_flow 267377 + resp_flow 126027 +connection_state_remove: [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49185/tcp, resp_h=2001:470:4867:99::21, resp_p=21/tcp] + orig_flow 0 + resp_flow 7407 +connection_state_remove: [orig_h=2001:470:4867:99::21, orig_p=55647/tcp, resp_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, resp_p=49190/tcp] + orig_flow 355265 + resp_flow 126028 diff --git a/testing/btest/core/ipv6-flow-labels.test b/testing/btest/core/ipv6-flow-labels.test new file mode 100644 index 0000000000..b4e60cb0a4 --- /dev/null +++ b/testing/btest/core/ipv6-flow-labels.test @@ -0,0 +1,32 @@ +# @TEST-EXEC: bro -b -r $TRACES/ipv6-ftp.trace %INPUT >output +# @TEST-EXEC: btest-diff output + +function print_connection(c: connection, event_name: string) + { + print fmt("%s: %s", event_name, c$id); + print fmt(" orig_flow %d", c$orig$flow_label); + print fmt(" resp_flow %d", c$resp$flow_label); + } + +event new_connection(c: connection) + { + print_connection(c, "new_connection"); + } + +event connection_established(c: connection) + { + print_connection(c, "connection_established"); + } + +event connection_state_remove(c: connection) + { + print_connection(c, "connection_state_remove"); + } + +event connection_flow_label_changed(c: connection, is_orig: bool, + old_label: count, new_label: count) + { + print_connection(c, fmt("connection_flow_label_changed(%s)", is_orig ? "orig" : "resp")); + print fmt(" old_label %d", old_label); + print fmt(" new_label %d", new_label); + } From 82a6f3832ae10c5a9b5a1646422511c9fce1b041 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 22 May 2012 13:51:50 -0700 Subject: [PATCH 126/149] fix two memory leaks which occured when one used filters. --- src/input/Manager.cc | 6 +++++- src/input/readers/Ascii.cc | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index bd6cd34991..6b179f66a1 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -750,7 +750,7 @@ Val* Manager::RecordValToIndexVal(RecordVal *r) { int num_fields = type->NumFields(); if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) { - idxval = r->Lookup(0); + idxval = r->LookupWithDefault(0); } else { ListVal *l = new ListVal(TYPE_ANY); for ( int j = 0 ; j < num_fields; j++ ) { @@ -902,6 +902,7 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { if ( result == false ) { Unref(predidx); + Unref(valval); if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... // (but why should it be in there? assert this). @@ -956,6 +957,9 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { Ref(oldval); // otherwise it is no longer accessible after the assignment filter->tab->Assign(idxval, k, valval); Unref(idxval); // asssign does not consume idxval. + if ( predidx != 0 ) { + Unref(predidx); + } filter->currDict->Insert(idxhash, ih); delete idxhash; diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index a1119ed253..a167408a0e 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -101,12 +101,16 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c file = new ifstream(path.c_str()); if ( !file->is_open() ) { Error(Fmt("Init: cannot open %s", fname.c_str())); + delete(file); + file = 0; return false; } if ( ReadHeader(false) == false ) { Error(Fmt("Init: cannot open %s; headers are incorrect", fname.c_str())); file->close(); + delete(file); + file = 0; return false; } From 074a0a9dce5dca4219e213c83264e16ef4450733 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 23 May 2012 14:29:16 -0500 Subject: [PATCH 127/149] Documentation fixes. --- scripts/base/init-bare.bro | 2 +- scripts/base/protocols/conn/main.bro | 2 +- src/event.bif | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 73f7d725d4..17748917b7 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -219,7 +219,7 @@ type connection: record { service: set[string]; addl: string; ##< Deprecated. hot: count; ##< Deprecated. - history: string; ##< State history of TCP connections. See *history* in :bro:see:`Conn::Info`. + history: string; ##< State history of connections. See *history* in :bro:see:`Conn::Info`. ## A globally unique connection identifier. For each connection, Bro creates an ID ## that is very likely unique across independent Bro runs. These IDs can thus be ## used to tag and locate information associated with that connection. diff --git a/scripts/base/protocols/conn/main.bro b/scripts/base/protocols/conn/main.bro index 34ec12fa56..c526681f2a 100644 --- a/scripts/base/protocols/conn/main.bro +++ b/scripts/base/protocols/conn/main.bro @@ -68,7 +68,7 @@ export { missed_bytes: count &log &default=0; ## Records the state history of connections as a string of letters. - ## For TCP connections the meaning of those letters is: + ## The meaning of those letters is: ## ## ====== ==================================================== ## Letter Meaning diff --git a/src/event.bif b/src/event.bif index ded054dd53..e3dcfb6aef 100644 --- a/src/event.bif +++ b/src/event.bif @@ -171,8 +171,11 @@ event new_connection_contents%(c: connection%); ## new_connection new_connection_contents partial_connection event connection_attempt%(c: connection%); -## Generated for an established TCP connection. The event is raised when the -## initial 3-way TCP handshake has successfully finished for a connection. +## Generated when a SYN-ACK packet is seen in response to SYN a packet during +## a TCP handshake. The final ACK of the handshake in response to SYN-ACK may +## or may not occur later, one way to tell is to check the *history* field of +## :bro:type:`connection` to see if the originator sent an ACK, indicated by +## 'A' in the history string. ## ## c: The connection. ## @@ -335,8 +338,6 @@ event connection_SYN_packet%(c: connection, pkt: SYN_packet%); ## ## c: The connection. ## -## pkt: Information extracted from the SYN packet. -## ## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt ## connection_established connection_external connection_finished ## connection_half_finished connection_partial_close connection_pending From aaa16133a7ddb8c3f960dd4e08d27d27eae79e08 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 24 May 2012 16:48:15 -0700 Subject: [PATCH 128/149] Make tests even quieter. --- testing/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/Makefile b/testing/Makefile index 1c82580ec4..c4a2aeddda 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -6,10 +6,10 @@ all: make-verbose coverage brief: make-brief coverage make-verbose: - @for repo in $(DIRS); do (cd $$repo && make ); done + @for repo in $(DIRS); do (cd $$repo && make -s ); done make-brief: - @for repo in $(DIRS); do (cd $$repo && make brief ); done + @for repo in $(DIRS); do (cd $$repo && make -s brief ); done coverage: @for repo in $(DIRS); do (cd $$repo && echo "Coverage for '$$repo' dir:" && make coverage); done From 2933961042fc6a586de23e4f1d13578e74e359e9 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 24 May 2012 16:48:28 -0700 Subject: [PATCH 129/149] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index e0da8d0e28..3ee8d4b323 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit e0da8d0e284bbebbaef711c91c1b961580f225d2 +Subproject commit 3ee8d4b3232d74ed7bd475819193ad3a4055e2f5 From d2c756cac43f0fbaa2ac05844ab68c245c47e6a7 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 24 May 2012 17:33:02 -0700 Subject: [PATCH 130/149] Make tests even quieter. --- testing/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/Makefile b/testing/Makefile index c4a2aeddda..d56ee4e0e1 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -12,7 +12,7 @@ make-brief: @for repo in $(DIRS); do (cd $$repo && make -s brief ); done coverage: - @for repo in $(DIRS); do (cd $$repo && echo "Coverage for '$$repo' dir:" && make coverage); done + @for repo in $(DIRS); do (cd $$repo && echo "Coverage for '$$repo' dir:" && make -s coverage); done @test -f btest/coverage.log && cp btest/coverage.log `mktemp brocov.tmp.XXX` || true @for f in external/*/coverage.log; do test -f $$f && cp $$f `mktemp brocov.tmp.XXX` || true; done @echo "Complete test suite code coverage:" From 3d2009cacfda99acefd16ed64c5e5f8da4f43d10 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 24 May 2012 17:43:35 -0700 Subject: [PATCH 131/149] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- aux/broctl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broccoli b/aux/broccoli index 95c93494d7..07866915a1 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 95c93494d7192f69d30f208c4caa3bd38adda6fd +Subproject commit 07866915a1450ddd25b888917f494b4824b0cc3f diff --git a/aux/broctl b/aux/broctl index ba9e1aa2f2..892b60edb9 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit ba9e1aa2f2159deac0cf96863f54405643764df0 +Subproject commit 892b60edb967bb456872638f22ba994e84530137 From b5417a32be4cfe4babdf26f942b3b2b84e785781 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 25 May 2012 08:31:06 -0700 Subject: [PATCH 132/149] Some tweaks to the DS doc. Also including a section with deficiencies. --- doc/logging-dataseries.rst | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/doc/logging-dataseries.rst b/doc/logging-dataseries.rst index b41b9fb0b7..554600f055 100644 --- a/doc/logging-dataseries.rst +++ b/doc/logging-dataseries.rst @@ -6,8 +6,8 @@ Binary Output with DataSeries .. rst-class:: opening Bro's default ASCII log format is not exactly the most efficient - way for storing large volumes of data. An an alternative, Bro comes - with experimental support for `DataSeries + way for storing and searching large volumes of data. An an + alternative, Bro comes with experimental support for `DataSeries `_ output, an efficient binary format for recording structured bulk data. DataSeries is developed and maintained at HP Labs. @@ -35,9 +35,12 @@ To build and install the two into ````, do:: Please refer to the packages' documentation for more information about the installation process. In particular, there's more information on required and optional `dependencies for Lintel -`_ +`_ and `dependencies for DataSeries -`_ +`_. +For users on RedHat-style systems, you'll need the following:: + + yum install libxml2-devel boost-devel Compiling Bro with DataSeries Support ------------------------------------- @@ -166,3 +169,18 @@ with the output files. The ``man`` pages for these tool show further options, and their ``-h`` option gives some more information (either can be a bit cryptic unfortunately though). + +Deficiencies +------------ + +Due to limitations of the DataSeries format, one cannot inspect its +files before they have been fully written. In other words, when using +DataSeries, it's currently it's not possible to inspect the live log +files inside the spool directory before they are rotated to their +final location. It seems that this could be fixed with some effort, +and we will work with DataSeries development team on that if the +format gains traction among Bro users. + +Likewise, we're considering writing custom command line tools for +interacting with DataSeries files, making that a bit more convenient +than what the standard utilities provide. From da34266a526eb08f757fe4a6a7bf61fbdb82a3d6 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 25 May 2012 08:36:36 -0700 Subject: [PATCH 133/149] Switching default DS compression to gzip. --- scripts/base/frameworks/logging/writers/dataseries.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/frameworks/logging/writers/dataseries.bro b/scripts/base/frameworks/logging/writers/dataseries.bro index ccee500c3a..e85d9c8c49 100644 --- a/scripts/base/frameworks/logging/writers/dataseries.bro +++ b/scripts/base/frameworks/logging/writers/dataseries.bro @@ -10,7 +10,7 @@ export { ## 'lzo' -- LZO compression. Very fast decompression times. ## 'gz' -- GZIP compression. Slower than LZF, but also produces smaller output. ## 'bz2' -- BZIP2 compression. Slower than GZIP, but also produces smaller output. - const compression = "lzo" &redef; + const compression = "gz" &redef; ## The extent buffer size. ## Larger values here lead to better compression and more efficient writes, but From 2034c10e9766d833e7bd32d011c5557ab1f90a7f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 10:33:22 -0700 Subject: [PATCH 134/149] make input framework source (hopefully) adhere to the usual indentation style. No functional changes. --- src/input/Manager.cc | 1224 ++++++++++++++++++-------------- src/input/ReaderBackend.cc | 148 ++-- src/input/ReaderFrontend.cc | 40 +- src/input/readers/Ascii.cc | 314 ++++---- src/input/readers/Benchmark.cc | 118 ++- src/input/readers/Raw.cc | 155 ++-- 6 files changed, 1102 insertions(+), 897 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 6b179f66a1..0fde16b87d 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -24,25 +24,30 @@ using threading::Value; using threading::Field; /** - * InputHashes are used as Dictionaries to store the value and index hashes for all lines currently stored in a table. - * Index hash is stored as HashKey*, because it is thrown into other Bro functions that need the complex structure of it. - * For everything we do (with values), we just take the hash_t value and compare it directly with == + * InputHashes are used as Dictionaries to store the value and index hashes for all + * lines currently stored in a table. Index hash is stored as HashKey*, because it is + * thrown into other Bro functions that need the complex structure of it. + * For everything we do (with values), we just take the hash_t value and compare it + * directly with == */ -struct InputHash { +struct InputHash + { hash_t valhash; HashKey* idxkey; ~InputHash(); -}; + }; -InputHash::~InputHash() { +InputHash::~InputHash() + { if ( idxkey ) delete idxkey; -} + } -static void input_hash_delete_func(void* val) { +static void input_hash_delete_func(void* val) + { InputHash* h = (InputHash*) val; delete h; -} + } declare(PDict, InputHash); @@ -68,14 +73,16 @@ public: virtual ~Stream(); }; -Manager::Stream::Stream() { +Manager::Stream::Stream() + { type = 0; reader = 0; description = 0; removed = false; -} + } -Manager::Stream::~Stream() { +Manager::Stream::~Stream() + { if ( type ) Unref(type); if ( description ) @@ -83,7 +90,7 @@ Manager::Stream::~Stream() { if ( reader ) delete(reader); -} + } class Manager::TableStream: public Manager::Stream { public: @@ -120,7 +127,8 @@ public: ~EventStream(); }; -Manager::TableStream::TableStream() : Manager::Stream::Stream() { +Manager::TableStream::TableStream() : Manager::Stream::Stream() + { filter_type = TABLE_FILTER; tab = 0; @@ -131,20 +139,22 @@ Manager::TableStream::TableStream() : Manager::Stream::Stream() { lastDict = 0; pred = 0; -} + } -Manager::EventStream::EventStream() : Manager::Stream::Stream() { +Manager::EventStream::EventStream() : Manager::Stream::Stream() + { fields = 0; filter_type = EVENT_FILTER; -} + } -Manager::EventStream::~EventStream() { - if ( fields ) { +Manager::EventStream::~EventStream() + { + if ( fields ) Unref(fields); - } -} + } -Manager::TableStream::~TableStream() { +Manager::TableStream::~TableStream() + { if ( tab ) Unref(tab); if ( itype ) @@ -152,22 +162,24 @@ Manager::TableStream::~TableStream() { if ( rtype ) // can be 0 for sets Unref(rtype); - if ( currDict != 0 ) { + if ( currDict != 0 ) + { currDict->Clear(); delete currDict; - } + } - if ( lastDict != 0 ) { + if ( lastDict != 0 ) + { lastDict->Clear();; delete lastDict; - } -} + } + } struct ReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages bool (*init)(); // optional one-time inifializing function - ReaderBackend* (*factory)(ReaderFrontend* frontend); // factory function for creating instances + ReaderBackend* (*factory)(ReaderFrontend* frontend); // factory function for creating instances }; ReaderDefinition input_readers[] = { @@ -180,49 +192,55 @@ ReaderDefinition input_readers[] = { }; Manager::Manager() -{ -} - -Manager::~Manager() { - for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { - delete s->second; - delete s->first; + { } -} - -ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) { - ReaderDefinition* ir = input_readers; - - while ( true ) { - if ( ir->type == BifEnum::Input::READER_DEFAULT ) +Manager::~Manager() + { + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { - reporter->Error("The reader that was requested was not found and could not be initialized."); - return 0; + delete s->second; + delete s->first; } - if ( ir->type != type ) { + } + +ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) + { + ReaderDefinition* ir = input_readers; + + while ( true ) + { + if ( ir->type == BifEnum::Input::READER_DEFAULT ) + { + reporter->Error("The reader that was requested was not found and could not be initialized."); + return 0; + } + + if ( ir->type != type ) + { // no, didn't find the right one... ++ir; continue; - } + } // call init function of writer if presnt if ( ir->init ) - { + { if ( (*ir->init)() ) { - //clear it to be not called again - ir->init = 0; - } else { + //clear it to be not called again + ir->init = 0; + } + else { // ohok. init failed, kill factory for all eternity ir->factory = 0; DBG_LOG(DBG_LOGGING, "Failed to init input class %s", ir->name); return 0; } - } + } if ( !ir->factory ) // no factory? @@ -230,7 +248,8 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) // all done. break. break; - } + } + assert(ir->factory); ReaderBackend* backend = (*ir->factory)(frontend); @@ -238,31 +257,34 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) frontend->ty_name = ir->name; return backend; -} + } // create a new input reader object to be used at whomevers leisure lateron. bool Manager::CreateStream(Stream* info, RecordVal* description) -{ + { ReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); - if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) || same_type(rtype, BifType::Record::Input::EventDescription, 0) ) ) - { + if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) + || same_type(rtype, BifType::Record::Input::EventDescription, 0) ) ) + { reporter->Error("Streamdescription argument not of right type for new input stream"); return false; - } + } Val* name_val = description->LookupWithDefault(rtype->FieldOffset("name")); string name = name_val->AsString()->CheckString(); Unref(name_val); - { + { Stream *i = FindStream(name); - if ( i != 0 ) { - reporter->Error("Trying create already existing input stream %s", name.c_str()); + if ( i != 0 ) + { + reporter->Error("Trying create already existing input stream %s", + name.c_str()); return false; + } } - } EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); Val *autostart = description->LookupWithDefault(rtype->FieldOffset("autostart")); @@ -293,25 +315,27 @@ bool Manager::CreateStream(Stream* info, RecordVal* description) return true; -} + } -bool Manager::CreateEventStream(RecordVal* fval) { +bool Manager::CreateEventStream(RecordVal* fval) + { RecordType* rtype = fval->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::EventDescription, 0) ) - { + { reporter->Error("filter argument not of right type"); return false; - } + } EventStream* filter = new EventStream(); - { + { bool res = CreateStream(filter, fval); - if ( res == false ) { + if ( res == false ) + { delete filter; return false; + } } - } RecordType *fields = fval->LookupWithDefault(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); @@ -322,77 +346,87 @@ bool Manager::CreateEventStream(RecordVal* fval) { Func* event = event_val->AsFunc(); Unref(event_val); - { + { FuncType* etype = event->FType()->AsFuncType(); - if ( ! etype->IsEvent() ) { + if ( ! etype->IsEvent() ) + { reporter->Error("stream event is a function, not an event"); return false; - } + } const type_list* args = etype->ArgTypes()->Types(); - if ( args->length() < 2 ) { + if ( args->length() < 2 ) + { reporter->Error("event takes not enough arguments"); return false; - } + } if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) - { + { reporter->Error("events second attribute must be of type Input::Event"); return false; - } + } if ( ! same_type((*args)[0], BifType::Record::Input::EventDescription, 0) ) - { + { reporter->Error("events first attribute must be of type Input::EventDescription"); return false; - } + } - if ( want_record->InternalInt() == 0 ) { - if ( args->length() != fields->NumFields() + 2 ) { + if ( want_record->InternalInt() == 0 ) + { + if ( args->length() != fields->NumFields() + 2 ) + { reporter->Error("event has wrong number of arguments"); return false; - } + } - for ( int i = 0; i < fields->NumFields(); i++ ) { - if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) { + for ( int i = 0; i < fields->NumFields(); i++ ) + { + if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) + { reporter->Error("Incompatible type for event"); return false; + } } - } - } else if ( want_record->InternalInt() == 1 ) { - if ( args->length() != 3 ) { + } + else if ( want_record->InternalInt() == 1 ) + { + if ( args->length() != 3 ) + { reporter->Error("event has wrong number of arguments"); return false; - } + } - if ( !same_type((*args)[2], fields ) ) { + if ( !same_type((*args)[2], fields ) ) + { reporter->Error("Incompatible type for event"); return false; - } + } - } else { + } + else assert(false); - } - } + } vector fieldsV; // vector, because UnrollRecordType needs it bool status = !UnrollRecordType(&fieldsV, fields, ""); - if ( status ) { + if ( status ) + { reporter->Error("Problem unrolling"); return false; - } + } Field** logf = new Field*[fieldsV.size()]; - for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) logf[i] = fieldsV[i]; - } Unref(fields); // ref'd by lookupwithdefault filter->num_fields = fieldsV.size(); @@ -412,56 +446,64 @@ bool Manager::CreateEventStream(RecordVal* fval) { return true; } -bool Manager::CreateTableStream(RecordVal* fval) { +bool Manager::CreateTableStream(RecordVal* fval) + { RecordType* rtype = fval->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::TableDescription, 0) ) - { + { reporter->Error("filter argument not of right type"); return false; - } + } TableStream* filter = new TableStream(); - { + { bool res = CreateStream(filter, fval); - if ( res == false ) { + if ( res == false ) + { delete filter; return false; + } } - } Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = 0; - if ( fval->LookupWithDefault(rtype->FieldOffset("val")) != 0 ) { + + if ( fval->LookupWithDefault(rtype->FieldOffset("val")) != 0 ) + { val = fval->LookupWithDefault(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); Unref(val); // The lookupwithdefault in the if-clause ref'ed val. - } + } + TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); // check if index fields match table description - { + { int num = idx->NumFields(); const type_list* tl = dst->Type()->AsTableType()->IndexTypes(); loop_over_list(*tl, j) { - if ( j >= num ) { + if ( j >= num ) + { reporter->Error("Table type has more indexes than index definition"); return false; - } + } - if ( !same_type(idx->FieldType(j), (*tl)[j]) ) { + if ( !same_type(idx->FieldType(j), (*tl)[j]) ) + { reporter->Error("Table type does not match index type"); return false; - } + } } - if ( num != j ) { + if ( num != j ) + { reporter->Error("Table has less elements than index definition"); return false; + } } - } Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); @@ -469,51 +511,57 @@ bool Manager::CreateTableStream(RecordVal* fval) { Func* event = event_val ? event_val->AsFunc() : 0; Unref(event_val); - if ( event ) { + if ( event ) + { FuncType* etype = event->FType()->AsFuncType(); - if ( ! etype->IsEvent() ) { + if ( ! etype->IsEvent() ) + { reporter->Error("stream event is a function, not an event"); return false; - } + } const type_list* args = etype->ArgTypes()->Types(); if ( args->length() != 4 ) - { + { reporter->Error("Table event must take 4 arguments"); return false; - } + } if ( ! same_type((*args)[0], BifType::Record::Input::TableDescription, 0) ) - { + { reporter->Error("table events first attribute must be of type Input::TableDescription"); return false; - } + } if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) - { + { reporter->Error("table events second attribute must be of type Input::Event"); return false; - } + } if ( ! same_type((*args)[2], idx) ) - { + { reporter->Error("table events index attributes do not match"); return false; - } + } if ( want_record->InternalInt() == 1 && ! same_type((*args)[3], val) ) - { + { reporter->Error("table events value attributes do not match"); return false; - } else if ( want_record->InternalInt() == 0 && !same_type((*args)[3], val->FieldType(0) ) ) { + } + else if ( want_record->InternalInt() == 0 + && !same_type((*args)[3], val->FieldType(0) ) ) + { reporter->Error("table events value attribute does not match"); return false; - } + } + assert(want_record->InternalInt() == 1 || want_record->InternalInt() == 0); - } + } vector fieldsV; // vector, because we don't know the length beforehands @@ -529,16 +577,16 @@ bool Manager::CreateTableStream(RecordVal* fval) { if ( !val ) assert(valfields == 0); - if ( status ) { + if ( status ) + { reporter->Error("Problem unrolling"); return false; - } + } Field** fields = new Field*[fieldsV.size()]; - for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) fields[i] = fieldsV[i]; - } filter->pred = pred ? pred->AsFunc() : 0; filter->num_idx_fields = idxfields; @@ -556,13 +604,15 @@ bool Manager::CreateTableStream(RecordVal* fval) { Unref(want_record); // ref'd by lookupwithdefault Unref(pred); - if ( valfields > 1 ) { - if ( ! filter->want_record ) { + if ( valfields > 1 ) + { + if ( ! filter->want_record ) + { reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", filter->name.c_str()); delete filter; return false; + } } - } assert(filter->reader); @@ -574,7 +624,7 @@ bool Manager::CreateTableStream(RecordVal* fval) { filter->name.c_str()); return true; -} + } bool Manager::IsCompatibleType(BroType* t, bool atomic_only) @@ -582,7 +632,7 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) if ( ! t ) return false; - switch ( t->Tag() ) { + switch ( t->Tag() ) { case TYPE_BOOL: case TYPE_INT: case TYPE_COUNT: @@ -624,20 +674,21 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) } return false; -} + } -bool Manager::RemoveStream(const string &name) { +bool Manager::RemoveStream(const string &name) + { Stream *i = FindStream(name); - if ( i == 0 ) { + if ( i == 0 ) return false; // not found - } - if ( i->removed ) { + if ( i->removed ) + { reporter->Error("Stream %s is already queued for removal. Ignoring remove.", name.c_str()); return false; - } + } i->removed = true; @@ -649,54 +700,66 @@ bool Manager::RemoveStream(const string &name) { #endif return true; -} - -bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) { - Stream *i = FindStream(reader); - - if ( i == 0 ) { - reporter->Error("Stream not found in RemoveStreamContinuation"); - return false; } +bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) + { + Stream *i = FindStream(reader); + + if ( i == 0 ) + { + reporter->Error("Stream not found in RemoveStreamContinuation"); + return false; + } #ifdef DEBUG DBG_LOG(DBG_INPUT, "Successfully executed removal of stream %s", i->name.c_str()); #endif + readers.erase(reader); delete(i); return true; -} + } -bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { - for ( int i = 0; i < rec->NumFields(); i++ ) +bool Manager::UnrollRecordType(vector *fields, + const RecordType *rec, const string& nameprepend) { - if ( !IsCompatibleType(rec->FieldType(i)) ) { + for ( int i = 0; i < rec->NumFields(); i++ ) + { + + if ( !IsCompatibleType(rec->FieldType(i)) ) + { reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); return false; - } + } if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) - { + { string prep = nameprepend + rec->FieldName(i) + "."; if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) - { + { return false; - } + } - } else { + } + else + { Field* field = new Field(); field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); - if ( field->type == TYPE_TABLE ) { + if ( field->type == TYPE_TABLE ) + { field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); - } else if ( field->type == TYPE_VECTOR ) { + } + else if ( field->type == TYPE_VECTOR ) + { field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); - } else if ( field->type == TYPE_PORT && - rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) { + } else if ( field->type == TYPE_PORT && + rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) + { // we have an annotation for the second column Val* c = rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN)->AttrExpr()->Eval(0); @@ -705,31 +768,32 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co assert(c->Type()->Tag() == TYPE_STRING); field->secondary_name = c->AsStringVal()->AsString()->CheckString(); - } + } - if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) { + if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) field->optional = true; - } fields->push_back(field); + } } - } return true; -} + } bool Manager::ForceUpdate(const string &name) -{ + { Stream *i = FindStream(name); - if ( i == 0 ) { + if ( i == 0 ) + { reporter->Error("Stream %s not found", name.c_str()); return false; - } + } - if ( i->removed ) { + if ( i->removed ) + { reporter->Error("Stream %s is already queued for removal. Ignoring force update.", name.c_str()); return false; - } + } i->reader->Update(); @@ -742,31 +806,34 @@ bool Manager::ForceUpdate(const string &name) } -Val* Manager::RecordValToIndexVal(RecordVal *r) { +Val* Manager::RecordValToIndexVal(RecordVal *r) + { Val* idxval; RecordType *type = r->Type()->AsRecordType(); int num_fields = type->NumFields(); - if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) { + if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) + { idxval = r->LookupWithDefault(0); - } else { + } + else + { ListVal *l = new ListVal(TYPE_ANY); - for ( int j = 0 ; j < num_fields; j++ ) { - //Val* rval = r->Lookup(j); - //assert(rval != 0); + for ( int j = 0 ; j < num_fields; j++ ) l->Append(r->LookupWithDefault(j)); - } + idxval = l; - } + } return idxval; -} + } -Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) { +Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) + { Val* idxval; int position = 0; @@ -776,47 +843,54 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); - for ( int j = 0 ; j < type->NumFields(); j++ ) { - if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { - l->Append(ValueToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); - } else { + for ( int j = 0 ; j < type->NumFields(); j++ ) + { + if ( type->FieldType(j)->Tag() == TYPE_RECORD ) + l->Append(ValueToRecordVal(vals, + type->FieldType(j)->AsRecordType(), &position)); + else + { l->Append(ValueToVal(vals[position], type->FieldType(j))); position++; + } } - } idxval = l; - } + } assert ( position == num_fields ); return idxval; -} + } -void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { +void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) + { Stream *i = FindStream(reader); - if ( i == 0 ) { + if ( i == 0 ) + { reporter->InternalError("Unknown reader in SendEntry"); return; - } + } int readFields; - if ( i->filter_type == TABLE_FILTER ) { + if ( i->filter_type == TABLE_FILTER ) readFields = SendEntryTable(i, vals); - } else if ( i->filter_type == EVENT_FILTER ) { + else if ( i->filter_type == EVENT_FILTER ) + { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); readFields = SendEventStreamEvent(i, type, vals); - } else { + } + else assert(false); - } - for ( int i = 0; i < readFields; i++ ) { + for ( int i = 0; i < readFields; i++ ) delete vals[i]; - } - delete [] vals; -} -int Manager::SendEntryTable(Stream* i, const Value* const *vals) { + delete [] vals; + } + +int Manager::SendEntryTable(Stream* i, const Value* const *vals) + { bool updated = false; assert(i); @@ -826,59 +900,66 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { HashKey* idxhash = HashValues(filter->num_idx_fields, vals); - if ( idxhash == 0 ) { + if ( idxhash == 0 ) + { reporter->Error("Could not hash line. Ignoring"); return filter->num_val_fields + filter->num_idx_fields; - } + } hash_t valhash = 0; - if ( filter->num_val_fields > 0 ) { + if ( filter->num_val_fields > 0 ) + { HashKey* valhashkey = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); if ( valhashkey == 0 ) { // empty line. index, but no values. // hence we also have no hash value... - } else { + } + else + { valhash = valhashkey->Hash(); delete(valhashkey); + } } - } InputHash *h = filter->lastDict->Lookup(idxhash); - if ( h != 0 ) { + if ( h != 0 ) + { // seen before - if ( filter->num_val_fields == 0 || h->valhash == valhash ) { + if ( filter->num_val_fields == 0 || h->valhash == valhash ) + { // ok, exact duplicate, move entry to new dicrionary and do nothing else. filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); delete idxhash; return filter->num_val_fields + filter->num_idx_fields; - } else { + } + else + { assert( filter->num_val_fields > 0 ); // entry was updated in some way filter->lastDict->Remove(idxhash); // keep h for predicates updated = true; + } } - } Val* valval; RecordVal* predidx = 0; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 0 ) { + if ( filter->num_val_fields == 0 ) valval = 0; - } else if ( filter->num_val_fields == 1 && !filter->want_record ) { + else if ( filter->num_val_fields == 1 && !filter->want_record ) valval = ValueToVal(vals[position], filter->rtype->FieldType(0)); - } else { + else valval = ValueToRecordVal(vals, filter->rtype, &position); - } - // call filter first to determine if we really add / change the entry - if ( filter->pred ) { + if ( filter->pred ) + { EnumVal* ev; //Ref(idxval); int startpos = 0; @@ -886,68 +967,74 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { predidx = ValueToRecordVal(vals, filter->itype, &startpos); //ValueToRecordVal(vals, filter->itype, &startpos); - if ( updated ) { + if ( updated ) ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); - } else { + else ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - } bool result; - if ( filter->num_val_fields > 0 ) { // we have values + if ( filter->num_val_fields > 0 ) // we have values result = CallPred(filter->pred, 3, ev, predidx->Ref(), valval->Ref()); - } else { - // no values + else // no values result = CallPred(filter->pred, 2, ev, predidx->Ref()); - } - if ( result == false ) { + if ( result == false ) + { Unref(predidx); Unref(valval); - if ( !updated ) { + if ( !updated ) + { // throw away. Hence - we quit. And remove the entry from the current dictionary... // (but why should it be in there? assert this). assert ( filter->currDict->RemoveEntry(idxhash) == 0 ); delete idxhash; delete h; return filter->num_val_fields + filter->num_idx_fields; - } else { + } + else + { // keep old one filter->currDict->Insert(idxhash, h); delete idxhash; return filter->num_val_fields + filter->num_idx_fields; + } } - } - } + } // now we don't need h anymore - if we are here, the entry is updated and a new h is created. - if ( h ) { + if ( h ) + { delete h; h = 0; - } + } Val* idxval; - if ( predidx != 0 ) { + if ( predidx != 0 ) + { idxval = RecordValToIndexVal(predidx); // I think there is an unref missing here. But if I insert is, it crashes :) - } else { + } + else idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); - } + Val* oldval = 0; - if ( updated == true ) { + if ( updated == true ) + { assert(filter->num_val_fields > 0); // in that case, we need the old value to send the event (if we send an event). oldval = filter->tab->Lookup(idxval, false); - } + } //i->tab->Assign(idxval, valval); assert(idxval); HashKey* k = filter->tab->ComputeHash(idxval); - if ( !k ) { + if ( !k ) + { reporter->InternalError("could not hash"); assert(false); - } + } InputHash* ih = new InputHash(); ih->idxkey = new HashKey(k->Key(), k->Size(), k->Hash()); @@ -955,57 +1042,64 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { if ( filter->event && updated ) Ref(oldval); // otherwise it is no longer accessible after the assignment + filter->tab->Assign(idxval, k, valval); Unref(idxval); // asssign does not consume idxval. - if ( predidx != 0 ) { + if ( predidx != 0 ) Unref(predidx); - } filter->currDict->Insert(idxhash, ih); delete idxhash; - if ( filter->event ) { + if ( filter->event ) + { EnumVal* ev; int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - if ( updated ) { // in case of update send back the old value. + if ( updated ) + { // in case of update send back the old value. assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); - } else { + } + else + { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - if ( filter->num_val_fields == 0 ) { + if ( filter->num_val_fields == 0 ) + { Ref(filter->description); SendEvent(filter->event, 3, filter->description->Ref(), ev, predidx); - } else { + } + else SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval->Ref()); + } - } - } + } return filter->num_val_fields + filter->num_idx_fields; -} + } -void Manager::EndCurrentSend(ReaderFrontend* reader) { +void Manager::EndCurrentSend(ReaderFrontend* reader) + { Stream *i = FindStream(reader); - if ( i == 0 ) { + + if ( i == 0 ) + { reporter->InternalError("Unknown reader in EndCurrentSend"); return; - } + } #ifdef DEBUG DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", i->name.c_str()); #endif - if ( i->filter_type == EVENT_FILTER ) { - // nothing to do.. + if ( i->filter_type == EVENT_FILTER ) // nothing to do.. return; - } assert(i->filter_type == TABLE_FILTER); TableStream* filter = (TableStream*) i; @@ -1016,8 +1110,8 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) { - + while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) + { ListVal * idx = 0; Val *val = 0; @@ -1025,16 +1119,18 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { EnumVal* ev = 0; int startpos = 0; - if ( filter->pred || filter->event ) { + if ( filter->pred || filter->event ) + { idx = filter->tab->RecoverIndex(ih->idxkey); assert(idx != 0); val = filter->tab->Lookup(idx); assert(val != 0); predidx = ListValToRecordVal(idx, filter->itype, &startpos); ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - } + } - if ( filter->pred ) { + if ( filter->pred ) + { // ask predicate, if we want to expire this element... Ref(ev); @@ -1043,7 +1139,8 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { bool result = CallPred(filter->pred, 3, ev, predidx, val); - if ( result == false ) { + if ( result == false ) + { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... Unref(predidx); @@ -1051,15 +1148,16 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); delete lastDictIdxKey; continue; + } } - } - if ( filter->event ) { + if ( filter->event ) + { Ref(predidx); Ref(val); Ref(ev); SendEvent(filter->event, 3, ev, predidx, val); - } + } if ( predidx ) // if we have a filter or an event... Unref(predidx); @@ -1070,9 +1168,9 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { filter->lastDict->Remove(lastDictIdxKey); // delete in next line delete lastDictIdxKey; delete(ih); - } + } - filter->lastDict->Clear(); // should be empty->->-> but->->-> well->->-> who knows->->-> + filter->lastDict->Clear(); // should be empt. buti- well... who knows... delete(filter->lastDict); filter->lastDict = filter->currDict; @@ -1086,39 +1184,40 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { // Send event that the current update is indeed finished. EventHandler* handler = event_registry->Lookup("Input::update_finished"); - if ( handler == 0 ) { + if ( handler == 0 ) reporter->InternalError("Input::update_finished not found!"); - } - SendEvent(handler, 2, new StringVal(i->name.c_str()), new StringVal(i->source.c_str())); -} + } -void Manager::Put(ReaderFrontend* reader, Value* *vals) { +void Manager::Put(ReaderFrontend* reader, Value* *vals) + { Stream *i = FindStream(reader); - if ( i == 0 ) { + if ( i == 0 ) + { reporter->InternalError("Unknown reader in Put"); return; - } + } int readFields; - if ( i->filter_type == TABLE_FILTER ) { + if ( i->filter_type == TABLE_FILTER ) readFields = PutTable(i, vals); - } else if ( i->filter_type == EVENT_FILTER ) { + else if ( i->filter_type == EVENT_FILTER ) + { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); readFields = SendEventStreamEvent(i, type, vals); - } else { + } + else assert(false); - } - for ( int i = 0; i < readFields; i++ ) { + for ( int i = 0; i < readFields; i++ ) delete vals[i]; - } + delete [] vals; + } -} - -int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) { +int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) + { assert(i); assert(i->filter_type == EVENT_FILTER); @@ -1133,29 +1232,37 @@ int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const * out_vals.push_back(type); int position = 0; - if ( filter->want_record ) { + if ( filter->want_record ) + { RecordVal * r = ValueToRecordVal(vals, filter->fields, &position); out_vals.push_back(r); - } else { - for ( int j = 0; j < filter->fields->NumFields(); j++) { + } + else + { + for ( int j = 0; j < filter->fields->NumFields(); j++) + { Val* val = 0; - if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) { - val = ValueToRecordVal(vals, filter->fields->FieldType(j)->AsRecordType(), &position); - } else { + if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) + val = ValueToRecordVal(vals, + filter->fields->FieldType(j)->AsRecordType(), + &position); + else + { val = ValueToVal(vals[position], filter->fields->FieldType(j)); position++; - } + } out_vals.push_back(val); + } } - } SendEvent(filter->event, out_vals); return filter->fields->NumFields(); -} + } -int Manager::PutTable(Stream* i, const Value* const *vals) { +int Manager::PutTable(Stream* i, const Value* const *vals) + { assert(i); assert(i->filter_type == TABLE_FILTER); @@ -1165,102 +1272,115 @@ int Manager::PutTable(Stream* i, const Value* const *vals) { Val* valval; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 0 ) { + if ( filter->num_val_fields == 0 ) valval = 0; - } else if ( filter->num_val_fields == 1 && filter->want_record == 0 ) { + else if ( filter->num_val_fields == 1 && filter->want_record == 0 ) valval = ValueToVal(vals[position], filter->rtype->FieldType(0)); - } else { + else valval = ValueToRecordVal(vals, filter->rtype, &position); - } // if we have a subscribed event, we need to figure out, if this is an update or not // same for predicates - if ( filter->pred || filter->event ) { + if ( filter->pred || filter->event ) + { bool updated = false; Val* oldval = 0; - if ( filter->num_val_fields > 0 ) { + if ( filter->num_val_fields > 0 ) + { // in that case, we need the old value to send the event (if we send an event). oldval = filter->tab->Lookup(idxval, false); - } + } - if ( oldval != 0 ) { + if ( oldval != 0 ) + { // it is an update updated = true; Ref(oldval); // have to do that, otherwise it may disappear in assign - } + } // predicate if we want the update or not - if ( filter->pred ) { + if ( filter->pred ) + { EnumVal* ev; int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); Ref(valval); - if ( updated ) { - ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); - } else { - ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - } + if ( updated ) + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, + BifType::Enum::Input::Event); + else + ev = new EnumVal(BifEnum::Input::EVENT_NEW, + BifType::Enum::Input::Event); bool result; - if ( filter->num_val_fields > 0 ) { // we have values + if ( filter->num_val_fields > 0 ) // we have values result = CallPred(filter->pred, 3, ev, predidx, valval); - } else { - // no values + else // no values result = CallPred(filter->pred, 2, ev, predidx); - } - if ( result == false ) { + if ( result == false ) + { // do nothing Unref(idxval); Unref(valval); Unref(oldval); return filter->num_val_fields + filter->num_idx_fields; - } + } - } + } filter->tab->Assign(idxval, valval); - if ( filter->event ) { + if ( filter->event ) + { EnumVal* ev; int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - if ( updated ) { // in case of update send back the old value. + if ( updated ) + { + // in case of update send back the old value. assert ( filter->num_val_fields > 0 ); - ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, + BifType::Enum::Input::Event); assert ( oldval != 0 ); - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); - } else { - ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - if ( filter->num_val_fields == 0 ) { - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx); - } else { - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval->Ref()); + SendEvent(filter->event, 4, filter->description->Ref(), + ev, predidx, oldval); + } + else + { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, + BifType::Enum::Input::Event); + if ( filter->num_val_fields == 0 ) + SendEvent(filter->event, 4, filter->description->Ref(), + ev, predidx); + else + SendEvent(filter->event, 4, filter->description->Ref(), + ev, predidx, valval->Ref()); } + } - } - - - } else { - // no predicates or other stuff + } + else // no predicates or other stuff filter->tab->Assign(idxval, valval); - } + return filter->num_idx_fields + filter->num_val_fields; -} + } // Todo:: perhaps throw some kind of clear-event? -void Manager::Clear(ReaderFrontend* reader) { +void Manager::Clear(ReaderFrontend* reader) + { Stream *i = FindStream(reader); - if ( i == 0 ) { + if ( i == 0 ) + { reporter->InternalError("Unknown reader in Clear"); return; - } + } #ifdef DEBUG DBG_LOG(DBG_INPUT, "Got Clear for stream %s", @@ -1271,30 +1391,35 @@ void Manager::Clear(ReaderFrontend* reader) { TableStream* filter = (TableStream*) i; filter->tab->RemoveAll(); -} + } // put interface: delete old entry from table. -bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { +bool Manager::Delete(ReaderFrontend* reader, Value* *vals) + { Stream *i = FindStream(reader); - if ( i == 0 ) { + if ( i == 0 ) + { reporter->InternalError("Unknown reader in Delete"); return false; - } + } bool success = false; int readVals = 0; - if ( i->filter_type == TABLE_FILTER ) { + if ( i->filter_type == TABLE_FILTER ) + { TableStream* filter = (TableStream*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); assert(idxval != 0); readVals = filter->num_idx_fields + filter->num_val_fields; bool filterresult = true; - if ( filter->pred || filter->event ) { + if ( filter->pred || filter->event ) + { Val *val = filter->tab->Lookup(idxval); - if ( filter->pred ) { + if ( filter->pred ) + { Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); int startpos = 0; @@ -1302,62 +1427,68 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { filterresult = CallPred(filter->pred, 3, ev, predidx, val); - if ( filterresult == false ) { + if ( filterresult == false ) + { // keep it. Unref(idxval); success = true; + } + } - } - // only if filter = true -> no filtering - if ( filterresult && filter->event ) { + if ( filterresult && filter->event ) + { Ref(idxval); assert(val != 0); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); SendEvent(filter->event, 4, filter->description->Ref(), ev, idxval, val); + } } - } // only if filter = true -> no filtering - if ( filterresult ) { + if ( filterresult ) + { Val* retptr = filter->tab->Delete(idxval); success = ( retptr != 0 ); - if ( !success ) { + if ( !success ) reporter->Error("Internal error while deleting values from input table"); - } else { + else Unref(retptr); } - } - } else if ( i->filter_type == EVENT_FILTER ) { + + } + else if ( i->filter_type == EVENT_FILTER ) + { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); readVals = SendEventStreamEvent(i, type, vals); success = true; - } else { + } + else + { assert(false); return false; - } + } - for ( int i = 0; i < readVals; i++ ) { + for ( int i = 0; i < readVals; i++ ) delete vals[i]; - } + delete [] vals; return success; -} + } bool Manager::CallPred(Func* pred_func, const int numvals, ...) -{ + { bool result; val_list vl(numvals); va_list lP; va_start(lP, numvals); for ( int i = 0; i < numvals; i++ ) - { vl.append( va_arg(lP, Val*) ); - } + va_end(lP); Val* v = pred_func->Call(&vl); @@ -1365,120 +1496,131 @@ bool Manager::CallPred(Func* pred_func, const int numvals, ...) Unref(v); return(result); -} + } bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) -{ + { EventHandler* handler = event_registry->Lookup(name.c_str()); - if ( handler == 0 ) { + if ( handler == 0 ) + { reporter->Error("Event %s not found", name.c_str()); return false; - } + } RecordType *type = handler->FType()->Args(); int num_event_vals = type->NumFields(); - if ( num_vals != num_event_vals ) { + if ( num_vals != num_event_vals ) + { reporter->Error("Wrong number of values for event %s", name.c_str()); return false; - } + } val_list* vl = new val_list; - for ( int i = 0; i < num_vals; i++) { + for ( int i = 0; i < num_vals; i++) vl->append(ValueToVal(vals[i], type->FieldType(i))); - } mgr.Dispatch(new Event(handler, vl)); - for ( int i = 0; i < num_vals; i++ ) { + for ( int i = 0; i < num_vals; i++ ) delete vals[i]; - } + delete [] vals; return true; } void Manager::SendEvent(EventHandlerPtr ev, const int numvals, ...) -{ + { val_list* vl = new val_list; va_list lP; va_start(lP, numvals); for ( int i = 0; i < numvals; i++ ) - { vl->append( va_arg(lP, Val*) ); - } + va_end(lP); mgr.QueueEvent(ev, vl, SOURCE_LOCAL); -} - -void Manager::SendEvent(EventHandlerPtr ev, list events) -{ - val_list* vl = new val_list; - - for ( list::iterator i = events.begin(); i != events.end(); i++ ) { - vl->append( *i ); } - mgr.QueueEvent(ev, vl, SOURCE_LOCAL); -} +void Manager::SendEvent(EventHandlerPtr ev, list events) + { + val_list* vl = new val_list; + + for ( list::iterator i = events.begin(); i != events.end(); i++ ) + { + vl->append( *i ); + } -// Convert a bro list value to a bro record value. I / we could think about moving this functionality to val.cc -RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) { + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); + } + +// Convert a bro list value to a bro record value. +// I / we could think about moving this functionality to val.cc +RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) + { assert(position != 0 ); // we need the pointer to point to data; - if ( request_type->Tag() != TYPE_RECORD ) { + if ( request_type->Tag() != TYPE_RECORD ) + { reporter->InternalError("ListValToRecordVal called on non-record-value."); return 0; - } + } RecordVal* rec = new RecordVal(request_type->AsRecordType()); assert(list != 0); int maxpos = list->Length(); - for ( int i = 0; i < request_type->NumFields(); i++ ) { + for ( int i = 0; i < request_type->NumFields(); i++ ) + { assert ( (*position) <= maxpos ); Val* fieldVal = 0; - if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) fieldVal = ListValToRecordVal(list, request_type->FieldType(i)->AsRecordType(), position); - } else { + else + { fieldVal = list->Index(*position); (*position)++; - } + } rec->Assign(i, fieldVal); - } + } return rec; -} + } // Convert a threading value to a record value -RecordVal* Manager::ValueToRecordVal(const Value* const *vals, RecordType *request_type, int* position) { +RecordVal* Manager::ValueToRecordVal(const Value* const *vals, + RecordType *request_type, int* position) + { assert(position != 0); // we need the pointer to point to data. - if ( request_type->Tag() != TYPE_RECORD ) { + if ( request_type->Tag() != TYPE_RECORD ) + { reporter->InternalError("ValueToRecordVal called on non-record-value."); return 0; - } + } RecordVal* rec = new RecordVal(request_type->AsRecordType()); - for ( int i = 0; i < request_type->NumFields(); i++ ) { + for ( int i = 0; i < request_type->NumFields(); i++ ) + { Val* fieldVal = 0; - if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); - } else { + else + { fieldVal = ValueToVal(vals[*position], request_type->FieldType(i)); (*position)++; - } + } rec->Assign(i, fieldVal); - } + } return rec; -} + } // Count the length of the values // used to create a correct length buffer for hashing later @@ -1495,7 +1637,7 @@ int Manager::GetValueLength(const Value* val) { case TYPE_COUNT: case TYPE_COUNTER: length += sizeof(val->val.uint_val); - break; + break; case TYPE_PORT: length += sizeof(val->val.port_val.port); @@ -1517,48 +1659,48 @@ int Manager::GetValueLength(const Value* val) { case TYPE_ADDR: { - switch ( val->val.addr_val.family ) { - case IPv4: - length += sizeof(val->val.addr_val.in.in4); - break; - case IPv6: - length += sizeof(val->val.addr_val.in.in6); - break; - default: - assert(false); - } - + switch ( val->val.addr_val.family ) { + case IPv4: + length += sizeof(val->val.addr_val.in.in4); + break; + case IPv6: + length += sizeof(val->val.addr_val.in.in6); + break; + default: + assert(false); + } } break; case TYPE_SUBNET: { - switch ( val->val.subnet_val.prefix.family ) { - case IPv4: - length += sizeof(val->val.subnet_val.prefix.in.in4)+sizeof(val->val.subnet_val.length); - break; - case IPv6: - length += sizeof(val->val.subnet_val.prefix.in.in6)+sizeof(val->val.subnet_val.length); - break; - default: - assert(false); - } - + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + length += sizeof(val->val.subnet_val.prefix.in.in4)+ + sizeof(val->val.subnet_val.length); + break; + case IPv6: + length += sizeof(val->val.subnet_val.prefix.in.in6)+ + sizeof(val->val.subnet_val.length); + break; + default: + assert(false); + } } break; - case TYPE_TABLE: { - for ( int i = 0; i < val->val.set_val.size; i++ ) { + case TYPE_TABLE: + { + for ( int i = 0; i < val->val.set_val.size; i++ ) length += GetValueLength(val->val.set_val.vals[i]); - } break; } - case TYPE_VECTOR: { + case TYPE_VECTOR: + { int j = val->val.vector_val.size; - for ( int i = 0; i < j; i++ ) { + for ( int i = 0; i < j; i++ ) length += GetValueLength(val->val.vector_val.vals[i]); - } break; } @@ -1572,7 +1714,8 @@ int Manager::GetValueLength(const Value* val) { // Given a threading::value, copy the raw data bytes into *data and return how many bytes were copied. // Used for hashing the values for lookup in the bro table -int Manager::CopyValue(char *data, const int startpos, const Value* val) { +int Manager::CopyValue(char *data, const int startpos, const Value* val) + { assert( val->present ); // presence has to be checked elsewhere switch ( val->type ) { @@ -1588,11 +1731,14 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { return sizeof(val->val.uint_val); break; - case TYPE_PORT: { + case TYPE_PORT: + { int length = 0; - memcpy(data+startpos, (const void*) &(val->val.port_val.port), sizeof(val->val.port_val.port)); + memcpy(data+startpos, (const void*) &(val->val.port_val.port), + sizeof(val->val.port_val.port)); length += sizeof(val->val.port_val.port); - memcpy(data+startpos+length, (const void*) &(val->val.port_val.proto), sizeof(val->val.port_val.proto)); + memcpy(data+startpos+length, (const void*) &(val->val.port_val.proto), + sizeof(val->val.port_val.proto)); length += sizeof(val->val.port_val.proto); return length; break; @@ -1602,7 +1748,8 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); + memcpy(data+startpos, (const void*) &(val->val.double_val), + sizeof(val->val.double_val)); return sizeof(val->val.double_val); break; @@ -1616,64 +1763,66 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { case TYPE_ADDR: { - int length; - switch ( val->val.addr_val.family ) { - case IPv4: - length = sizeof(val->val.addr_val.in.in4); - memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in4), length); - break; - case IPv6: - length = sizeof(val->val.addr_val.in.in6); - memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in6), length); - break; - default: - assert(false); - } - - return length; - + int length; + switch ( val->val.addr_val.family ) { + case IPv4: + length = sizeof(val->val.addr_val.in.in4); + memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in4), length); + break; + case IPv6: + length = sizeof(val->val.addr_val.in.in6); + memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in6), length); + break; + default: + assert(false); + } + return length; } break; case TYPE_SUBNET: { - int length; - switch ( val->val.subnet_val.prefix.family ) { - case IPv4: - length = sizeof(val->val.addr_val.in.in4); - memcpy(data + startpos, (const char*) &(val->val.subnet_val.prefix.in.in4), length); - break; - case IPv6: - length = sizeof(val->val.addr_val.in.in6); - memcpy(data + startpos, (const char*) &(val->val.subnet_val.prefix.in.in4), length); - break; - default: - assert(false); - } - int lengthlength = sizeof(val->val.subnet_val.length); - memcpy(data + startpos + length , (const char*) &(val->val.subnet_val.length), lengthlength); - length += lengthlength; - return length; - + int length; + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + length = sizeof(val->val.addr_val.in.in4); + memcpy(data + startpos, + (const char*) &(val->val.subnet_val.prefix.in.in4), length); + break; + case IPv6: + length = sizeof(val->val.addr_val.in.in6); + memcpy(data + startpos, + (const char*) &(val->val.subnet_val.prefix.in.in4), length); + break; + default: + assert(false); + } + int lengthlength = sizeof(val->val.subnet_val.length); + memcpy(data + startpos + length , + (const char*) &(val->val.subnet_val.length), lengthlength); + length += lengthlength; + return length; } break; - case TYPE_TABLE: { + case TYPE_TABLE: + { int length = 0; int j = val->val.set_val.size; - for ( int i = 0; i < j; i++ ) { + for ( int i = 0; i < j; i++ ) length += CopyValue(data, startpos+length, val->val.set_val.vals[i]); - } + return length; break; } - case TYPE_VECTOR: { + case TYPE_VECTOR: + { int length = 0; int j = val->val.vector_val.size; - for ( int i = 0; i < j; i++ ) { + for ( int i = 0; i < j; i++ ) length += CopyValue(data, startpos+length, val->val.vector_val.vals[i]); - } + return length; break; } @@ -1685,52 +1834,57 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { assert(false); return 0; -} + } // Hash num_elements threading values and return the HashKey for them. At least one of the vals has to be ->present. -HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { +HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) + { int length = 0; - for ( int i = 0; i < num_elements; i++ ) { + for ( int i = 0; i < num_elements; i++ ) + { const Value* val = vals[i]; if ( val->present ) length += GetValueLength(val); - } + } - if ( length == 0 ) { + if ( length == 0 ) + { reporter->Error("Input reader sent line where all elements are null values. Ignoring line"); return NULL; - } + } int position = 0; char *data = (char*) malloc(length); - if ( data == 0 ) { + if ( data == 0 ) reporter->InternalError("Could not malloc?"); - } - for ( int i = 0; i < num_elements; i++ ) { + + for ( int i = 0; i < num_elements; i++ ) + { const Value* val = vals[i]; if ( val->present ) position += CopyValue(data, position, val); - } + } HashKey *key = new HashKey(data, length); delete data; assert(position == length); return key; -} + } // convert threading value to Bro value -Val* Manager::ValueToVal(const Value* val, BroType* request_type) { +Val* Manager::ValueToVal(const Value* val, BroType* request_type) + { - if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { + if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) + { reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; - } + } - if ( !val->present ) { + if ( !val->present ) return 0; // unset field - } switch ( val->type ) { case TYPE_BOOL: @@ -1762,72 +1916,73 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { case TYPE_ADDR: { - IPAddr* addr; - switch ( val->val.addr_val.family ) { - case IPv4: - addr = new IPAddr(val->val.addr_val.in.in4); - break; - case IPv6: - addr = new IPAddr(val->val.addr_val.in.in6); - break; - default: - assert(false); - } - AddrVal* addrval = new AddrVal(*addr); - delete addr; - return addrval; - + IPAddr* addr; + switch ( val->val.addr_val.family ) { + case IPv4: + addr = new IPAddr(val->val.addr_val.in.in4); + break; + case IPv6: + addr = new IPAddr(val->val.addr_val.in.in6); + break; + default: + assert(false); + } + AddrVal* addrval = new AddrVal(*addr); + delete addr; + return addrval; } case TYPE_SUBNET: { - IPAddr* addr; - switch ( val->val.subnet_val.prefix.family ) { - case IPv4: - addr = new IPAddr(val->val.subnet_val.prefix.in.in4); - break; - case IPv6: - addr = new IPAddr(val->val.subnet_val.prefix.in.in6); - break; - default: - assert(false); - } - SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length); - delete addr; - return subnetval; - + IPAddr* addr; + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + addr = new IPAddr(val->val.subnet_val.prefix.in.in4); + break; + case IPv6: + addr = new IPAddr(val->val.subnet_val.prefix.in.in6); + break; + default: + assert(false); } + SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length); + delete addr; + return subnetval; break; + } - case TYPE_TABLE: { + case TYPE_TABLE: + { // all entries have to have the same type... BroType* type = request_type->AsTableType()->Indices()->PureType(); TypeList* set_index = new TypeList(type->Ref()); set_index->Append(type->Ref()); SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); - for ( int i = 0; i < val->val.set_val.size; i++ ) { + for ( int i = 0; i < val->val.set_val.size; i++ ) + { Val* assignval = ValueToVal( val->val.set_val.vals[i], type ); t->Assign(assignval, 0); Unref(assignval); // idex is not consumed by assign. - } + } Unref(s); return t; break; } - case TYPE_VECTOR: { + case TYPE_VECTOR: + { // all entries have to have the same type... BroType* type = request_type->AsVectorType()->YieldType(); VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); - for ( int i = 0; i < val->val.vector_val.size; i++ ) { + for ( int i = 0; i < val->val.vector_val.size; i++ ) v->Assign(i, ValueToVal( val->val.set_val.vals[i], type ), 0); - } + Unref(vt); return v; - + break; } case TYPE_ENUM: { @@ -1836,9 +1991,10 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { string module = extract_module_name(val->val.string_val->c_str()); string var = extract_var_name(val->val.string_val->c_str()); bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str()); - if ( index == -1 ) { - reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", module.c_str(), var.c_str()); - } + if ( index == -1 ) + reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", + module.c_str(), var.c_str()); + return new EnumVal(index, request_type->Ref()->AsEnumType() ); break; } @@ -1850,26 +2006,24 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { assert(false); return NULL; -} + } Manager::Stream* Manager::FindStream(const string &name) { for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { if ( (*s).second->name == name ) - { return (*s).second; } - } return 0; } Manager::Stream* Manager::FindStream(ReaderFrontend* reader) -{ + { map::iterator s = readers.find(reader); - if ( s != readers.end() ) { + if ( s != readers.end() ) return s->second; - } + return 0; -} + } diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 27401ffcb8..c625301383 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -15,10 +15,11 @@ public: : threading::OutputMessage("Put", reader), val(val) {} - virtual bool Process() { + virtual bool Process() + { input_mgr->Put(Object(), val); return true; - } + } private: Value* *val; @@ -30,9 +31,10 @@ public: : threading::OutputMessage("Delete", reader), val(val) {} - virtual bool Process() { + virtual bool Process() + { return input_mgr->Delete(Object(), val); - } + } private: Value* *val; @@ -43,10 +45,11 @@ public: ClearMessage(ReaderFrontend* reader) : threading::OutputMessage("Clear", reader) {} - virtual bool Process() { + virtual bool Process() + { input_mgr->Clear(Object()); return true; - } + } private: }; @@ -57,14 +60,15 @@ public: : threading::OutputMessage("SendEvent", reader), name(name), num_vals(num_vals), val(val) {} - virtual bool Process() { + virtual bool Process() + { bool success = input_mgr->SendEvent(name, num_vals, val); if ( !success ) reporter->Error("SendEvent for event %s failed", name.c_str()); return true; // we do not want to die if sendEvent fails because the event did not return. - } + } private: const string name; @@ -78,10 +82,11 @@ public: : threading::OutputMessage("SendEntry", reader), val(val) { } - virtual bool Process() { + virtual bool Process() + { input_mgr->SendEntry(Object(), val); return true; - } + } private: Value* *val; @@ -92,10 +97,11 @@ public: EndCurrentSendMessage(ReaderFrontend* reader) : threading::OutputMessage("EndCurrentSend", reader) {} - virtual bool Process() { + virtual bool Process() + { input_mgr->EndCurrentSend(Object()); return true; - } + } private: }; @@ -105,9 +111,10 @@ public: ReaderClosedMessage(ReaderFrontend* reader) : threading::OutputMessage("ReaderClosed", reader) {} - virtual bool Process() { + virtual bool Process() + { return input_mgr->RemoveStreamContinuation(Object()); - } + } private: }; @@ -119,12 +126,16 @@ public: DisableMessage(ReaderFrontend* writer) : threading::OutputMessage("Disable", writer) {} - virtual bool Process() { Object()->SetDisable(); return true; } + virtual bool Process() + { + Object()->SetDisable(); + return true; + } }; ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() -{ + { buf = 0; buf_len = 1024; disabled = true; // disabled will be set correcty in init. @@ -132,45 +143,45 @@ ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() frontend = arg_frontend; SetName(frontend->Name()); -} + } ReaderBackend::~ReaderBackend() -{ - -} + { + } void ReaderBackend::Put(Value* *val) -{ + { SendOut(new PutMessage(frontend, val)); -} + } void ReaderBackend::Delete(Value* *val) -{ + { SendOut(new DeleteMessage(frontend, val)); -} + } void ReaderBackend::Clear() -{ + { SendOut(new ClearMessage(frontend)); -} + } void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) -{ + { SendOut(new SendEventMessage(frontend, name, num_vals, vals)); -} + } void ReaderBackend::EndCurrentSend() -{ + { SendOut(new EndCurrentSendMessage(frontend)); -} + } void ReaderBackend::SendEntry(Value* *vals) -{ + { SendOut(new SendEntryMessage(frontend, vals)); -} + } -bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* arg_fields) -{ +bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, + const threading::Field* const* arg_fields) + { source = arg_source; SetName("InputReader/"+source); @@ -180,89 +191,90 @@ bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, // disable if DoInit returns error. int success = DoInit(arg_source, mode, arg_num_fields, arg_fields); - if ( !success ) { + if ( !success ) + { Error("Init failed"); DisableFrontend(); - } + } disabled = !success; return success; -} + } void ReaderBackend::Close() -{ + { DoClose(); disabled = true; DisableFrontend(); SendOut(new ReaderClosedMessage(frontend)); - if ( fields != 0 ) { - - for ( unsigned int i = 0; i < num_fields; i++ ) { + if ( fields != 0 ) + { + for ( unsigned int i = 0; i < num_fields; i++ ) delete(fields[i]); - } delete[] (fields); fields = 0; + } } -} bool ReaderBackend::Update() -{ + { if ( disabled ) return false; bool success = DoUpdate(); - if ( !success ) { + if ( !success ) DisableFrontend(); - } return success; -} + } void ReaderBackend::DisableFrontend() -{ - disabled = true; // we also set disabled here, because there still may be other messages queued and we will dutifully ignore these from now + { + disabled = true; + // we also set disabled here, because there still may be other messages queued and we will dutifully ignore these from now SendOut(new DisableMessage(frontend)); -} + } bool ReaderBackend::DoHeartbeat(double network_time, double current_time) -{ + { MsgThread::DoHeartbeat(network_time, current_time); - return true; -} - -TransportProto ReaderBackend::StringToProto(const string &proto) { - if ( proto == "unknown" ) { - return TRANSPORT_UNKNOWN; - } else if ( proto == "tcp" ) { - return TRANSPORT_TCP; - } else if ( proto == "udp" ) { - return TRANSPORT_UDP; - } else if ( proto == "icmp" ) { - return TRANSPORT_ICMP; } +TransportProto ReaderBackend::StringToProto(const string &proto) + { + if ( proto == "unknown" ) + return TRANSPORT_UNKNOWN; + else if ( proto == "tcp" ) + return TRANSPORT_TCP; + else if ( proto == "udp" ) + return TRANSPORT_UDP; + else if ( proto == "icmp" ) + return TRANSPORT_ICMP; + Error(Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); return TRANSPORT_UNKNOWN; -} + } // more or less verbose copy from IPAddr.cc -- which uses reporter -Value::addr_t ReaderBackend::StringToAddr(const string &s) { +Value::addr_t ReaderBackend::StringToAddr(const string &s) + { Value::addr_t val; if ( s.find(':') == std::string::npos ) // IPv4. { val.family = IPv4; - if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) { + if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) + { Error(Fmt("Bad addres: %s", s.c_str())); memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); - } + } } @@ -277,6 +289,6 @@ Value::addr_t ReaderBackend::StringToAddr(const string &s) { } return val; -} + } } diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index 6b3c2e6a67..f61fd357b9 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -46,19 +46,23 @@ public: }; -ReaderFrontend::ReaderFrontend(bro_int_t type) { +ReaderFrontend::ReaderFrontend(bro_int_t type) + { disabled = initialized = false; ty_name = ""; backend = input_mgr->CreateBackend(this, type); assert(backend); backend->Start(); -} + } -ReaderFrontend::~ReaderFrontend() { -} +ReaderFrontend::~ReaderFrontend() + { + } -void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, const threading::Field* const* fields) { +void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, + const threading::Field* const* fields) + { if ( disabled ) return; @@ -69,39 +73,43 @@ void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, con initialized = true; backend->SendIn(new InitMessage(backend, arg_source, mode, num_fields, fields)); -} + } -void ReaderFrontend::Update() { +void ReaderFrontend::Update() + { if ( disabled ) return; - if ( !initialized ) { + if ( !initialized ) + { reporter->Error("Tried to call update on uninitialized reader"); return; - } + } backend->SendIn(new UpdateMessage(backend)); -} + } -void ReaderFrontend::Close() { +void ReaderFrontend::Close() + { if ( disabled ) return; - if ( !initialized ) { + if ( !initialized ) + { reporter->Error("Tried to call finish on uninitialized reader"); return; - } + } backend->SendIn(new CloseMessage(backend)); -} + } string ReaderFrontend::Name() const -{ + { if ( source.size() ) return ty_name; return ty_name + "/" + source; -} + } } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index a167408a0e..c798c21a5e 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -23,69 +23,73 @@ using threading::Field; FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) : name(arg_name), type(arg_type) -{ + { position = arg_position; secondary_position = -1; present = true; -} + } -FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, + const TypeTag& arg_subtype, int arg_position) : name(arg_name), type(arg_type), subtype(arg_subtype) -{ + { position = arg_position; secondary_position = -1; present = true; -} + } FieldMapping::FieldMapping(const FieldMapping& arg) : name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present) -{ + { position = arg.position; secondary_position = arg.secondary_position; -} + } -FieldMapping FieldMapping::subType() { +FieldMapping FieldMapping::subType() + { return FieldMapping(name, subtype, position); -} + } Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) -{ + { file = 0; - //keyMap = new map(); - separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), BifConst::InputAscii::separator->Len()); - if ( separator.size() != 1 ) { + separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), + BifConst::InputAscii::separator->Len()); + if ( separator.size() != 1 ) Error("separator length has to be 1. Separator will be truncated."); - } - set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), BifConst::InputAscii::set_separator->Len()); - if ( set_separator.size() != 1 ) { + set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), + BifConst::InputAscii::set_separator->Len()); + if ( set_separator.size() != 1 ) Error("set_separator length has to be 1. Separator will be truncated."); - } - empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), BifConst::InputAscii::empty_field->Len()); - - unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); + empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), + BifConst::InputAscii::empty_field->Len()); + unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), + BifConst::InputAscii::unset_field->Len()); + } Ascii::~Ascii() -{ + { DoClose(); -} + } void Ascii::DoClose() -{ - if ( file != 0 ) { + { + if ( file != 0 ) + { file->close(); delete(file); file = 0; + } } -} bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) -{ + { fname = path; mode = arg_mode; mtime = 0; @@ -93,124 +97,135 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c num_fields = arg_num_fields; fields = arg_fields; - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) + { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; - } + } file = new ifstream(path.c_str()); - if ( !file->is_open() ) { + if ( !file->is_open() ) + { Error(Fmt("Init: cannot open %s", fname.c_str())); delete(file); file = 0; return false; - } + } - if ( ReadHeader(false) == false ) { + if ( ReadHeader(false) == false ) + { Error(Fmt("Init: cannot open %s; headers are incorrect", fname.c_str())); file->close(); delete(file); file = 0; return false; - } + } DoUpdate(); return true; -} + } -bool Ascii::ReadHeader(bool useCached) { +bool Ascii::ReadHeader(bool useCached) + { // try to read the header line... string line; map ifields; - if ( !useCached ) { - if ( !GetLine(line) ) { + if ( !useCached ) + { + if ( !GetLine(line) ) + { Error("could not read first line"); return false; - } - - + } headerline = line; - - } else { + } + else line = headerline; - } // construct list of field names. istringstream splitstream(line); int pos=0; - while ( splitstream ) { + while ( splitstream ) + { string s; if ( !getline(splitstream, s, separator[0])) break; ifields[s] = pos; pos++; - } + } //printf("Updating fields from description %s\n", line.c_str()); columnMap.clear(); - for ( unsigned int i = 0; i < num_fields; i++ ) { + for ( unsigned int i = 0; i < num_fields; i++ ) + { const Field* field = fields[i]; map::iterator fit = ifields.find(field->name); - if ( fit == ifields.end() ) { - if ( field->optional ) { + if ( fit == ifields.end() ) + { + if ( field->optional ) + { // we do not really need this field. mark it as not present and always send an undef back. FieldMapping f(field->name, field->type, field->subtype, -1); f.present = false; columnMap.push_back(f); continue; - } + } Error(Fmt("Did not find requested field %s in input data file %s.", field->name.c_str(), fname.c_str())); return false; - } + } FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]); - if ( field->secondary_name != "" ) { + if ( field->secondary_name != "" ) + { map::iterator fit2 = ifields.find(field->secondary_name); - if ( fit2 == ifields.end() ) { + if ( fit2 == ifields.end() ) + { Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); return false; - } + } f.secondary_position = ifields[field->secondary_name]; - } + } columnMap.push_back(f); - } + } // well, that seems to have worked... return true; -} + } -bool Ascii::GetLine(string& str) { - while ( getline(*file, str) ) { - if ( str[0] != '#' ) { +bool Ascii::GetLine(string& str) + { + while ( getline(*file, str) ) + { + if ( str[0] != '#' ) return true; - } - if ( str.compare(0,8, "#fields\t") == 0 ) { + if ( str.compare(0,8, "#fields\t") == 0 ) + { str = str.substr(8); return true; + } } - } return false; -} - - -Value* Ascii::EntryToVal(string s, FieldMapping field) { - - if ( s.compare(unset_field) == 0 ) { // field is not set... - return new Value(field.type, false); } + +Value* Ascii::EntryToVal(string s, FieldMapping field) + { + + if ( s.compare(unset_field) == 0 ) // field is not set... + return new Value(field.type, false); + Value* val = new Value(field.type, true); switch ( field.type ) { @@ -220,14 +235,15 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { break; case TYPE_BOOL: - if ( s == "T" ) { + if ( s == "T" ) val->val.int_val = 1; - } else if ( s == "F" ) { + else if ( s == "F" ) val->val.int_val = 0; - } else { + else + { Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str())); return false; - } + } break; case TYPE_INT: @@ -250,7 +266,8 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { val->val.port_val.proto = TRANSPORT_UNKNOWN; break; - case TYPE_SUBNET: { + case TYPE_SUBNET: + { size_t pos = s.find("/"); if ( pos == s.npos ) { Error(Fmt("Invalid value for subnet: %s", s.c_str())); @@ -261,8 +278,8 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { val->val.subnet_val.prefix = StringToAddr(addr); val->val.subnet_val.length = width; - } break; + } case TYPE_ADDR: val->val.addr_val = StringToAddr(s); @@ -287,47 +304,56 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { Value** lvals = new Value* [length]; - if ( field.type == TYPE_TABLE ) { + if ( field.type == TYPE_TABLE ) + { val->val.set_val.vals = lvals; val->val.set_val.size = length; - } else if ( field.type == TYPE_VECTOR ) { + } + else if ( field.type == TYPE_VECTOR ) + { val->val.vector_val.vals = lvals; val->val.vector_val.size = length; - } else { + } + else + { assert(false); - } + } if ( length == 0 ) break; //empty istringstream splitstream(s); - while ( splitstream ) { + while ( splitstream ) + { string element; if ( !getline(splitstream, element, set_separator[0]) ) break; - if ( pos >= length ) { - Error(Fmt("Internal error while parsing set. pos %d >= length %d. Element: %s", pos, length, element.c_str())); + if ( pos >= length ) + { + Error(Fmt("Internal error while parsing set. pos %d >= length %d." + " Element: %s", pos, length, element.c_str())); break; - } + } Value* newval = EntryToVal(element, field.subType()); - if ( newval == 0 ) { + if ( newval == 0 ) + { Error("Error while reading set"); return 0; - } + } lvals[pos] = newval; pos++; - - } + } - if ( pos != length ) { + if ( pos != length ) + { Error("Internal error while parsing set: did not find all elements"); return 0; - } + } break; } @@ -340,24 +366,23 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { } return val; - -} + } // read the entire file and send appropriate thingies back to InputMgr -bool Ascii::DoUpdate() { +bool Ascii::DoUpdate() + { switch ( mode ) { case REREAD: // check if the file has changed struct stat sb; - if ( stat(fname.c_str(), &sb) == -1 ) { + if ( stat(fname.c_str(), &sb) == -1 ) + { Error(Fmt("Could not get stat for %s", fname.c_str())); return false; - } + } - if ( sb.st_mtime <= mtime ) { - // no change + if ( sb.st_mtime <= mtime ) // no change return true; - } mtime = sb.st_mtime; // file changed. reread. @@ -366,57 +391,56 @@ bool Ascii::DoUpdate() { case MANUAL: case STREAM: - // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) - if ( file && file->is_open() ) { - if ( mode == STREAM ) { + // dirty, fix me. (well, apparently after trying seeking, etc + // - this is not that bad) + if ( file && file->is_open() ) + { + if ( mode == STREAM ) + { file->clear(); // remove end of file evil bits if ( !ReadHeader(true) ) // in case filters changed - { return false; // header reading failed - } + break; - } + } file->close(); - } + } file = new ifstream(fname.c_str()); - if ( !file->is_open() ) { + if ( !file->is_open() ) + { Error(Fmt("cannot open %s", fname.c_str())); return false; - } + } - if ( ReadHeader(false) == false ) { + if ( ReadHeader(false) == false ) + { return false; - } + } break; default: assert(false); - } - - - // - - // file->seekg(0, ios::beg); // do not forget clear. - - + } string line; - while ( GetLine(line ) ) { + while ( GetLine(line ) ) + { // split on tabs istringstream splitstream(line); map stringfields; int pos = 0; - while ( splitstream ) { + while ( splitstream ) + { string s; if ( !getline(splitstream, s, separator[0]) ) break; stringfields[pos] = s; pos++; - } + } pos--; // for easy comparisons of max element. @@ -426,69 +450,60 @@ bool Ascii::DoUpdate() { int fpos = 0; for ( vector::iterator fit = columnMap.begin(); fit != columnMap.end(); - fit++ ){ + fit++ ) + { - if ( ! fit->present ) { + if ( ! fit->present ) + { // add non-present field fields[fpos] = new Value((*fit).type, false); fpos++; continue; - } + } assert(fit->position >= 0 ); - if ( (*fit).position > pos || (*fit).secondary_position > pos ) { + if ( (*fit).position > pos || (*fit).secondary_position > pos ) + { Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); return false; - } + } Value* val = EntryToVal(stringfields[(*fit).position], *fit); - if ( val == 0 ) { + if ( val == 0 ) + { Error("Could not convert String value to Val"); return false; - } + } - if ( (*fit).secondary_position != -1 ) { + if ( (*fit).secondary_position != -1 ) + { // we have a port definition :) assert(val->type == TYPE_PORT ); // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); - } + } fields[fpos] = val; fpos++; - } + } //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); assert ( (unsigned int) fpos == num_fields ); - if ( mode == STREAM ) { + if ( mode == STREAM ) Put(fields); - } else { + else SendEntry(fields); } - /* Do not do this, ownership changes to other thread - * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - delete fields[i]; - } - delete [] fields; - */ - - } - - - //file->clear(); // remove end of file evil bits - //file->seekg(0, ios::beg); // and seek to start. - - if ( mode != STREAM ) { + if ( mode != STREAM ) EndCurrentSend(); - } - + return true; -} + } bool Ascii::DoHeartbeat(double network_time, double current_time) { @@ -500,12 +515,13 @@ bool Ascii::DoHeartbeat(double network_time, double current_time) break; case REREAD: case STREAM: - Update(); // call update and not DoUpdate, because update actually checks disabled. + Update(); // call update and not DoUpdate, because update + // checks disabled. break; default: assert(false); } return true; -} + } diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index deff2b038d..29f0070fec 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -22,7 +22,7 @@ using threading::Field; Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) -{ + { multiplication_factor = double(BifConst::InputBenchmark::factor); autospread = double(BifConst::InputBenchmark::autospread); spread = int(BifConst::InputBenchmark::spread); @@ -32,19 +32,19 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) timedspread = double(BifConst::InputBenchmark::timedspread); heart_beat_interval = double(BifConst::Threading::heart_beat_interval); -} + } Benchmark::~Benchmark() -{ + { DoClose(); -} + } void Benchmark::DoClose() -{ -} + { + } bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) -{ + { mode = arg_mode; num_fields = arg_num_fields; @@ -54,18 +54,20 @@ bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Fiel if ( autospread != 0.0 ) autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) + { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; - } + } heartbeatstarttime = CurrTime(); DoUpdate(); return true; -} + } -string Benchmark::RandomString(const int len) { +string Benchmark::RandomString(const int len) + { string s(len, ' '); static const char values[] = @@ -73,65 +75,65 @@ string Benchmark::RandomString(const int len) { "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"; - for (int i = 0; i < len; ++i) { + for (int i = 0; i < len; ++i) s[i] = values[rand() / (RAND_MAX / sizeof(values))]; - } return s; -} + } -double Benchmark::CurrTime() { +double Benchmark::CurrTime() + { struct timeval tv; assert ( gettimeofday(&tv, 0) >= 0 ); return double(tv.tv_sec) + double(tv.tv_usec) / 1e6; -} + } // read the entire file and send appropriate thingies back to InputMgr -bool Benchmark::DoUpdate() { +bool Benchmark::DoUpdate() + { int linestosend = num_lines * heart_beat_interval; - for ( int i = 0; i < linestosend; i++ ) { + for ( int i = 0; i < linestosend; i++ ) + { Value** field = new Value*[num_fields]; - for (unsigned int j = 0; j < num_fields; j++ ) { + for (unsigned int j = 0; j < num_fields; j++ ) field[j] = EntryToVal(fields[j]->type, fields[j]->subtype); - } - if ( mode == STREAM ) { + if ( mode == STREAM ) // do not do tracking, spread out elements over the second that we have... Put(field); - } else { + else SendEntry(field); - } - if ( stopspreadat == 0 || num_lines < stopspreadat ) { + if ( stopspreadat == 0 || num_lines < stopspreadat ) + { if ( spread != 0 ) usleep(spread); if ( autospread_time != 0 ) usleep( autospread_time ); - } + } - if ( timedspread != 0.0 ) { + if ( timedspread != 0.0 ) + { double diff; - do { + do diff = CurrTime() - heartbeatstarttime; - //printf("%d %f\n", i, diff); - //} while ( diff < i/threading::Manager::HEART_BEAT_INTERVAL*(num_lines + (num_lines * timedspread) ) ); - } while ( diff/heart_beat_interval < i/(linestosend + (linestosend * timedspread) ) ); - //} while ( diff < 0.8); - } + while ( diff/heart_beat_interval < i/(linestosend + + (linestosend * timedspread) ) ); + } } - if ( mode != STREAM ) { + if ( mode != STREAM ) EndCurrentSend(); - } return true; } -threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { +threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) + { Value* val = new Value(type, true); // basically construct something random from the fields that we want. @@ -170,7 +172,8 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { val->val.port_val.proto = TRANSPORT_UNKNOWN; break; - case TYPE_SUBNET: { + case TYPE_SUBNET: + { val->val.subnet_val.prefix = StringToAddr("192.168.17.1"); val->val.subnet_val.length = 16; } @@ -192,28 +195,32 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { Value** lvals = new Value* [length]; - if ( type == TYPE_TABLE ) { + if ( type == TYPE_TABLE ) + { val->val.set_val.vals = lvals; val->val.set_val.size = length; - } else if ( type == TYPE_VECTOR ) { + } + else if ( type == TYPE_VECTOR ) + { val->val.vector_val.vals = lvals; val->val.vector_val.size = length; - } else { + } + else assert(false); - } if ( length == 0 ) break; //empty - for ( unsigned int pos = 0; pos < length; pos++ ) { - + for ( unsigned int pos = 0; pos < length; pos++ ) + { Value* newval = EntryToVal(subtype, TYPE_ENUM); - if ( newval == 0 ) { + if ( newval == 0 ) + { Error("Error while reading set"); return 0; - } + } lvals[pos] = newval; - } + } break; } @@ -226,20 +233,11 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { return val; -} + } bool Benchmark::DoHeartbeat(double network_time, double current_time) { - /* - * This does not work the way I envisioned it, because the queueing is the problem. - printf("%f\n", CurrTime() - current_time); - if ( CurrTime() - current_time > 0.25 ) { - // event has hung for a time. refuse. - SendEvent("EndBenchmark", 0, 0); - return true; - } */ - ReaderBackend::DoHeartbeat(network_time, current_time); num_lines = (int) ( (double) num_lines*multiplication_factor); num_lines += add; @@ -251,7 +249,8 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) break; case REREAD: case STREAM: - if ( multiplication_factor != 1 || add != 0 ) { + if ( multiplication_factor != 1 || add != 0 ) + { // we have to document at what time we changed the factor to what value. Value** v = new Value*[2]; v[0] = new Value(TYPE_COUNT, true); @@ -260,12 +259,11 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) v[1]->val.double_val = CurrTime(); SendEvent("lines_changed", 2, v); - } + } - if ( autospread != 0.0 ) { - autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); + if ( autospread != 0.0 ) // because executing this in every loop is apparently too expensive. - } + autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); Update(); // call update and not DoUpdate, because update actually checks disabled. diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index f656be769c..43c782de29 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -24,79 +24,86 @@ using threading::Value; using threading::Field; Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend) -{ + { file = 0; in = 0; - //keyMap = new map(); - separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(), BifConst::InputRaw::record_separator->Len()); - if ( separator.size() != 1 ) { + if ( separator.size() != 1 ) Error("separator length has to be 1. Separator will be truncated."); } -} - Raw::~Raw() -{ + { DoClose(); -} + } void Raw::DoClose() -{ - if ( file != 0 ) { + { + if ( file != 0 ) + { Close(); + } } -} bool Raw::Open() -{ - if ( execute ) { + { + if ( execute ) + { file = popen(fname.c_str(), "r"); - if ( file == NULL ) { + if ( file == NULL ) + { Error(Fmt("Could not execute command %s", fname.c_str())); return false; + } } - } else { + else + { file = fopen(fname.c_str(), "r"); - if ( file == NULL ) { + if ( file == NULL ) + { Error(Fmt("Init: cannot open %s", fname.c_str())); return false; + } } - } in = new boost::fdistream(fileno(file)); - if ( execute && mode == STREAM ) { + if ( execute && mode == STREAM ) + { fcntl(fileno(file), F_SETFL, O_NONBLOCK); - } + } return true; -} + } bool Raw::Close() -{ - if ( file == NULL ) { + { + if ( file == NULL ) + { InternalError(Fmt("Trying to close closed file for stream %s", fname.c_str())); return false; - } + } - if ( execute ) { + if ( execute ) + { delete(in); pclose(file); - } else { + } + else + { delete(in); fclose(file); - } + } in = NULL; file = NULL; return true; -} + } bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) -{ + { fname = path; mode = arg_mode; mtime = 0; @@ -107,24 +114,30 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con num_fields = arg_num_fields; fields = arg_fields; - if ( path.length() == 0 ) { + if ( path.length() == 0 ) + { Error("No source path provided"); return false; - } + } - if ( arg_num_fields != 1 ) { - Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); + if ( arg_num_fields != 1 ) + { + Error("Filter for raw reader contains more than one field. " + "Filters for the raw reader may only contain exactly one string field. " + "Filter ignored."); return false; - } + } - if ( fields[0]->type != TYPE_STRING ) { + if ( fields[0]->type != TYPE_STRING ) + { Error("Filter for raw reader contains a field that is not of type string."); return false; - } + } // do Initialization char last = path[path.length()-1]; - if ( last == '|' ) { + if ( last == '|' ) + { execute = true; fname = path.substr(0, fname.length() - 1); @@ -137,19 +150,17 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con } else { execute = false; - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) + { Error(Fmt("Unsupported read mode %d for source %s", mode, fname.c_str())); return false; - } + } result = Open(); + } - } - - if ( result == false ) { + if ( result == false ) return result; - } - #ifdef DEBUG Debug(DBG_INPUT, "Raw reader created, will perform first update"); @@ -162,62 +173,68 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con Debug(DBG_INPUT, "First update went through"); #endif return true; -} - - -bool Raw::GetLine(string& str) { - while ( getline(*in, str, separator[0]) ) { - return true; } + +bool Raw::GetLine(string& str) + { + while ( getline(*in, str, separator[0]) ) + return true; + return false; -} + } // read the entire file and send appropriate thingies back to InputMgr -bool Raw::DoUpdate() { - if ( firstrun ) { +bool Raw::DoUpdate() + { + if ( firstrun ) firstrun = false; - } else { + else + { switch ( mode ) { case REREAD: + { // check if the file has changed struct stat sb; - if ( stat(fname.c_str(), &sb) == -1 ) { + if ( stat(fname.c_str(), &sb) == -1 ) + { Error(Fmt("Could not get stat for %s", fname.c_str())); return false; - } + } - if ( sb.st_mtime <= mtime ) { + if ( sb.st_mtime <= mtime ) // no change return true; - } mtime = sb.st_mtime; // file changed. reread. // fallthrough + } case MANUAL: case STREAM: - if ( mode == STREAM && file != NULL && in != NULL ) { + if ( mode == STREAM && file != NULL && in != NULL ) + { //fpurge(file); in->clear(); // remove end of file evil bits break; - } + } Close(); - if ( !Open() ) { + if ( !Open() ) return false; - } + break; default: assert(false); } - } + } string line; - while ( GetLine(line) ) { + while ( GetLine(line) ) + { assert (num_fields == 1); Value** fields = new Value*[1]; @@ -228,14 +245,14 @@ bool Raw::DoUpdate() { fields[0] = val; Put(fields); - } + } return true; -} + } bool Raw::DoHeartbeat(double network_time, double current_time) -{ + { ReaderBackend::DoHeartbeat(network_time, current_time); switch ( mode ) { @@ -244,12 +261,12 @@ bool Raw::DoHeartbeat(double network_time, double current_time) break; case REREAD: case STREAM: - Update(); // call update and not DoUpdate, because update actually checks disabled. + Update(); // call update and not DoUpdate, because update + // checks disabled. break; default: assert(false); } return true; -} - + } From 2e452dc29ff39c557d7de5e6b21af93ce2e80690 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 10:49:17 -0700 Subject: [PATCH 135/149] remove last remnants of autostart, which has been removed for quite a while. --- scripts/base/frameworks/input/main.bro | 6 ------ src/input/Manager.cc | 1 - src/input/ReaderBackend.h | 1 - 3 files changed, 8 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index e06dfae005..a52cd97b4b 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -24,9 +24,6 @@ export { ## Read mode to use for this stream mode: Mode &default=default_mode; - ## Automatically start the input stream after the first filter has been added - autostart: bool &default=T; - ## Descriptive name. Used to remove a filter at a later time name: string; @@ -68,9 +65,6 @@ export { ## Read mode to use for this stream mode: Mode &default=default_mode; - ## Automatically start the input stream after the first filter has been added - autostart: bool &default=T; - ## Descriptive name. Used to remove a filter at a later time name: string; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 0fde16b87d..3f7fcea078 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -287,7 +287,6 @@ bool Manager::CreateStream(Stream* info, RecordVal* description) } EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); - Val *autostart = description->LookupWithDefault(rtype->FieldOffset("autostart")); ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); assert(reader_obj); diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 5b230ca652..ca54d8a204 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -227,7 +227,6 @@ private: // For implementing Fmt(). char* buf; unsigned int buf_len; - bool autostart; unsigned int num_fields; const threading::Field* const * fields; // raw mapping From 96a7e068f085291c3ee7db0bca39cb1df18055ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 11:29:57 -0700 Subject: [PATCH 136/149] baselines for the autostart removal. --- .../scripts.base.frameworks.input.event/out | 14 ++++---- .../out | 18 +++++------ .../scripts.base.frameworks.input.raw/out | 16 +++++----- .../scripts.base.frameworks.input.reread/out | 16 +++++----- .../out | 32 +++++++++---------- .../out | 16 +++++----- .../out | 14 ++++---- .../out | 6 ++-- 8 files changed, 66 insertions(+), 66 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out index 59070cd88e..bb3b6d0a9e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.event/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out @@ -1,4 +1,4 @@ -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -8,7 +8,7 @@ print A::b; Input::EVENT_NEW 1 T -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -18,7 +18,7 @@ print A::b; Input::EVENT_NEW 2 T -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -28,7 +28,7 @@ print A::b; Input::EVENT_NEW 3 F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -38,7 +38,7 @@ print A::b; Input::EVENT_NEW 4 F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -48,7 +48,7 @@ print A::b; Input::EVENT_NEW 5 F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -58,7 +58,7 @@ print A::b; Input::EVENT_NEW 6 F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out index 06e28de441..bb69da3267 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out @@ -1,4 +1,4 @@ -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -14,7 +14,7 @@ Input::remove(input); }] Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -30,7 +30,7 @@ Input::remove(input); }] Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -46,7 +46,7 @@ Input::remove(input); }] Input::EVENT_NEW q3r3057fdf -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -62,7 +62,7 @@ Input::remove(input); }] Input::EVENT_NEW sdfs\d -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -78,7 +78,7 @@ Input::remove(input); }] Input::EVENT_NEW -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -94,7 +94,7 @@ Input::remove(input); }] Input::EVENT_NEW dfsdf -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -110,7 +110,7 @@ Input::remove(input); }] Input::EVENT_NEW sdf -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -126,7 +126,7 @@ Input::remove(input); }] Input::EVENT_NEW 3rw43wRRERLlL#RWERERERE. -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out index 34a5599dc9..55e7610e1e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out @@ -1,4 +1,4 @@ -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -6,7 +6,7 @@ print A::s; }] Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -14,7 +14,7 @@ print A::s; }] Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -22,7 +22,7 @@ print A::s; }] Input::EVENT_NEW q3r3057fdf -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -30,7 +30,7 @@ print A::s; }] Input::EVENT_NEW sdfs\d -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -38,7 +38,7 @@ print A::s; }] Input::EVENT_NEW -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -46,7 +46,7 @@ print A::s; }] Input::EVENT_NEW dfsdf -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -54,7 +54,7 @@ print A::s; }] Input::EVENT_NEW sdf -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 46a30f387f..5cce15f6c7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -15,7 +15,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -96,7 +96,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -201,7 +201,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -366,7 +366,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -489,7 +489,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -612,7 +612,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -735,7 +735,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -858,7 +858,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out index d85c8f2e83..9d62fdbef4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out @@ -1,4 +1,4 @@ -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -6,7 +6,7 @@ print A::s; }] Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -14,7 +14,7 @@ print A::s; }] Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -22,7 +22,7 @@ print A::s; }] Input::EVENT_NEW q3r3057fdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -30,7 +30,7 @@ print A::s; }] Input::EVENT_NEW sdfs\d -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -38,7 +38,7 @@ print A::s; }] Input::EVENT_NEW -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -46,7 +46,7 @@ print A::s; }] Input::EVENT_NEW dfsdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -54,7 +54,7 @@ print A::s; }] Input::EVENT_NEW sdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -62,7 +62,7 @@ print A::s; }] Input::EVENT_NEW 3rw43wRRERLlL#RWERERERE. -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -70,7 +70,7 @@ print A::s; }] Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -78,7 +78,7 @@ print A::s; }] Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -86,7 +86,7 @@ print A::s; }] Input::EVENT_NEW q3r3057fdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -94,7 +94,7 @@ print A::s; }] Input::EVENT_NEW sdfs\d -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -102,7 +102,7 @@ print A::s; }] Input::EVENT_NEW -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -110,7 +110,7 @@ print A::s; }] Input::EVENT_NEW dfsdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -118,7 +118,7 @@ print A::s; }] Input::EVENT_NEW sdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out index 937acf428e..07a3ffdba5 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out @@ -1,4 +1,4 @@ -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -13,7 +13,7 @@ Input::remove(input); }] Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -28,7 +28,7 @@ Input::remove(input); }] Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -43,7 +43,7 @@ Input::remove(input); }] Input::EVENT_NEW q3r3057fdf -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -58,7 +58,7 @@ Input::remove(input); }] Input::EVENT_NEW sdfs\d -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -73,7 +73,7 @@ Input::remove(input); }] Input::EVENT_NEW -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -88,7 +88,7 @@ Input::remove(input); }] Input::EVENT_NEW dfsdf -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -103,7 +103,7 @@ Input::remove(input); }] Input::EVENT_NEW sdf -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out index 56b36a1a0e..a1bbb9bbe4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -1,4 +1,4 @@ -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -16,7 +16,7 @@ print right; Input::EVENT_NEW [i=1] T -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -34,7 +34,7 @@ print right; Input::EVENT_NEW [i=2] T -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -52,7 +52,7 @@ print right; Input::EVENT_NEW [i=3] F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -70,7 +70,7 @@ print right; Input::EVENT_NEW [i=4] F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -88,7 +88,7 @@ print right; Input::EVENT_NEW [i=5] F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -106,7 +106,7 @@ print right; Input::EVENT_NEW [i=6] F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out index a61a4a2993..41d9438da0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out @@ -30,7 +30,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -120,7 +120,7 @@ BB } ============EVENT============ Description -[source=../input2.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh2, destination={ +[source=../input2.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh2, destination={ [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -240,7 +240,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, From 4de6d76488e0d85f7085aa53528bee93b7d7b8b7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 11:30:18 -0700 Subject: [PATCH 137/149] fix up the executeraw test - now it works for the first time and does not always fail --- src/input/readers/Raw.cc | 10 ++++++++++ .../scripts.base.frameworks.input.executeraw/out | 9 +++++++++ .../scripts/base/frameworks/input/executeraw.bro | 16 ++++++++++------ 3 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index 43c782de29..ce0b4f8a5f 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -178,6 +178,12 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con bool Raw::GetLine(string& str) { + if ( in->peek() == std::iostream::traits_type::eof() ) + return false; + + if ( in->eofbit == true || in->failbit == true ) + return false; + while ( getline(*in, str, separator[0]) ) return true; @@ -247,6 +253,10 @@ bool Raw::DoUpdate() Put(fields); } +#ifdef DEBUG + Debug(DBG_INPUT, "DoUpdate finished successfully"); +#endif + return true; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out new file mode 100644 index 0000000000..8611b35dd3 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out @@ -0,0 +1,9 @@ +[source=wc -l ../input.log |, reader=Input::READER_RAW, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line +{ +print outfile, description; +print outfile, tpe; +print outfile, s; +close(outfile); +}] +Input::EVENT_NEW + 8 ../input.log diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro index 6fceebf885..6d07a9bf29 100644 --- a/testing/btest/scripts/base/frameworks/input/executeraw.bro +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -1,5 +1,6 @@ # -# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 1 # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -13,21 +14,24 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE +@load frameworks/communication/listen -module A; +global outfile: file; type Val: record { s: string; }; event line(description: Input::EventDescription, tpe: Input::Event, s: string) { - print description; - print tpe; - print s; + print outfile, description; + print outfile, tpe; + print outfile, s; + close(outfile); } event bro_init() { - Input::add_event([$source="wc input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]); + outfile = open ("../out"); + Input::add_event([$source="wc -l ../input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]); Input::remove("input"); } From 24173807ea8b1c895bc9ee75cb9924ba198901ea Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 11:35:56 -0700 Subject: [PATCH 138/149] reactivate network_time check in threading manager. previously this line made all input framework tests fail - it works now. Some of the other recent changes of the threading manager must have fixed that problem. This was easy :) --- src/threading/Manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 491b8379e8..4a05fb8d41 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -125,7 +125,7 @@ void Manager::Process() if ( msg->Process() ) { - //if ( network_time ) // FIXME: ask robin again if he needs this. makes input interface not work in bro_init. + if ( network_time ) did_process = true; } From 658b188dff8cf4112f0752f1f63422bd13bde51d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 14:26:11 -0700 Subject: [PATCH 139/149] filters have been called streams for eternity. And I always was too lazy to change it everywhere... Fix that. --- scripts/base/frameworks/input/main.bro | 4 +- src/input/Manager.cc | 342 ++++++++++++------------- src/input/Manager.h | 30 ++- src/input/ReaderBackend.h | 42 +-- src/input/readers/Ascii.cc | 2 +- src/input/readers/Ascii.h | 2 +- src/input/readers/Raw.h | 3 - 7 files changed, 217 insertions(+), 208 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index a52cd97b4b..c9ce0e321e 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -24,7 +24,7 @@ export { ## Read mode to use for this stream mode: Mode &default=default_mode; - ## Descriptive name. Used to remove a filter at a later time + ## Descriptive name. Used to remove a stream at a later time name: string; ## Special definitions for tables @@ -65,7 +65,7 @@ export { ## Read mode to use for this stream mode: Mode &default=default_mode; - ## Descriptive name. Used to remove a filter at a later time + ## Descriptive name. Used to remove a stream at a later time name: string; ## Special definitions for events diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 3f7fcea078..3bae7dbb28 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -62,7 +62,7 @@ public: int mode; - StreamType filter_type; // to distinguish between event and table filters + StreamType stream_type; // to distinguish between event and table streams EnumVal* type; ReaderFrontend* reader; @@ -129,7 +129,7 @@ public: Manager::TableStream::TableStream() : Manager::Stream::Stream() { - filter_type = TABLE_FILTER; + stream_type = TABLE_FILTER; tab = 0; itype = 0; @@ -144,7 +144,7 @@ Manager::TableStream::TableStream() : Manager::Stream::Stream() Manager::EventStream::EventStream() : Manager::Stream::Stream() { fields = 0; - filter_type = EVENT_FILTER; + stream_type = EVENT_FILTER; } Manager::EventStream::~EventStream() @@ -322,16 +322,16 @@ bool Manager::CreateEventStream(RecordVal* fval) RecordType* rtype = fval->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::EventDescription, 0) ) { - reporter->Error("filter argument not of right type"); + reporter->Error("EventDescription argument not of right type"); return false; } - EventStream* filter = new EventStream(); + EventStream* stream = new EventStream(); { - bool res = CreateStream(filter, fval); + bool res = CreateStream(stream, fval); if ( res == false ) { - delete filter; + delete stream; return false; } } @@ -428,19 +428,19 @@ bool Manager::CreateEventStream(RecordVal* fval) logf[i] = fieldsV[i]; Unref(fields); // ref'd by lookupwithdefault - filter->num_fields = fieldsV.size(); - filter->fields = fields->Ref()->AsRecordType(); - filter->event = event_registry->Lookup(event->GetID()->Name()); - filter->want_record = ( want_record->InternalInt() == 1 ); + stream->num_fields = fieldsV.size(); + stream->fields = fields->Ref()->AsRecordType(); + stream->event = event_registry->Lookup(event->GetID()->Name()); + stream->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault - assert(filter->reader); - filter->reader->Init(filter->source, filter->mode, filter->num_fields, logf ); + assert(stream->reader); + stream->reader->Init(stream->source, stream->mode, stream->num_fields, logf ); - readers[filter->reader] = filter; + readers[stream->reader] = stream; DBG_LOG(DBG_INPUT, "Successfully created event stream %s", - filter->name.c_str()); + stream->name.c_str()); return true; } @@ -450,16 +450,16 @@ bool Manager::CreateTableStream(RecordVal* fval) RecordType* rtype = fval->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::TableDescription, 0) ) { - reporter->Error("filter argument not of right type"); + reporter->Error("TableDescription argument not of right type"); return false; } - TableStream* filter = new TableStream(); + TableStream* stream = new TableStream(); { - bool res = CreateStream(filter, fval); + bool res = CreateStream(stream, fval); if ( res == false ) { - delete filter; + delete stream; return false; } } @@ -587,40 +587,40 @@ bool Manager::CreateTableStream(RecordVal* fval) for ( unsigned int i = 0; i < fieldsV.size(); i++ ) fields[i] = fieldsV[i]; - filter->pred = pred ? pred->AsFunc() : 0; - filter->num_idx_fields = idxfields; - filter->num_val_fields = valfields; - filter->tab = dst->AsTableVal(); - filter->rtype = val ? val->AsRecordType() : 0; - filter->itype = idx->AsRecordType(); - filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; - filter->currDict = new PDict(InputHash); - filter->currDict->SetDeleteFunc(input_hash_delete_func); - filter->lastDict = new PDict(InputHash); - filter->lastDict->SetDeleteFunc(input_hash_delete_func); - filter->want_record = ( want_record->InternalInt() == 1 ); + stream->pred = pred ? pred->AsFunc() : 0; + stream->num_idx_fields = idxfields; + stream->num_val_fields = valfields; + stream->tab = dst->AsTableVal(); + stream->rtype = val ? val->AsRecordType() : 0; + stream->itype = idx->AsRecordType(); + stream->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + stream->currDict = new PDict(InputHash); + stream->currDict->SetDeleteFunc(input_hash_delete_func); + stream->lastDict = new PDict(InputHash); + stream->lastDict->SetDeleteFunc(input_hash_delete_func); + stream->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault Unref(pred); if ( valfields > 1 ) { - if ( ! filter->want_record ) + if ( ! stream->want_record ) { - reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", filter->name.c_str()); - delete filter; + reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", stream->name.c_str()); + delete stream; return false; } } - assert(filter->reader); - filter->reader->Init(filter->source, filter->mode, fieldsV.size(), fields ); + assert(stream->reader); + stream->reader->Init(stream->source, stream->mode, fieldsV.size(), fields ); - readers[filter->reader] = filter; + readers[stream->reader] = stream; DBG_LOG(DBG_INPUT, "Successfully created table stream %s", - filter->name.c_str()); + stream->name.c_str()); return true; } @@ -872,9 +872,9 @@ void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) } int readFields; - if ( i->filter_type == TABLE_FILTER ) + if ( i->stream_type == TABLE_FILTER ) readFields = SendEntryTable(i, vals); - else if ( i->filter_type == EVENT_FILTER ) + else if ( i->stream_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); readFields = SendEventStreamEvent(i, type, vals); @@ -894,21 +894,21 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) assert(i); - assert(i->filter_type == TABLE_FILTER); - TableStream* filter = (TableStream*) i; + assert(i->stream_type == TABLE_FILTER); + TableStream* stream = (TableStream*) i; - HashKey* idxhash = HashValues(filter->num_idx_fields, vals); + HashKey* idxhash = HashValues(stream->num_idx_fields, vals); if ( idxhash == 0 ) { reporter->Error("Could not hash line. Ignoring"); - return filter->num_val_fields + filter->num_idx_fields; + return stream->num_val_fields + stream->num_idx_fields; } hash_t valhash = 0; - if ( filter->num_val_fields > 0 ) + if ( stream->num_val_fields > 0 ) { - HashKey* valhashkey = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); + HashKey* valhashkey = HashValues(stream->num_val_fields, vals+stream->num_idx_fields); if ( valhashkey == 0 ) { // empty line. index, but no values. // hence we also have no hash value... @@ -920,23 +920,23 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) } } - InputHash *h = filter->lastDict->Lookup(idxhash); + InputHash *h = stream->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before - if ( filter->num_val_fields == 0 || h->valhash == valhash ) + if ( stream->num_val_fields == 0 || h->valhash == valhash ) { // ok, exact duplicate, move entry to new dicrionary and do nothing else. - filter->lastDict->Remove(idxhash); - filter->currDict->Insert(idxhash, h); + stream->lastDict->Remove(idxhash); + stream->currDict->Insert(idxhash, h); delete idxhash; - return filter->num_val_fields + filter->num_idx_fields; + return stream->num_val_fields + stream->num_idx_fields; } else { - assert( filter->num_val_fields > 0 ); + assert( stream->num_val_fields > 0 ); // entry was updated in some way - filter->lastDict->Remove(idxhash); + stream->lastDict->Remove(idxhash); // keep h for predicates updated = true; @@ -947,24 +947,24 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) Val* valval; RecordVal* predidx = 0; - int position = filter->num_idx_fields; - if ( filter->num_val_fields == 0 ) + int position = stream->num_idx_fields; + if ( stream->num_val_fields == 0 ) valval = 0; - else if ( filter->num_val_fields == 1 && !filter->want_record ) - valval = ValueToVal(vals[position], filter->rtype->FieldType(0)); + else if ( stream->num_val_fields == 1 && !stream->want_record ) + valval = ValueToVal(vals[position], stream->rtype->FieldType(0)); else - valval = ValueToRecordVal(vals, filter->rtype, &position); + valval = ValueToRecordVal(vals, stream->rtype, &position); - // call filter first to determine if we really add / change the entry - if ( filter->pred ) + // call stream first to determine if we really add / change the entry + if ( stream->pred ) { EnumVal* ev; //Ref(idxval); int startpos = 0; - //Val* predidx = ListValToRecordVal(idxval->AsListVal(), filter->itype, &startpos); - predidx = ValueToRecordVal(vals, filter->itype, &startpos); - //ValueToRecordVal(vals, filter->itype, &startpos); + //Val* predidx = ListValToRecordVal(idxval->AsListVal(), stream->itype, &startpos); + predidx = ValueToRecordVal(vals, stream->itype, &startpos); + //ValueToRecordVal(vals, stream->itype, &startpos); if ( updated ) ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); @@ -972,10 +972,10 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); bool result; - if ( filter->num_val_fields > 0 ) // we have values - result = CallPred(filter->pred, 3, ev, predidx->Ref(), valval->Ref()); + if ( stream->num_val_fields > 0 ) // we have values + result = CallPred(stream->pred, 3, ev, predidx->Ref(), valval->Ref()); else // no values - result = CallPred(filter->pred, 2, ev, predidx->Ref()); + result = CallPred(stream->pred, 2, ev, predidx->Ref()); if ( result == false ) { @@ -985,17 +985,17 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { // throw away. Hence - we quit. And remove the entry from the current dictionary... // (but why should it be in there? assert this). - assert ( filter->currDict->RemoveEntry(idxhash) == 0 ); + assert ( stream->currDict->RemoveEntry(idxhash) == 0 ); delete idxhash; delete h; - return filter->num_val_fields + filter->num_idx_fields; + return stream->num_val_fields + stream->num_idx_fields; } else { // keep old one - filter->currDict->Insert(idxhash, h); + stream->currDict->Insert(idxhash, h); delete idxhash; - return filter->num_val_fields + filter->num_idx_fields; + return stream->num_val_fields + stream->num_idx_fields; } } @@ -1016,19 +1016,19 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) // I think there is an unref missing here. But if I insert is, it crashes :) } else - idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); + idxval = ValueToIndexVal(stream->num_idx_fields, stream->itype, vals); Val* oldval = 0; if ( updated == true ) { - assert(filter->num_val_fields > 0); + assert(stream->num_val_fields > 0); // in that case, we need the old value to send the event (if we send an event). - oldval = filter->tab->Lookup(idxval, false); + oldval = stream->tab->Lookup(idxval, false); } //i->tab->Assign(idxval, valval); assert(idxval); - HashKey* k = filter->tab->ComputeHash(idxval); + HashKey* k = stream->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); @@ -1039,46 +1039,46 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) ih->idxkey = new HashKey(k->Key(), k->Size(), k->Hash()); ih->valhash = valhash; - if ( filter->event && updated ) + if ( stream->event && updated ) Ref(oldval); // otherwise it is no longer accessible after the assignment - filter->tab->Assign(idxval, k, valval); + stream->tab->Assign(idxval, k, valval); Unref(idxval); // asssign does not consume idxval. if ( predidx != 0 ) Unref(predidx); - filter->currDict->Insert(idxhash, ih); + stream->currDict->Insert(idxhash, ih); delete idxhash; - if ( filter->event ) + if ( stream->event ) { EnumVal* ev; int startpos = 0; - Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); if ( updated ) { // in case of update send back the old value. - assert ( filter->num_val_fields > 0 ); + assert ( stream->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - if ( filter->num_val_fields == 0 ) + if ( stream->num_val_fields == 0 ) { - Ref(filter->description); - SendEvent(filter->event, 3, filter->description->Ref(), ev, predidx); + Ref(stream->description); + SendEvent(stream->event, 3, stream->description->Ref(), ev, predidx); } else - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval->Ref()); + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, valval->Ref()); } } - return filter->num_val_fields + filter->num_idx_fields; + return stream->num_val_fields + stream->num_idx_fields; } @@ -1097,19 +1097,19 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) i->name.c_str()); #endif - if ( i->filter_type == EVENT_FILTER ) // nothing to do.. + if ( i->stream_type == EVENT_FILTER ) // nothing to do.. return; - assert(i->filter_type == TABLE_FILTER); - TableStream* filter = (TableStream*) i; + assert(i->stream_type == TABLE_FILTER); + TableStream* stream = (TableStream*) i; // lastdict contains all deleted entries and should be empty apart from that - IterCookie *c = filter->lastDict->InitForIteration(); - filter->lastDict->MakeRobustCookie(c); + IterCookie *c = stream->lastDict->InitForIteration(); + stream->lastDict->MakeRobustCookie(c); InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) + while ( ( ih = stream->lastDict->NextEntry(lastDictIdxKey, c) ) ) { ListVal * idx = 0; Val *val = 0; @@ -1118,17 +1118,17 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) EnumVal* ev = 0; int startpos = 0; - if ( filter->pred || filter->event ) + if ( stream->pred || stream->event ) { - idx = filter->tab->RecoverIndex(ih->idxkey); + idx = stream->tab->RecoverIndex(ih->idxkey); assert(idx != 0); - val = filter->tab->Lookup(idx); + val = stream->tab->Lookup(idx); assert(val != 0); - predidx = ListValToRecordVal(idx, filter->itype, &startpos); + predidx = ListValToRecordVal(idx, stream->itype, &startpos); ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); } - if ( filter->pred ) + if ( stream->pred ) { // ask predicate, if we want to expire this element... @@ -1136,7 +1136,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) Ref(predidx); Ref(val); - bool result = CallPred(filter->pred, 3, ev, predidx, val); + bool result = CallPred(stream->pred, 3, ev, predidx, val); if ( result == false ) { @@ -1144,37 +1144,37 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) // ah well - and we have to add the entry to currDict... Unref(predidx); Unref(ev); - filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); + stream->currDict->Insert(lastDictIdxKey, stream->lastDict->RemoveEntry(lastDictIdxKey)); delete lastDictIdxKey; continue; } } - if ( filter->event ) + if ( stream->event ) { Ref(predidx); Ref(val); Ref(ev); - SendEvent(filter->event, 3, ev, predidx, val); + SendEvent(stream->event, 3, ev, predidx, val); } - if ( predidx ) // if we have a filter or an event... + if ( predidx ) // if we have a stream or an event... Unref(predidx); if ( ev ) Unref(ev); - Unref(filter->tab->Delete(ih->idxkey)); - filter->lastDict->Remove(lastDictIdxKey); // delete in next line + Unref(stream->tab->Delete(ih->idxkey)); + stream->lastDict->Remove(lastDictIdxKey); // delete in next line delete lastDictIdxKey; delete(ih); } - filter->lastDict->Clear(); // should be empt. buti- well... who knows... - delete(filter->lastDict); + stream->lastDict->Clear(); // should be empt. buti- well... who knows... + delete(stream->lastDict); - filter->lastDict = filter->currDict; - filter->currDict = new PDict(InputHash); - filter->currDict->SetDeleteFunc(input_hash_delete_func); + stream->lastDict = stream->currDict; + stream->currDict = new PDict(InputHash); + stream->currDict->SetDeleteFunc(input_hash_delete_func); #ifdef DEBUG DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s, queueing update_finished event", @@ -1199,9 +1199,9 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) } int readFields; - if ( i->filter_type == TABLE_FILTER ) + if ( i->stream_type == TABLE_FILTER ) readFields = PutTable(i, vals); - else if ( i->filter_type == EVENT_FILTER ) + else if ( i->stream_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); readFields = SendEventStreamEvent(i, type, vals); @@ -1219,44 +1219,44 @@ int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const * { assert(i); - assert(i->filter_type == EVENT_FILTER); - EventStream* filter = (EventStream*) i; + assert(i->stream_type == EVENT_FILTER); + EventStream* stream = (EventStream*) i; Val *val; list out_vals; - Ref(filter->description); - out_vals.push_back(filter->description); + Ref(stream->description); + out_vals.push_back(stream->description); // no tracking, send everything with a new event... //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); out_vals.push_back(type); int position = 0; - if ( filter->want_record ) + if ( stream->want_record ) { - RecordVal * r = ValueToRecordVal(vals, filter->fields, &position); + RecordVal * r = ValueToRecordVal(vals, stream->fields, &position); out_vals.push_back(r); } else { - for ( int j = 0; j < filter->fields->NumFields(); j++) + for ( int j = 0; j < stream->fields->NumFields(); j++) { Val* val = 0; - if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) + if ( stream->fields->FieldType(j)->Tag() == TYPE_RECORD ) val = ValueToRecordVal(vals, - filter->fields->FieldType(j)->AsRecordType(), + stream->fields->FieldType(j)->AsRecordType(), &position); else { - val = ValueToVal(vals[position], filter->fields->FieldType(j)); + val = ValueToVal(vals[position], stream->fields->FieldType(j)); position++; } out_vals.push_back(val); } } - SendEvent(filter->event, out_vals); + SendEvent(stream->event, out_vals); - return filter->fields->NumFields(); + return stream->fields->NumFields(); } @@ -1264,31 +1264,31 @@ int Manager::PutTable(Stream* i, const Value* const *vals) { assert(i); - assert(i->filter_type == TABLE_FILTER); - TableStream* filter = (TableStream*) i; + assert(i->stream_type == TABLE_FILTER); + TableStream* stream = (TableStream*) i; - Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); + Val* idxval = ValueToIndexVal(stream->num_idx_fields, stream->itype, vals); Val* valval; - int position = filter->num_idx_fields; - if ( filter->num_val_fields == 0 ) + int position = stream->num_idx_fields; + if ( stream->num_val_fields == 0 ) valval = 0; - else if ( filter->num_val_fields == 1 && filter->want_record == 0 ) - valval = ValueToVal(vals[position], filter->rtype->FieldType(0)); + else if ( stream->num_val_fields == 1 && stream->want_record == 0 ) + valval = ValueToVal(vals[position], stream->rtype->FieldType(0)); else - valval = ValueToRecordVal(vals, filter->rtype, &position); + valval = ValueToRecordVal(vals, stream->rtype, &position); // if we have a subscribed event, we need to figure out, if this is an update or not // same for predicates - if ( filter->pred || filter->event ) + if ( stream->pred || stream->event ) { bool updated = false; Val* oldval = 0; - if ( filter->num_val_fields > 0 ) + if ( stream->num_val_fields > 0 ) { // in that case, we need the old value to send the event (if we send an event). - oldval = filter->tab->Lookup(idxval, false); + oldval = stream->tab->Lookup(idxval, false); } if ( oldval != 0 ) @@ -1300,11 +1300,11 @@ int Manager::PutTable(Stream* i, const Value* const *vals) // predicate if we want the update or not - if ( filter->pred ) + if ( stream->pred ) { EnumVal* ev; int startpos = 0; - Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); Ref(valval); if ( updated ) @@ -1315,10 +1315,10 @@ int Manager::PutTable(Stream* i, const Value* const *vals) BifType::Enum::Input::Event); bool result; - if ( filter->num_val_fields > 0 ) // we have values - result = CallPred(filter->pred, 3, ev, predidx, valval); + if ( stream->num_val_fields > 0 ) // we have values + result = CallPred(stream->pred, 3, ev, predidx, valval); else // no values - result = CallPred(filter->pred, 2, ev, predidx); + result = CallPred(stream->pred, 2, ev, predidx); if ( result == false ) { @@ -1326,38 +1326,38 @@ int Manager::PutTable(Stream* i, const Value* const *vals) Unref(idxval); Unref(valval); Unref(oldval); - return filter->num_val_fields + filter->num_idx_fields; + return stream->num_val_fields + stream->num_idx_fields; } } - filter->tab->Assign(idxval, valval); + stream->tab->Assign(idxval, valval); - if ( filter->event ) + if ( stream->event ) { EnumVal* ev; int startpos = 0; - Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); if ( updated ) { // in case of update send back the old value. - assert ( filter->num_val_fields > 0 ); + assert ( stream->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); - SendEvent(filter->event, 4, filter->description->Ref(), + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - if ( filter->num_val_fields == 0 ) - SendEvent(filter->event, 4, filter->description->Ref(), + if ( stream->num_val_fields == 0 ) + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx); else - SendEvent(filter->event, 4, filter->description->Ref(), + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, valval->Ref()); } @@ -1365,10 +1365,10 @@ int Manager::PutTable(Stream* i, const Value* const *vals) } else // no predicates or other stuff - filter->tab->Assign(idxval, valval); + stream->tab->Assign(idxval, valval); - return filter->num_idx_fields + filter->num_val_fields; + return stream->num_idx_fields + stream->num_val_fields; } // Todo:: perhaps throw some kind of clear-event? @@ -1386,10 +1386,10 @@ void Manager::Clear(ReaderFrontend* reader) i->name.c_str()); #endif - assert(i->filter_type == TABLE_FILTER); - TableStream* filter = (TableStream*) i; + assert(i->stream_type == TABLE_FILTER); + TableStream* stream = (TableStream*) i; - filter->tab->RemoveAll(); + stream->tab->RemoveAll(); } // put interface: delete old entry from table. @@ -1405,28 +1405,28 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) bool success = false; int readVals = 0; - if ( i->filter_type == TABLE_FILTER ) + if ( i->stream_type == TABLE_FILTER ) { - TableStream* filter = (TableStream*) i; - Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); + TableStream* stream = (TableStream*) i; + Val* idxval = ValueToIndexVal(stream->num_idx_fields, stream->itype, vals); assert(idxval != 0); - readVals = filter->num_idx_fields + filter->num_val_fields; - bool filterresult = true; + readVals = stream->num_idx_fields + stream->num_val_fields; + bool streamresult = true; - if ( filter->pred || filter->event ) + if ( stream->pred || stream->event ) { - Val *val = filter->tab->Lookup(idxval); + Val *val = stream->tab->Lookup(idxval); - if ( filter->pred ) + if ( stream->pred ) { Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); int startpos = 0; - Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); - filterresult = CallPred(filter->pred, 3, ev, predidx, val); + streamresult = CallPred(stream->pred, 3, ev, predidx, val); - if ( filterresult == false ) + if ( streamresult == false ) { // keep it. Unref(idxval); @@ -1435,21 +1435,21 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) } - // only if filter = true -> no filtering - if ( filterresult && filter->event ) + // only if stream = true -> no streaming + if ( streamresult && stream->event ) { Ref(idxval); assert(val != 0); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(filter->event, 4, filter->description->Ref(), ev, idxval, val); + SendEvent(stream->event, 4, stream->description->Ref(), ev, idxval, val); } } - // only if filter = true -> no filtering - if ( filterresult ) + // only if stream = true -> no streaming + if ( streamresult ) { - Val* retptr = filter->tab->Delete(idxval); + Val* retptr = stream->tab->Delete(idxval); success = ( retptr != 0 ); if ( !success ) reporter->Error("Internal error while deleting values from input table"); @@ -1458,7 +1458,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) } } - else if ( i->filter_type == EVENT_FILTER ) + else if ( i->stream_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); readVals = SendEventStreamEvent(i, type, vals); diff --git a/src/input/Manager.h b/src/input/Manager.h index 0bdb2eb58d..d15febe0d6 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -1,6 +1,6 @@ // See the file "COPYING" in the main distribution directory for copyright. // -// Class for managing input streams and filters +// Class for managing input streams #ifndef INPUT_MANAGER_H #define INPUT_MANAGER_H @@ -34,12 +34,7 @@ public: ~Manager(); /** - * Creates a new input stream. - * Add a filter to an input source, which will write the data from the data source into - * a Bro table. - * Add a filter to an input source, which sends events for read input data. - * - * @param id The enum value corresponding the input stream. + * Creates a new input stream which will write the data from the data source into * * @param description A record of script type \c Input:StreamDescription. * @@ -47,6 +42,15 @@ public: * input.bif, which just forwards here. */ bool CreateTableStream(RecordVal* description); + + /** + * Creates a new input stream which sends events for read input data. + * + * @param description A record of script type \c Input:StreamDescription. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool CreateEventStream(RecordVal* description); @@ -104,11 +108,11 @@ protected: // doing so creates a new thread!). ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); - // Functions are called from the ReaderBackend to notify the manager, that a filter has been removed + // Functions are called from the ReaderBackend to notify the manager, that a stream has been removed // or a stream has been closed. - // Used to prevent race conditions where data for a specific filter is still in the queue when the + // Used to prevent race conditions where data for a specific stream is still in the queue when the // RemoveStream directive is executed by the main thread. - // This makes sure all data that has ben queued for a filter is still received. + // This makes sure all data that has ben queued for a stream is still received. bool RemoveStreamContinuation(ReaderFrontend* reader); private: @@ -118,13 +122,13 @@ private: bool CreateStream(Stream*, RecordVal* description); - // SendEntry implementation for Tablefilter + // SendEntry implementation for Table stream int SendEntryTable(Stream* i, const threading::Value* const *vals); - // Put implementation for Tablefilter + // Put implementation for Table stream int PutTable(Stream* i, const threading::Value* const *vals); - // SendEntry and Put implementation for Eventfilter + // SendEntry and Put implementation for Event stream int SendEventStreamEvent(Stream* i, EnumVal* type, const threading::Value* const *vals); // Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index ca54d8a204..b4d9101bc8 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -55,7 +55,8 @@ public: * * @param arg_num_fields number of fields contained in \a fields * - * @param fields the types and names of the fields to be retrieved from the input source + * @param fields the types and names of the fields to be retrieved + * from the input source * * @return False if an error occured. */ @@ -72,7 +73,8 @@ public: /** * Force trigger an update of the input stream. - * The action that will be taken depends on the current read mode and the individual input backend + * The action that will be taken depends on the current read mode and the + * individual input backend * * An backend can choose to ignore this. * @@ -90,8 +92,8 @@ protected: // Methods that have to be overwritten by the individual readers /** - * Reader-specific intialization method. Note that data may only be read from the input source - * after the Start function has been called. + * Reader-specific intialization method. Note that data may only be + * read from the input source after the Start function has been called. * * A reader implementation must override this method. If it returns * false, it will be assumed that a fatal error has occured that @@ -145,29 +147,32 @@ protected: */ void SendEvent(const string& name, const int num_vals, threading::Value* *vals); - // Content-sending-functions (simple mode). Including table-specific stuff that simply is not used if we have no table + // Content-sending-functions (simple mode). Including table-specific stuff that + // simply is not used if we have no table /** - * Method allowing a reader to send a list of values read for a specific filter back to the manager. + * Method allowing a reader to send a list of values read for a specific stream + * back to the manager. * - * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised + * If the stream is a table stream, the values are inserted into the table; + * if it is an event stream, the event is raised. * - * @param val list of threading::Values expected by the filter + * @param val list of threading::Values expected by the stream */ void Put(threading::Value* *val); /** * Method allowing a reader to delete a specific value from a bro table. * - * If the receiving filter is an event, only a removed event is raised + * If the receiving stream is an event stream, only a removed event is raised * - * @param val list of threading::Values expected by the filter + * @param val list of threading::Values expected by the stream */ void Delete(threading::Value* *val); /** * Method allowing a reader to clear a value from a bro table. * - * If the receiving filter is an event, this is ignored. + * If the receiving stream is an event stream, this is ignored. * */ void Clear(); @@ -176,19 +181,22 @@ protected: /** - * Method allowing a reader to send a list of values read for a specific filter back to the manager. + * Method allowing a reader to send a list of values read for a specific stream + * back to the manager. * - * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised. + * If the stream is a table stream, the values are inserted into the table; + * if it is an event stream, the event is raised. * - * @param val list of threading::Values expected by the filter + * @param val list of threading::Values expected by the stream */ void SendEntry(threading::Value* *vals); /** - * Method telling the manager, that the current list of entries sent by SendEntry is finished. + * Method telling the manager, that the current list of entries sent by SendEntry + * is finished. * - * For table filters, all entries that were not updated since the last EndCurrentSend will be deleted, because they are no longer - * present in the input source + * For table streams, all entries that were not updated since the last EndCurrentSend + * will be deleted, because they are no longer present in the input source * */ void EndCurrentSend(); diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index c798c21a5e..8223d6e201 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -398,7 +398,7 @@ bool Ascii::DoUpdate() if ( mode == STREAM ) { file->clear(); // remove end of file evil bits - if ( !ReadHeader(true) ) // in case filters changed + if ( !ReadHeader(true) ) return false; // header reading failed break; diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 0953075bff..e5f3070724 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -74,7 +74,7 @@ private: string unset_field; - // keep a copy of the headerline to determine field locations when filters change + // keep a copy of the headerline to determine field locations when stream descriptions change string headerline; int mode; diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index 59f9202960..9f575bb89c 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -42,9 +42,6 @@ private: // Options set from the script-level. string separator; - // keep a copy of the headerline to determine field locations when filters change - string headerline; - int mode; bool execute; bool firstrun; From b37f9e38f6ac9d82bc3500cd5b7aadff7e35ea54 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 25 May 2012 15:14:25 -0700 Subject: [PATCH 140/149] Input framework merge in progress. --- scripts/base/frameworks/input/main.bro | 53 +- .../base/frameworks/input/readers/ascii.bro | 2 + src/input/Manager.cc | 1019 +++++++++-------- src/input/Manager.h | 140 +-- src/input/ReaderBackend.cc | 84 +- src/input/ReaderBackend.h | 162 ++- src/input/ReaderFrontend.cc | 32 +- src/input/ReaderFrontend.h | 44 +- src/input/readers/Ascii.cc | 211 ++-- src/input/readers/Ascii.h | 60 +- src/input/readers/Benchmark.cc | 80 +- src/input/readers/Benchmark.h | 31 +- src/input/readers/Raw.cc | 151 +-- src/input/readers/Raw.h | 39 +- .../out | 2 +- .../base/frameworks/input/executeraw.bro | 3 +- 16 files changed, 1063 insertions(+), 1050 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index c9ce0e321e..7a372dc120 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,7 +4,7 @@ module Input; export { - + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; @@ -13,52 +13,56 @@ export { ## TableFilter description type used for the `table` method. type TableDescription: record { ## Common definitions for tables and events - + ## String that allows the reader to find the source. ## For `READER_ASCII`, this is the filename. source: string; - - ## Reader to use for this steam + + ## Reader to use for this stream reader: Reader &default=default_reader; ## Read mode to use for this stream mode: Mode &default=default_mode; ## Descriptive name. Used to remove a stream at a later time - name: string; + name: string; - ## Special definitions for tables + # Special definitions for tables - ## Table which will contain the data read by the input framework + ## Table which will receive the data read by the input framework destination: any; + ## Record that defines the values used as the index of the table idx: any; - ## Record that defines the values used as the values of the table + + ## Record that defines the values used as the elements of the table ## If val is undefined, destination has to be a set. val: any &optional; - ## Defines if the value of the table is a record (default), or a single value. - ## Val can only contain one element when this is set to false. + + ## Defines if the value of the table is a record (default), or a single value. Val + ## can only contain one element when this is set to false. want_record: bool &default=T; - ## The event that is raised each time a value is added to, changed in or removed from the table. - ## The event will receive an Input::Event enum as the first argument, the idx record as the second argument - ## and the value (record) as the third argument. - ev: any &optional; # event containing idx, val as values. + ## The event that is raised each time a value is added to, changed in or removed + ## from the table. The event will receive an Input::Event enum as the first + ## argument, the idx record as the second argument and the value (record) as the + ## third argument. + ev: any &optional; # event containing idx, val as values. - ## Predicate function, that can decide if an insertion, update or removal should really be executed. - ## Parameters are the same as for the event. If true is returned, the update is performed. If false - ## is returned, it is skipped + ## Predicate function that can decide if an insertion, update or removal should + ## really be executed. Parameters are the same as for the event. If true is + ## returned, the update is performed. If false is returned, it is skipped. pred: function(typ: Input::Event, left: any, right: any): bool &optional; }; ## EventFilter description type used for the `event` method. type EventDescription: record { ## Common definitions for tables and events - + ## String that allows the reader to find the source. ## For `READER_ASCII`, this is the filename. source: string; - + ## Reader to use for this steam reader: Reader &default=default_reader; @@ -66,19 +70,20 @@ export { mode: Mode &default=default_mode; ## Descriptive name. Used to remove a stream at a later time - name: string; + name: string; + + # Special definitions for events - ## Special definitions for events - ## Record describing the fields to be retrieved from the source input. fields: any; + ## If want_record if false (default), the event receives each value in fields as a seperate argument. ## If it is set to true, the event receives all fields in a signle record value. want_record: bool &default=F; ## The event that is rised each time a new line is received from the reader. ## The event will receive an Input::Event enum as the first element, and the fields as the following arguments. - ev: any; + ev: any; }; @@ -86,7 +91,7 @@ export { ## ## description: `TableDescription` record describing the source. global add_table: function(description: Input::TableDescription) : bool; - + ## Create a new event input from a given source. Returns true on success. ## ## description: `TableDescription` record describing the source. diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro index 14c04757f7..7fca1ad795 100644 --- a/scripts/base/frameworks/input/readers/ascii.bro +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -1,4 +1,6 @@ ##! Interface for the ascii input reader. +##! +##! The defaults are set to match Bro's ASCII output. module InputAscii; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 3bae7dbb28..6cae5e2f34 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -23,27 +23,41 @@ using namespace input; using threading::Value; using threading::Field; +struct ReaderDefinition { + bro_int_t type; // The reader type. + const char *name; // Descriptive name for error messages. + bool (*init)(); // Optional one-time initializing function. + ReaderBackend* (*factory)(ReaderFrontend* frontend); // Factory function for creating instances. +}; + +ReaderDefinition input_readers[] = { + { BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate }, + { BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate }, + { BifEnum::Input::READER_BENCHMARK, "Benchmark", 0, reader::Benchmark::Instantiate }, + + // End marker + { BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 } +}; + /** - * InputHashes are used as Dictionaries to store the value and index hashes for all - * lines currently stored in a table. Index hash is stored as HashKey*, because it is - * thrown into other Bro functions that need the complex structure of it. - * For everything we do (with values), we just take the hash_t value and compare it - * directly with == + * InputHashes are used as Dictionaries to store the value and index hashes + * for all lines currently stored in a table. Index hash is stored as + * HashKey*, because it is thrown into other Bro functions that need the + * complex structure of it. For everything we do (with values), we just take + * the hash_t value and compare it directly with "==" */ -struct InputHash - { +struct InputHash { hash_t valhash; - HashKey* idxkey; + HashKey* idxkey; ~InputHash(); - }; +}; -InputHash::~InputHash() +InputHash::~InputHash() { - if ( idxkey ) - delete idxkey; - } + delete idxkey; + } -static void input_hash_delete_func(void* val) +static void input_hash_delete_func(void* val) { InputHash* h = (InputHash*) val; delete h; @@ -52,14 +66,14 @@ static void input_hash_delete_func(void* val) declare(PDict, InputHash); /** - * Base stuff that every stream can do + * Base stuff that every stream can do. */ class Manager::Stream { public: string name; string source; bool removed; - + int mode; StreamType stream_type; // to distinguish between event and table streams @@ -73,23 +87,24 @@ public: virtual ~Stream(); }; -Manager::Stream::Stream() +Manager::Stream::Stream() { - type = 0; - reader = 0; - description = 0; + type = 0; + reader = 0; + description = 0; removed = false; } -Manager::Stream::~Stream() +Manager::Stream::~Stream() { - if ( type ) + if ( type ) Unref(type); - if ( description ) + + if ( description ) Unref(description); - if ( reader ) - delete(reader); + if ( reader ) + delete(reader); } class Manager::TableStream: public Manager::Stream { @@ -98,7 +113,7 @@ public: unsigned int num_idx_fields; unsigned int num_val_fields; bool want_record; - EventHandlerPtr table_event; + EventHandlerPtr table_event; TableVal* tab; RecordType* rtype; @@ -107,9 +122,9 @@ public: PDict(InputHash)* currDict; PDict(InputHash)* lastDict; - Func* pred; + Func* pred; - EventHandlerPtr event; + EventHandlerPtr event; TableStream(); ~TableStream(); @@ -122,15 +137,15 @@ public: RecordType* fields; unsigned int num_fields; - bool want_record; + bool want_record; EventStream(); ~EventStream(); }; -Manager::TableStream::TableStream() : Manager::Stream::Stream() +Manager::TableStream::TableStream() : Manager::Stream::Stream() { - stream_type = TABLE_FILTER; - + stream_type = TABLE_STREAM; + tab = 0; itype = 0; rtype = 0; @@ -141,61 +156,47 @@ Manager::TableStream::TableStream() : Manager::Stream::Stream() pred = 0; } -Manager::EventStream::EventStream() : Manager::Stream::Stream() +Manager::EventStream::EventStream() : Manager::Stream::Stream() { fields = 0; - stream_type = EVENT_FILTER; + stream_type = EVENT_STREAM; } -Manager::EventStream::~EventStream() +Manager::EventStream::~EventStream() { - if ( fields ) + if ( fields ) Unref(fields); } -Manager::TableStream::~TableStream() +Manager::TableStream::~TableStream() { if ( tab ) Unref(tab); - if ( itype ) + + if ( itype ) Unref(itype); + if ( rtype ) // can be 0 for sets Unref(rtype); - if ( currDict != 0 ) + if ( currDict != 0 ) { currDict->Clear(); delete currDict; } - if ( lastDict != 0 ) + if ( lastDict != 0 ) { lastDict->Clear();; delete lastDict; } - } - -struct ReaderDefinition { - bro_int_t type; // the type - const char *name; // descriptive name for error messages - bool (*init)(); // optional one-time inifializing function - ReaderBackend* (*factory)(ReaderFrontend* frontend); // factory function for creating instances -}; - -ReaderDefinition input_readers[] = { - { BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate }, - { BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate }, - { BifEnum::Input::READER_BENCHMARK, "Benchmark", 0, reader::Benchmark::Instantiate }, - - // End marker - { BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 } -}; + } Manager::Manager() { } -Manager::~Manager() +Manager::~Manager() { for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -205,47 +206,48 @@ Manager::~Manager() } -ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) +ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) { ReaderDefinition* ir = input_readers; - while ( true ) + while ( true ) { - if ( ir->type == BifEnum::Input::READER_DEFAULT ) + if ( ir->type == BifEnum::Input::READER_DEFAULT ) { reporter->Error("The reader that was requested was not found and could not be initialized."); return 0; } - if ( ir->type != type ) + if ( ir->type != type ) { // no, didn't find the right one... ++ir; continue; } - + // call init function of writer if presnt - if ( ir->init ) + if ( ir->init ) { - if ( (*ir->init)() ) + if ( (*ir->init)() ) { //clear it to be not called again ir->init = 0; - } + } + else { // ohok. init failed, kill factory for all eternity ir->factory = 0; DBG_LOG(DBG_LOGGING, "Failed to init input class %s", ir->name); return 0; } - + } - - if ( !ir->factory ) + + if ( ! ir->factory ) // no factory? return 0; - + // all done. break. break; } @@ -259,45 +261,43 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) return backend; } -// create a new input reader object to be used at whomevers leisure lateron. -bool Manager::CreateStream(Stream* info, RecordVal* description) +// Create a new input reader object to be used at whomevers leisure lateron. +bool Manager::CreateStream(Stream* info, RecordVal* description) { ReaderDefinition* ir = input_readers; - + RecordType* rtype = description->Type()->AsRecordType(); - if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) + if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) || same_type(rtype, BifType::Record::Input::EventDescription, 0) ) ) { reporter->Error("Streamdescription argument not of right type for new input stream"); return false; } - + Val* name_val = description->LookupWithDefault(rtype->FieldOffset("name")); string name = name_val->AsString()->CheckString(); Unref(name_val); + Stream *i = FindStream(name); + if ( i != 0 ) { - Stream *i = FindStream(name); - if ( i != 0 ) - { - reporter->Error("Trying create already existing input stream %s", - name.c_str()); - return false; - } + reporter->Error("Trying create already existing input stream %s", + name.c_str()); + return false; } EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); - ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); - assert(reader_obj); - - // get the source... + ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); + assert(reader_obj); + + // get the source ... Val* sourceval = description->LookupWithDefault(rtype->FieldOffset("source")); assert ( sourceval != 0 ); const BroString* bsource = sourceval->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); Unref(sourceval); - + EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); info->mode = mode->InternalInt(); Unref(mode); @@ -311,25 +311,23 @@ bool Manager::CreateStream(Stream* info, RecordVal* description) DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", name.c_str()); - + return true; - } -bool Manager::CreateEventStream(RecordVal* fval) +bool Manager::CreateEventStream(RecordVal* fval) { - RecordType* rtype = fval->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::EventDescription, 0) ) { reporter->Error("EventDescription argument not of right type"); return false; } - + EventStream* stream = new EventStream(); { bool res = CreateStream(stream, fval); - if ( res == false ) + if ( res == false ) { delete stream; return false; @@ -337,94 +335,93 @@ bool Manager::CreateEventStream(RecordVal* fval) } - RecordType *fields = fval->LookupWithDefault(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); - - Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + RecordType *fields = fval->LookupWithDefault(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); Func* event = event_val->AsFunc(); Unref(event_val); + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) { - FuncType* etype = event->FType()->AsFuncType(); - - if ( ! etype->IsEvent() ) + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() < 2 ) + { + reporter->Error("event takes not enough arguments"); + return false; + } + + if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("events second attribute must be of type Input::Event"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Record::Input::EventDescription, 0) ) + { + reporter->Error("events first attribute must be of type Input::EventDescription"); + return false; + } + + if ( want_record->InternalInt() == 0 ) + { + if ( args->length() != fields->NumFields() + 2 ) { - reporter->Error("stream event is a function, not an event"); + reporter->Error("event has wrong number of arguments"); return false; } - const type_list* args = etype->ArgTypes()->Types(); - - if ( args->length() < 2 ) + for ( int i = 0; i < fields->NumFields(); i++ ) { - reporter->Error("event takes not enough arguments"); - return false; - } - - if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) - { - reporter->Error("events second attribute must be of type Input::Event"); - return false; - } - - if ( ! same_type((*args)[0], BifType::Record::Input::EventDescription, 0) ) - { - reporter->Error("events first attribute must be of type Input::EventDescription"); - return false; - } - - if ( want_record->InternalInt() == 0 ) - { - if ( args->length() != fields->NumFields() + 2 ) - { - reporter->Error("event has wrong number of arguments"); - return false; - } - - for ( int i = 0; i < fields->NumFields(); i++ ) - { - if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) - { - reporter->Error("Incompatible type for event"); - return false; - } - } - - } - else if ( want_record->InternalInt() == 1 ) - { - if ( args->length() != 3 ) - { - reporter->Error("event has wrong number of arguments"); - return false; - } - - if ( !same_type((*args)[2], fields ) ) + if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) { reporter->Error("Incompatible type for event"); return false; } - - } - else - assert(false); + } - } + } + + else if ( want_record->InternalInt() == 1 ) + { + if ( args->length() != 3 ) + { + reporter->Error("event has wrong number of arguments"); + return false; + } + + if ( !same_type((*args)[2], fields ) ) + { + reporter->Error("Incompatible type for event"); + return false; + } + + } + + else + assert(false); vector fieldsV; // vector, because UnrollRecordType needs it bool status = !UnrollRecordType(&fieldsV, fields, ""); - if ( status ) + if ( status ) { reporter->Error("Problem unrolling"); return false; } - + Field** logf = new Field*[fieldsV.size()]; - for ( unsigned int i = 0; i < fieldsV.size(); i++ ) + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) logf[i] = fieldsV[i]; Unref(fields); // ref'd by lookupwithdefault @@ -445,7 +442,7 @@ bool Manager::CreateEventStream(RecordVal* fval) return true; } -bool Manager::CreateTableStream(RecordVal* fval) +bool Manager::CreateTableStream(RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::TableDescription, 0) ) @@ -457,7 +454,7 @@ bool Manager::CreateTableStream(RecordVal* fval) TableStream* stream = new TableStream(); { bool res = CreateStream(stream, fval); - if ( res == false ) + if ( res == false ) { delete stream; return false; @@ -468,8 +465,8 @@ bool Manager::CreateTableStream(RecordVal* fval) RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = 0; - - if ( fval->LookupWithDefault(rtype->FieldOffset("val")) != 0 ) + + if ( fval->LookupWithDefault(rtype->FieldOffset("val")) != 0 ) { val = fval->LookupWithDefault(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); Unref(val); // The lookupwithdefault in the if-clause ref'ed val. @@ -478,30 +475,28 @@ bool Manager::CreateTableStream(RecordVal* fval) TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); // check if index fields match table description + int num = idx->NumFields(); + const type_list* tl = dst->Type()->AsTableType()->IndexTypes(); + + loop_over_list(*tl, j) { - int num = idx->NumFields(); - const type_list* tl = dst->Type()->AsTableType()->IndexTypes(); - - loop_over_list(*tl, j) + if ( j >= num ) { - if ( j >= num ) - { - reporter->Error("Table type has more indexes than index definition"); - return false; - } - - if ( !same_type(idx->FieldType(j), (*tl)[j]) ) - { - reporter->Error("Table type does not match index type"); - return false; - } - } - - if ( num != j ) - { - reporter->Error("Table has less elements than index definition"); + reporter->Error("Table type has more indexes than index definition"); return false; } + + if ( ! same_type(idx->FieldType(j), (*tl)[j]) ) + { + reporter->Error("Table type does not match index type"); + return false; + } + } + + if ( num != j ) + { + reporter->Error("Table has less elements than index definition"); + return false; } Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); @@ -509,12 +504,12 @@ bool Manager::CreateTableStream(RecordVal* fval) Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); Func* event = event_val ? event_val->AsFunc() : 0; Unref(event_val); - - if ( event ) + + if ( event ) { FuncType* etype = event->FType()->AsFuncType(); - - if ( ! etype->IsEvent() ) + + if ( ! etype->IsEvent() ) { reporter->Error("stream event is a function, not an event"); return false; @@ -522,37 +517,37 @@ bool Manager::CreateTableStream(RecordVal* fval) const type_list* args = etype->ArgTypes()->Types(); - if ( args->length() != 4 ) + if ( args->length() != 4 ) { reporter->Error("Table event must take 4 arguments"); return false; } - if ( ! same_type((*args)[0], BifType::Record::Input::TableDescription, 0) ) + if ( ! same_type((*args)[0], BifType::Record::Input::TableDescription, 0) ) { reporter->Error("table events first attribute must be of type Input::TableDescription"); return false; - } + } - if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) + if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) { reporter->Error("table events second attribute must be of type Input::Event"); return false; - } + } - if ( ! same_type((*args)[2], idx) ) + if ( ! same_type((*args)[2], idx) ) { reporter->Error("table events index attributes do not match"); return false; - } - - if ( want_record->InternalInt() == 1 && ! same_type((*args)[3], val) ) + } + + if ( want_record->InternalInt() == 1 && ! same_type((*args)[3], val) ) { reporter->Error("table events value attributes do not match"); return false; - } - else if ( want_record->InternalInt() == 0 - && !same_type((*args)[3], val->FieldType(0) ) ) + } + else if ( want_record->InternalInt() == 0 + && !same_type((*args)[3], val->FieldType(0) ) ) { reporter->Error("table events value attribute does not match"); return false; @@ -560,33 +555,32 @@ bool Manager::CreateTableStream(RecordVal* fval) assert(want_record->InternalInt() == 1 || want_record->InternalInt() == 0); - } + } vector fieldsV; // vector, because we don't know the length beforehands bool status = !UnrollRecordType(&fieldsV, idx, ""); int idxfields = fieldsV.size(); - + if ( val ) // if we are not a set status = status || !UnrollRecordType(&fieldsV, val, ""); int valfields = fieldsV.size() - idxfields; - if ( !val ) + if ( ! val ) assert(valfields == 0); - if ( status ) + if ( status ) { reporter->Error("Problem unrolling"); return false; } - - + Field** fields = new Field*[fieldsV.size()]; - for ( unsigned int i = 0; i < fieldsV.size(); i++ ) + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) fields[i] = fieldsV[i]; - + stream->pred = pred ? pred->AsFunc() : 0; stream->num_idx_fields = idxfields; stream->num_val_fields = valfields; @@ -603,9 +597,9 @@ bool Manager::CreateTableStream(RecordVal* fval) Unref(want_record); // ref'd by lookupwithdefault Unref(pred); - if ( valfields > 1 ) + if ( valfields > 1 ) { - if ( ! stream->want_record ) + if ( ! stream->want_record ) { reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", stream->name.c_str()); delete stream; @@ -664,7 +658,7 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) { if ( atomic_only ) return false; - + return IsCompatibleType(t->AsVectorType()->YieldType(), true); } @@ -676,14 +670,14 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) } -bool Manager::RemoveStream(const string &name) +bool Manager::RemoveStream(const string &name) { Stream *i = FindStream(name); - if ( i == 0 ) + if ( i == 0 ) return false; // not found - if ( i->removed ) + if ( i->removed ) { reporter->Error("Stream %s is already queued for removal. Ignoring remove.", name.c_str()); return false; @@ -701,11 +695,11 @@ bool Manager::RemoveStream(const string &name) return true; } -bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) +bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) { Stream *i = FindStream(reader); - if ( i == 0 ) + if ( i == 0 ) { reporter->Error("Stream not found in RemoveStreamContinuation"); return false; @@ -718,49 +712,51 @@ bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) readers.erase(reader); delete(i); + return true; } -bool Manager::UnrollRecordType(vector *fields, - const RecordType *rec, const string& nameprepend) +bool Manager::UnrollRecordType(vector *fields, + const RecordType *rec, const string& nameprepend) { - for ( int i = 0; i < rec->NumFields(); i++ ) + for ( int i = 0; i < rec->NumFields(); i++ ) { - if ( !IsCompatibleType(rec->FieldType(i)) ) + if ( ! IsCompatibleType(rec->FieldType(i)) ) { reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); return false; } - if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) + if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) { string prep = nameprepend + rec->FieldName(i) + "."; - - if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) + + if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) { return false; } - } - else + } + + else { Field* field = new Field(); field->name = nameprepend + rec->FieldName(i); - field->type = rec->FieldType(i)->Tag(); - if ( field->type == TYPE_TABLE ) - { + field->type = rec->FieldType(i)->Tag(); + + if ( field->type == TYPE_TABLE ) field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); - } - else if ( field->type == TYPE_VECTOR ) - { + + else if ( field->type == TYPE_VECTOR ) field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); - } else if ( field->type == TYPE_PORT && - rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) + + else if ( field->type == TYPE_PORT && + rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) { // we have an annotation for the second column - + Val* c = rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN)->AttrExpr()->Eval(0); assert(c); @@ -769,7 +765,7 @@ bool Manager::UnrollRecordType(vector *fields, field->secondary_name = c->AsStringVal()->AsString()->CheckString(); } - if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) + if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) field->optional = true; fields->push_back(field); @@ -782,30 +778,29 @@ bool Manager::UnrollRecordType(vector *fields, bool Manager::ForceUpdate(const string &name) { Stream *i = FindStream(name); - if ( i == 0 ) + if ( i == 0 ) { reporter->Error("Stream %s not found", name.c_str()); return false; } - - if ( i->removed ) + + if ( i->removed ) { reporter->Error("Stream %s is already queued for removal. Ignoring force update.", name.c_str()); return false; } - + i->reader->Update(); #ifdef DEBUG - DBG_LOG(DBG_INPUT, "Forcing update of stream %s", - name.c_str()); + DBG_LOG(DBG_INPUT, "Forcing update of stream %s", name.c_str()); #endif return true; // update is async :( } -Val* Manager::RecordValToIndexVal(RecordVal *r) +Val* Manager::RecordValToIndexVal(RecordVal *r) { Val* idxval; @@ -813,16 +808,15 @@ Val* Manager::RecordValToIndexVal(RecordVal *r) int num_fields = type->NumFields(); - if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) - { + if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) idxval = r->LookupWithDefault(0); - } - else + + else { ListVal *l = new ListVal(TYPE_ANY); - for ( int j = 0 ; j < num_fields; j++ ) + for ( int j = 0 ; j < num_fields; j++ ) l->Append(r->LookupWithDefault(j)); - + idxval = l; } @@ -831,23 +825,27 @@ Val* Manager::RecordValToIndexVal(RecordVal *r) } -Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) +Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) { Val* idxval; int position = 0; - if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { + if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) + { idxval = ValueToVal(vals[0], type->FieldType(0)); position = 1; - } else { + } + + else + { ListVal *l = new ListVal(TYPE_ANY); - for ( int j = 0 ; j < type->NumFields(); j++ ) + for ( int j = 0 ; j < type->NumFields(); j++ ) { - if ( type->FieldType(j)->Tag() == TYPE_RECORD ) - l->Append(ValueToRecordVal(vals, + if ( type->FieldType(j)->Tag() == TYPE_RECORD ) + l->Append(ValueToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); - else + else { l->Append(ValueToVal(vals[position], type->FieldType(j))); position++; @@ -862,66 +860,70 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu } -void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) +void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { Stream *i = FindStream(reader); - if ( i == 0 ) + if ( i == 0 ) { reporter->InternalError("Unknown reader in SendEntry"); return; } - int readFields; - if ( i->stream_type == TABLE_FILTER ) + int readFields = 0; + + if ( i->stream_type == TABLE_STREAM ) readFields = SendEntryTable(i, vals); - else if ( i->stream_type == EVENT_FILTER ) + + else if ( i->stream_type == EVENT_STREAM ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - readFields = SendEventStreamEvent(i, type, vals); - } - else + readFields = SendEventStreamEvent(i, type, vals); + } + + else assert(false); - for ( int i = 0; i < readFields; i++ ) + for ( int i = 0; i < readFields; i++ ) delete vals[i]; - delete [] vals; + delete [] vals; } -int Manager::SendEntryTable(Stream* i, const Value* const *vals) +int Manager::SendEntryTable(Stream* i, const Value* const *vals) { bool updated = false; assert(i); - assert(i->stream_type == TABLE_FILTER); + assert(i->stream_type == TABLE_STREAM); TableStream* stream = (TableStream*) i; HashKey* idxhash = HashValues(stream->num_idx_fields, vals); - - if ( idxhash == 0 ) + + if ( idxhash == 0 ) { reporter->Error("Could not hash line. Ignoring"); return stream->num_val_fields + stream->num_idx_fields; - } - + } + hash_t valhash = 0; - if ( stream->num_val_fields > 0 ) + if ( stream->num_val_fields > 0 ) { HashKey* valhashkey = HashValues(stream->num_val_fields, vals+stream->num_idx_fields); - if ( valhashkey == 0 ) { + if ( valhashkey == 0 ) + { // empty line. index, but no values. // hence we also have no hash value... } - else + else { - valhash = valhashkey->Hash(); - delete(valhashkey); + valhash = valhashkey->Hash(); + delete(valhashkey); } } - InputHash *h = stream->lastDict->Lookup(idxhash); - if ( h != 0 ) + InputHash *h = stream->lastDict->Lookup(idxhash); + if ( h != 0 ) { // seen before if ( stream->num_val_fields == 0 || h->valhash == valhash ) @@ -932,41 +934,41 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) delete idxhash; return stream->num_val_fields + stream->num_idx_fields; } - else + + else { assert( stream->num_val_fields > 0 ); // entry was updated in some way stream->lastDict->Remove(idxhash); // keep h for predicates updated = true; - + } } - Val* valval; RecordVal* predidx = 0; - + int position = stream->num_idx_fields; - if ( stream->num_val_fields == 0 ) + + if ( stream->num_val_fields == 0 ) valval = 0; - else if ( stream->num_val_fields == 1 && !stream->want_record ) + + else if ( stream->num_val_fields == 1 && !stream->want_record ) valval = ValueToVal(vals[position], stream->rtype->FieldType(0)); - else + + else valval = ValueToRecordVal(vals, stream->rtype, &position); // call stream first to determine if we really add / change the entry - if ( stream->pred ) + if ( stream->pred ) { EnumVal* ev; - //Ref(idxval); - int startpos = 0; - //Val* predidx = ListValToRecordVal(idxval->AsListVal(), stream->itype, &startpos); + int startpos = 0; predidx = ValueToRecordVal(vals, stream->itype, &startpos); - //ValueToRecordVal(vals, stream->itype, &startpos); - if ( updated ) + if ( updated ) ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); else ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); @@ -976,12 +978,13 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) result = CallPred(stream->pred, 3, ev, predidx->Ref(), valval->Ref()); else // no values result = CallPred(stream->pred, 2, ev, predidx->Ref()); - - if ( result == false ) + + if ( result == false ) { Unref(predidx); Unref(valval); - if ( !updated ) + + if ( ! updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... // (but why should it be in there? assert this). @@ -989,8 +992,9 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) delete idxhash; delete h; return stream->num_val_fields + stream->num_idx_fields; - } - else + } + + else { // keep old one stream->currDict->Insert(idxhash, h); @@ -998,42 +1002,37 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) return stream->num_val_fields + stream->num_idx_fields; } } - - } + } // now we don't need h anymore - if we are here, the entry is updated and a new h is created. - if ( h ) + if ( h ) { delete h; h = 0; } - + Val* idxval; - if ( predidx != 0 ) + if ( predidx != 0 ) { idxval = RecordValToIndexVal(predidx); // I think there is an unref missing here. But if I insert is, it crashes :) - } - else + } + else idxval = ValueToIndexVal(stream->num_idx_fields, stream->itype, vals); - + Val* oldval = 0; - if ( updated == true ) + if ( updated == true ) { assert(stream->num_val_fields > 0); // in that case, we need the old value to send the event (if we send an event). oldval = stream->tab->Lookup(idxval, false); } - //i->tab->Assign(idxval, valval); assert(idxval); HashKey* k = stream->tab->ComputeHash(idxval); - if ( !k ) - { + if ( ! k ) reporter->InternalError("could not hash"); - assert(false); - } InputHash* ih = new InputHash(); ih->idxkey = new HashKey(k->Key(), k->Size(), k->Hash()); @@ -1044,63 +1043,62 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) stream->tab->Assign(idxval, k, valval); Unref(idxval); // asssign does not consume idxval. + if ( predidx != 0 ) Unref(predidx); stream->currDict->Insert(idxhash, ih); delete idxhash; - if ( stream->event ) + if ( stream->event ) { EnumVal* ev; int startpos = 0; Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); - if ( updated ) + if ( updated ) { // in case of update send back the old value. assert ( stream->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, oldval); - } - else + } + + else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - if ( stream->num_val_fields == 0 ) + if ( stream->num_val_fields == 0 ) { Ref(stream->description); SendEvent(stream->event, 3, stream->description->Ref(), ev, predidx); - } - else + } + else SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, valval->Ref()); - + } - } + } - - return stream->num_val_fields + stream->num_idx_fields; + return stream->num_val_fields + stream->num_idx_fields; } - -void Manager::EndCurrentSend(ReaderFrontend* reader) +void Manager::EndCurrentSend(ReaderFrontend* reader) { Stream *i = FindStream(reader); - if ( i == 0 ) + if ( i == 0 ) { reporter->InternalError("Unknown reader in EndCurrentSend"); return; } #ifdef DEBUG - DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", - i->name.c_str()); + DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", i->name.c_str()); #endif - if ( i->stream_type == EVENT_FILTER ) // nothing to do.. + if ( i->stream_type == EVENT_STREAM ) // nothing to do.. return; - assert(i->stream_type == TABLE_FILTER); + assert(i->stream_type == TABLE_STREAM); TableStream* stream = (TableStream*) i; // lastdict contains all deleted entries and should be empty apart from that @@ -1108,17 +1106,17 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) stream->lastDict->MakeRobustCookie(c); InputHash* ih; HashKey *lastDictIdxKey; - //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = stream->lastDict->NextEntry(lastDictIdxKey, c) ) ) + + while ( ( ih = stream->lastDict->NextEntry(lastDictIdxKey, c) ) ) { ListVal * idx = 0; Val *val = 0; - + Val* predidx = 0; EnumVal* ev = 0; int startpos = 0; - if ( stream->pred || stream->event ) + if ( stream->pred || stream->event ) { idx = stream->tab->RecoverIndex(ih->idxkey); assert(idx != 0); @@ -1128,7 +1126,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); } - if ( stream->pred ) + if ( stream->pred ) { // ask predicate, if we want to expire this element... @@ -1138,7 +1136,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) bool result = CallPred(stream->pred, 3, ev, predidx, val); - if ( result == false ) + if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... @@ -1147,10 +1145,10 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) stream->currDict->Insert(lastDictIdxKey, stream->lastDict->RemoveEntry(lastDictIdxKey)); delete lastDictIdxKey; continue; - } - } + } + } - if ( stream->event ) + if ( stream->event ) { Ref(predidx); Ref(val); @@ -1160,7 +1158,8 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) if ( predidx ) // if we have a stream or an event... Unref(predidx); - if ( ev ) + + if ( ev ) Unref(ev); Unref(stream->tab->Delete(ih->idxkey)); @@ -1172,54 +1171,57 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) stream->lastDict->Clear(); // should be empt. buti- well... who knows... delete(stream->lastDict); - stream->lastDict = stream->currDict; + stream->lastDict = stream->currDict; stream->currDict = new PDict(InputHash); stream->currDict->SetDeleteFunc(input_hash_delete_func); #ifdef DEBUG - DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s, queueing update_finished event", - i->name.c_str()); + DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s, queueing update_finished event", + i->name.c_str()); #endif // Send event that the current update is indeed finished. EventHandler* handler = event_registry->Lookup("Input::update_finished"); - if ( handler == 0 ) + if ( handler == 0 ) reporter->InternalError("Input::update_finished not found!"); SendEvent(handler, 2, new StringVal(i->name.c_str()), new StringVal(i->source.c_str())); } -void Manager::Put(ReaderFrontend* reader, Value* *vals) +void Manager::Put(ReaderFrontend* reader, Value* *vals) { Stream *i = FindStream(reader); - if ( i == 0 ) + if ( i == 0 ) { reporter->InternalError("Unknown reader in Put"); return; } - int readFields; - if ( i->stream_type == TABLE_FILTER ) + int readFields = 0; + + if ( i->stream_type == TABLE_STREAM ) readFields = PutTable(i, vals); - else if ( i->stream_type == EVENT_FILTER ) + + else if ( i->stream_type == EVENT_STREAM ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); readFields = SendEventStreamEvent(i, type, vals); - } - else + } + + else assert(false); - - for ( int i = 0; i < readFields; i++ ) + + for ( int i = 0; i < readFields; i++ ) delete vals[i]; - delete [] vals; + delete [] vals; } -int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) +int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) { assert(i); - assert(i->stream_type == EVENT_FILTER); + assert(i->stream_type == EVENT_STREAM); EventStream* stream = (EventStream*) i; Val *val; @@ -1227,71 +1229,77 @@ int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const * Ref(stream->description); out_vals.push_back(stream->description); // no tracking, send everything with a new event... - //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); out_vals.push_back(type); int position = 0; - if ( stream->want_record ) + + if ( stream->want_record ) { RecordVal * r = ValueToRecordVal(vals, stream->fields, &position); out_vals.push_back(r); } - else - { - for ( int j = 0; j < stream->fields->NumFields(); j++) + + else + { + for ( int j = 0; j < stream->fields->NumFields(); j++) { Val* val = 0; - if ( stream->fields->FieldType(j)->Tag() == TYPE_RECORD ) - val = ValueToRecordVal(vals, - stream->fields->FieldType(j)->AsRecordType(), + + if ( stream->fields->FieldType(j)->Tag() == TYPE_RECORD ) + val = ValueToRecordVal(vals, + stream->fields->FieldType(j)->AsRecordType(), &position); - else + + else { val = ValueToVal(vals[position], stream->fields->FieldType(j)); position++; } - out_vals.push_back(val); + + out_vals.push_back(val); } } SendEvent(stream->event, out_vals); return stream->fields->NumFields(); - } -int Manager::PutTable(Stream* i, const Value* const *vals) +int Manager::PutTable(Stream* i, const Value* const *vals) { assert(i); - assert(i->stream_type == TABLE_FILTER); - TableStream* stream = (TableStream*) i; + assert(i->stream_type == TABLE_STREAM); + TableStream* stream = (TableStream*) i; Val* idxval = ValueToIndexVal(stream->num_idx_fields, stream->itype, vals); Val* valval; int position = stream->num_idx_fields; - if ( stream->num_val_fields == 0 ) + + if ( stream->num_val_fields == 0 ) valval = 0; - else if ( stream->num_val_fields == 1 && stream->want_record == 0 ) + + else if ( stream->num_val_fields == 1 && stream->want_record == 0 ) valval = ValueToVal(vals[position], stream->rtype->FieldType(0)); - else + + else valval = ValueToRecordVal(vals, stream->rtype, &position); // if we have a subscribed event, we need to figure out, if this is an update or not // same for predicates - if ( stream->pred || stream->event ) + if ( stream->pred || stream->event ) { bool updated = false; Val* oldval = 0; - - if ( stream->num_val_fields > 0 ) + + if ( stream->num_val_fields > 0 ) { // in that case, we need the old value to send the event (if we send an event). oldval = stream->tab->Lookup(idxval, false); } - if ( oldval != 0 ) + if ( oldval != 0 ) { // it is an update updated = true; @@ -1300,27 +1308,27 @@ int Manager::PutTable(Stream* i, const Value* const *vals) // predicate if we want the update or not - if ( stream->pred ) + if ( stream->pred ) { EnumVal* ev; int startpos = 0; Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); Ref(valval); - if ( updated ) - ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, + if ( updated ) + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); - else - ev = new EnumVal(BifEnum::Input::EVENT_NEW, + else + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - + bool result; if ( stream->num_val_fields > 0 ) // we have values result = CallPred(stream->pred, 3, ev, predidx, valval); else // no values result = CallPred(stream->pred, 2, ev, predidx); - - if ( result == false ) + + if ( result == false ) { // do nothing Unref(idxval); @@ -1331,51 +1339,51 @@ int Manager::PutTable(Stream* i, const Value* const *vals) } - stream->tab->Assign(idxval, valval); + stream->tab->Assign(idxval, valval); - if ( stream->event ) - { + if ( stream->event ) + { EnumVal* ev; int startpos = 0; Val* predidx = ValueToRecordVal(vals, stream->itype, &startpos); - if ( updated ) - { + if ( updated ) + { // in case of update send back the old value. assert ( stream->num_val_fields > 0 ); - ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, oldval); - } - else + } + else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); if ( stream->num_val_fields == 0 ) - SendEvent(stream->event, 4, stream->description->Ref(), + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx); else - SendEvent(stream->event, 4, stream->description->Ref(), + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, valval->Ref()); } - + } - } + } + else // no predicates or other stuff stream->tab->Assign(idxval, valval); - return stream->num_idx_fields + stream->num_val_fields; } // Todo:: perhaps throw some kind of clear-event? -void Manager::Clear(ReaderFrontend* reader) +void Manager::Clear(ReaderFrontend* reader) { Stream *i = FindStream(reader); - if ( i == 0 ) + if ( i == 0 ) { reporter->InternalError("Unknown reader in Clear"); return; @@ -1386,17 +1394,17 @@ void Manager::Clear(ReaderFrontend* reader) i->name.c_str()); #endif - assert(i->stream_type == TABLE_FILTER); - TableStream* stream = (TableStream*) i; + assert(i->stream_type == TABLE_STREAM); + TableStream* stream = (TableStream*) i; stream->tab->RemoveAll(); } // put interface: delete old entry from table. -bool Manager::Delete(ReaderFrontend* reader, Value* *vals) +bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { Stream *i = FindStream(reader); - if ( i == 0 ) + if ( i == 0 ) { reporter->InternalError("Unknown reader in Delete"); return false; @@ -1405,19 +1413,19 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) bool success = false; int readVals = 0; - if ( i->stream_type == TABLE_FILTER ) + if ( i->stream_type == TABLE_STREAM ) { - TableStream* stream = (TableStream*) i; + TableStream* stream = (TableStream*) i; Val* idxval = ValueToIndexVal(stream->num_idx_fields, stream->itype, vals); assert(idxval != 0); readVals = stream->num_idx_fields + stream->num_val_fields; bool streamresult = true; - if ( stream->pred || stream->event ) + if ( stream->pred || stream->event ) { Val *val = stream->tab->Lookup(idxval); - if ( stream->pred ) + if ( stream->pred ) { Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); @@ -1426,7 +1434,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) streamresult = CallPred(stream->pred, 3, ev, predidx, val); - if ( streamresult == false ) + if ( streamresult == false ) { // keep it. Unref(idxval); @@ -1436,56 +1444,58 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) } // only if stream = true -> no streaming - if ( streamresult && stream->event ) + if ( streamresult && stream->event ) { Ref(idxval); assert(val != 0); - Ref(val); + Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); SendEvent(stream->event, 4, stream->description->Ref(), ev, idxval, val); } } // only if stream = true -> no streaming - if ( streamresult ) + if ( streamresult ) { Val* retptr = stream->tab->Delete(idxval); success = ( retptr != 0 ); - if ( !success ) + if ( ! success ) reporter->Error("Internal error while deleting values from input table"); else Unref(retptr); } - - } - else if ( i->stream_type == EVENT_FILTER ) + + } + + else if ( i->stream_type == EVENT_STREAM ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - readVals = SendEventStreamEvent(i, type, vals); + readVals = SendEventStreamEvent(i, type, vals); success = true; } - else + + else { assert(false); return false; } - for ( int i = 0; i < readVals; i++ ) + for ( int i = 0; i < readVals; i++ ) delete vals[i]; - delete [] vals; + delete [] vals; return success; - } + } -bool Manager::CallPred(Func* pred_func, const int numvals, ...) +bool Manager::CallPred(Func* pred_func, const int numvals, ...) { bool result; val_list vl(numvals); - + va_list lP; va_start(lP, numvals); - for ( int i = 0; i < numvals; i++ ) + for ( int i = 0; i < numvals; i++ ) vl.append( va_arg(lP, Val*) ); va_end(lP); @@ -1497,10 +1507,10 @@ bool Manager::CallPred(Func* pred_func, const int numvals, ...) return(result); } -bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) +bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); - if ( handler == 0 ) + if ( handler == 0 ) { reporter->Error("Event %s not found", name.c_str()); return false; @@ -1508,33 +1518,33 @@ bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) RecordType *type = handler->FType()->Args(); int num_event_vals = type->NumFields(); - if ( num_vals != num_event_vals ) + if ( num_vals != num_event_vals ) { reporter->Error("Wrong number of values for event %s", name.c_str()); return false; } val_list* vl = new val_list; - for ( int i = 0; i < num_vals; i++) + for ( int i = 0; i < num_vals; i++) vl->append(ValueToVal(vals[i], type->FieldType(i))); mgr.Dispatch(new Event(handler, vl)); - for ( int i = 0; i < num_vals; i++ ) + for ( int i = 0; i < num_vals; i++ ) delete vals[i]; - delete [] vals; + delete [] vals; return true; -} +} -void Manager::SendEvent(EventHandlerPtr ev, const int numvals, ...) +void Manager::SendEvent(EventHandlerPtr ev, const int numvals, ...) { val_list* vl = new val_list; - + va_list lP; va_start(lP, numvals); - for ( int i = 0; i < numvals; i++ ) + for ( int i = 0; i < numvals; i++ ) vl->append( va_arg(lP, Val*) ); va_end(lP); @@ -1545,8 +1555,8 @@ void Manager::SendEvent(EventHandlerPtr ev, const int numvals, ...) void Manager::SendEvent(EventHandlerPtr ev, list events) { val_list* vl = new val_list; - - for ( list::iterator i = events.begin(); i != events.end(); i++ ) + + for ( list::iterator i = events.begin(); i != events.end(); i++ ) { vl->append( *i ); } @@ -1554,31 +1564,31 @@ void Manager::SendEvent(EventHandlerPtr ev, list events) mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } -// Convert a bro list value to a bro record value. +// Convert a bro list value to a bro record value. // I / we could think about moving this functionality to val.cc -RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) +RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) { assert(position != 0 ); // we need the pointer to point to data; - if ( request_type->Tag() != TYPE_RECORD ) + if ( request_type->Tag() != TYPE_RECORD ) { reporter->InternalError("ListValToRecordVal called on non-record-value."); return 0; - } + } RecordVal* rec = new RecordVal(request_type->AsRecordType()); assert(list != 0); int maxpos = list->Length(); - for ( int i = 0; i < request_type->NumFields(); i++ ) + for ( int i = 0; i < request_type->NumFields(); i++ ) { assert ( (*position) <= maxpos ); Val* fieldVal = 0; if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) - fieldVal = ListValToRecordVal(list, request_type->FieldType(i)->AsRecordType(), position); - else + fieldVal = ListValToRecordVal(list, request_type->FieldType(i)->AsRecordType(), position); + else { fieldVal = list->Index(*position); (*position)++; @@ -1592,24 +1602,23 @@ RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, // Convert a threading value to a record value RecordVal* Manager::ValueToRecordVal(const Value* const *vals, - RecordType *request_type, int* position) + RecordType *request_type, int* position) { assert(position != 0); // we need the pointer to point to data. - if ( request_type->Tag() != TYPE_RECORD ) + if ( request_type->Tag() != TYPE_RECORD ) { reporter->InternalError("ValueToRecordVal called on non-record-value."); return 0; - } + } RecordVal* rec = new RecordVal(request_type->AsRecordType()); - for ( int i = 0; i < request_type->NumFields(); i++ ) + for ( int i = 0; i < request_type->NumFields(); i++ ) { - Val* fieldVal = 0; if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) - fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); - else + fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); + else { fieldVal = ValueToVal(vals[*position], request_type->FieldType(i)); (*position)++; @@ -1619,10 +1628,10 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals, } return rec; - } + } -// Count the length of the values -// used to create a correct length buffer for hashing later +// Count the length of the values used to create a correct length buffer for +// hashing later int Manager::GetValueLength(const Value* val) { assert( val->present ); // presence has to be checked elsewhere int length = 0; @@ -1642,7 +1651,7 @@ int Manager::GetValueLength(const Value* val) { length += sizeof(val->val.port_val.port); length += sizeof(val->val.port_val.proto); break; - + case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: @@ -1688,17 +1697,17 @@ int Manager::GetValueLength(const Value* val) { } break; - case TYPE_TABLE: + case TYPE_TABLE: { - for ( int i = 0; i < val->val.set_val.size; i++ ) + for ( int i = 0; i < val->val.set_val.size; i++ ) length += GetValueLength(val->val.set_val.vals[i]); break; } - case TYPE_VECTOR: + case TYPE_VECTOR: { int j = val->val.vector_val.size; - for ( int i = 0; i < j; i++ ) + for ( int i = 0; i < j; i++ ) length += GetValueLength(val->val.vector_val.vals[i]); break; } @@ -1708,12 +1717,12 @@ int Manager::GetValueLength(const Value* val) { } return length; - + } // Given a threading::value, copy the raw data bytes into *data and return how many bytes were copied. // Used for hashing the values for lookup in the bro table -int Manager::CopyValue(char *data, const int startpos, const Value* val) +int Manager::CopyValue(char *data, const int startpos, const Value* val) { assert( val->present ); // presence has to be checked elsewhere @@ -1722,42 +1731,37 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) case TYPE_INT: memcpy(data+startpos, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); return sizeof(val->val.int_val); - break; case TYPE_COUNT: case TYPE_COUNTER: memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); return sizeof(val->val.uint_val); - break; - case TYPE_PORT: + case TYPE_PORT: { int length = 0; - memcpy(data+startpos, (const void*) &(val->val.port_val.port), + memcpy(data+startpos, (const void*) &(val->val.port_val.port), sizeof(val->val.port_val.port)); length += sizeof(val->val.port_val.port); - memcpy(data+startpos+length, (const void*) &(val->val.port_val.proto), + memcpy(data+startpos+length, (const void*) &(val->val.port_val.proto), sizeof(val->val.port_val.proto)); length += sizeof(val->val.port_val.proto); return length; - break; } - + case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - memcpy(data+startpos, (const void*) &(val->val.double_val), + memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); return sizeof(val->val.double_val); - break; case TYPE_STRING: case TYPE_ENUM: { memcpy(data+startpos, val->val.string_val->c_str(), val->val.string_val->length()); return val->val.string_val->size(); - break; } case TYPE_ADDR: @@ -1768,86 +1772,89 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) length = sizeof(val->val.addr_val.in.in4); memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in4), length); break; + case IPv6: length = sizeof(val->val.addr_val.in.in6); memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in6), length); break; + default: assert(false); } + return length; } - break; - - case TYPE_SUBNET: + + case TYPE_SUBNET: { int length; switch ( val->val.subnet_val.prefix.family ) { case IPv4: length = sizeof(val->val.addr_val.in.in4); - memcpy(data + startpos, + memcpy(data + startpos, (const char*) &(val->val.subnet_val.prefix.in.in4), length); break; + case IPv6: length = sizeof(val->val.addr_val.in.in6); - memcpy(data + startpos, + memcpy(data + startpos, (const char*) &(val->val.subnet_val.prefix.in.in4), length); break; + default: assert(false); } + int lengthlength = sizeof(val->val.subnet_val.length); - memcpy(data + startpos + length , + memcpy(data + startpos + length , (const char*) &(val->val.subnet_val.length), lengthlength); length += lengthlength; + return length; } - break; - case TYPE_TABLE: + case TYPE_TABLE: { int length = 0; int j = val->val.set_val.size; - for ( int i = 0; i < j; i++ ) + for ( int i = 0; i < j; i++ ) length += CopyValue(data, startpos+length, val->val.set_val.vals[i]); return length; - break; } - case TYPE_VECTOR: + case TYPE_VECTOR: { int length = 0; int j = val->val.vector_val.size; - for ( int i = 0; i < j; i++ ) + for ( int i = 0; i < j; i++ ) length += CopyValue(data, startpos+length, val->val.vector_val.vals[i]); return length; - break; } default: reporter->InternalError("unsupported type %d for CopyValue", val->type); return 0; } - + assert(false); return 0; } // Hash num_elements threading values and return the HashKey for them. At least one of the vals has to be ->present. -HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) +HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { int length = 0; - for ( int i = 0; i < num_elements; i++ ) + for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; if ( val->present ) length += GetValueLength(val); } - if ( length == 0 ) + if ( length == 0 ) { reporter->Error("Input reader sent line where all elements are null values. Ignoring line"); return NULL; @@ -1855,10 +1862,10 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) int position = 0; char *data = (char*) malloc(length); - if ( data == 0 ) + if ( data == 0 ) reporter->InternalError("Could not malloc?"); - for ( int i = 0; i < num_elements; i++ ) + for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; if ( val->present ) @@ -1873,16 +1880,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) } // convert threading value to Bro value -Val* Manager::ValueToVal(const Value* val, BroType* request_type) +Val* Manager::ValueToVal(const Value* val, BroType* request_type) { - - if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) + + if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; } - if ( !val->present ) + if ( !val->present ) return 0; // unset field switch ( val->type ) { @@ -1894,24 +1901,20 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) case TYPE_COUNT: case TYPE_COUNTER: return new Val(val->val.uint_val, val->type); - break; - + case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: return new Val(val->val.double_val, val->type); - break; case TYPE_STRING: { BroString *s = new BroString(*(val->val.string_val)); return new StringVal(s); - break; } - + case TYPE_PORT: return new PortVal(val->val.port_val.port, val->val.port_val.proto); - break; case TYPE_ADDR: { @@ -1920,12 +1923,15 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) case IPv4: addr = new IPAddr(val->val.addr_val.in.in4); break; + case IPv6: addr = new IPAddr(val->val.addr_val.in.in6); break; + default: assert(false); } + AddrVal* addrval = new AddrVal(*addr); delete addr; return addrval; @@ -1938,19 +1944,21 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) case IPv4: addr = new IPAddr(val->val.subnet_val.prefix.in.in4); break; + case IPv6: addr = new IPAddr(val->val.subnet_val.prefix.in.in6); break; + default: assert(false); } + SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length); delete addr; return subnetval; - break; } - case TYPE_TABLE: + case TYPE_TABLE: { // all entries have to have the same type... BroType* type = request_type->AsTableType()->Indices()->PureType(); @@ -1958,7 +1966,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) set_index->Append(type->Ref()); SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); - for ( int i = 0; i < val->val.set_val.size; i++ ) + for ( int i = 0; i < val->val.set_val.size; i++ ) { Val* assignval = ValueToVal( val->val.set_val.vals[i], type ); t->Assign(assignval, 0); @@ -1967,21 +1975,19 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) Unref(s); return t; - break; } - case TYPE_VECTOR: + case TYPE_VECTOR: { // all entries have to have the same type... BroType* type = request_type->AsVectorType()->YieldType(); VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); - for ( int i = 0; i < val->val.vector_val.size; i++ ) + for ( int i = 0; i < val->val.vector_val.size; i++ ) v->Assign(i, ValueToVal( val->val.set_val.vals[i], type ), 0); Unref(vt); return v; - break; } case TYPE_ENUM: { @@ -1990,14 +1996,13 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) string module = extract_module_name(val->val.string_val->c_str()); string var = extract_var_name(val->val.string_val->c_str()); bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str()); - if ( index == -1 ) - reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", + if ( index == -1 ) + reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", module.c_str(), var.c_str()); - + return new EnumVal(index, request_type->Ref()->AsEnumType() ); - break; } - + default: reporter->InternalError("unsupported type for input_read"); @@ -2006,22 +2011,22 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) assert(false); return NULL; } - + Manager::Stream* Manager::FindStream(const string &name) { for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s).second->name == name ) + if ( (*s).second->name == name ) return (*s).second; } return 0; } -Manager::Stream* Manager::FindStream(ReaderFrontend* reader) +Manager::Stream* Manager::FindStream(ReaderFrontend* reader) { map::iterator s = readers.find(reader); - if ( s != readers.end() ) + if ( s != readers.end() ) return s->second; return 0; diff --git a/src/input/Manager.h b/src/input/Manager.h index d15febe0d6..984fcf3841 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -1,6 +1,6 @@ // See the file "COPYING" in the main distribution directory for copyright. // -// Class for managing input streams +// Class for managing input streams. #ifndef INPUT_MANAGER_H #define INPUT_MANAGER_H @@ -16,7 +16,7 @@ namespace input { class ReaderFrontend; -class ReaderBackend; +class ReaderBackend; /** * Singleton class for managing input streams. @@ -25,58 +25,60 @@ class Manager { public: /** * Constructor. - */ + */ Manager(); /** * Destructor. */ ~Manager(); - + /** - * Creates a new input stream which will write the data from the data source into + * Creates a new input stream which will write the data from the data + * source into a table. * - * @param description A record of script type \c Input:StreamDescription. + * @param description A record of script type \c + * Input:StreamDescription. * * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. - */ - bool CreateTableStream(RecordVal* description); + */ + bool CreateTableStream(RecordVal* description); /** * Creates a new input stream which sends events for read input data. * - * @param description A record of script type \c Input:StreamDescription. + * @param description A record of script type \c + * Input:StreamDescription. * * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. - */ - bool CreateEventStream(RecordVal* description); - + */ + bool CreateEventStream(RecordVal* description); /** - * Force update on a input stream. - * Forces a re-read of the whole input source. - * Usually used, when an input stream is opened in managed mode. - * Otherwise, this can be used to trigger a input source check before a heartbeat message arrives. - * May be ignored by the reader. + * Force update on a input stream. Forces a re-read of the whole + * input source. Usually used when an input stream is opened in + * managed mode. Otherwise, this can be used to trigger a input + * source check before a heartbeat message arrives. May be ignored by + * the reader. * - * @param id The enum value corresponding the input stream. + * @param id The enum value corresponding the input stream. * * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ bool ForceUpdate(const string &id); - + /** - * Deletes an existing input stream + * Deletes an existing input stream. * - * @param id The enum value corresponding the input stream. + * @param id The enum value corresponding the input stream. * * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ - bool RemoveStream(const string &id); + bool RemoveStream(const string &id); protected: friend class ReaderFrontend; @@ -88,90 +90,100 @@ protected: friend class EndCurrentSendMessage; friend class ReaderClosedMessage; - // For readers to write to input stream in direct mode (reporting new/deleted values directly) - // Functions take ownership of threading::Value fields + // For readers to write to input stream in direct mode (reporting + // new/deleted values directly). Functions take ownership of + // threading::Value fields. void Put(ReaderFrontend* reader, threading::Value* *vals); void Clear(ReaderFrontend* reader); bool Delete(ReaderFrontend* reader, threading::Value* *vals); - // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) - // Functions take ownership of threading::Value fields + // For readers to write to input stream in indirect mode (manager is + // monitoring new/deleted values) Functions take ownership of + // threading::Value fields. void SendEntry(ReaderFrontend* reader, threading::Value* *vals); void EndCurrentSend(ReaderFrontend* reader); - - // Allows readers to directly send Bro events. - // The num_vals and vals must be the same the named event expects. - // Takes ownership of threading::Value fields + + // Allows readers to directly send Bro events. The num_vals and vals + // must be the same the named event expects. Takes ownership of + // threading::Value fields. bool SendEvent(const string& name, const int num_vals, threading::Value* *vals); // Instantiates a new ReaderBackend of the given type (note that - // doing so creates a new thread!). - ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); - - // Functions are called from the ReaderBackend to notify the manager, that a stream has been removed - // or a stream has been closed. - // Used to prevent race conditions where data for a specific stream is still in the queue when the - // RemoveStream directive is executed by the main thread. - // This makes sure all data that has ben queued for a stream is still received. + // doing so creates a new thread!). + ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); + + // Function called from the ReaderBackend to notify the manager that + // a stream has been removed or a stream has been closed. Used to + // prevent race conditions where data for a specific stream is still + // in the queue when the RemoveStream directive is executed by the + // main thread. This makes sure all data that has ben queued for a + // stream is still received. bool RemoveStreamContinuation(ReaderFrontend* reader); - + private: class Stream; class TableStream; class EventStream; - - bool CreateStream(Stream*, RecordVal* description); - // SendEntry implementation for Table stream - int SendEntryTable(Stream* i, const threading::Value* const *vals); + bool CreateStream(Stream*, RecordVal* description); - // Put implementation for Table stream - int PutTable(Stream* i, const threading::Value* const *vals); + // SendEntry implementation for Table stream. + int SendEntryTable(Stream* i, const threading::Value* const *vals); - // SendEntry and Put implementation for Event stream + // Put implementation for Table stream. + int PutTable(Stream* i, const threading::Value* const *vals); + + // SendEntry and Put implementation for Event stream. int SendEventStreamEvent(Stream* i, EnumVal* type, const threading::Value* const *vals); - // Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types - // from the log framework + // Checks that a Bro type can be used for data reading. The + // equivalend in threading cannot be used, because we have support + // different types from the log framework bool IsCompatibleType(BroType* t, bool atomic_only=false); - // Check if a record is made up of compatible types and return a list of all fields that are in the record in order. - // Recursively unrolls records + // Check if a record is made up of compatible types and return a list + // of all fields that are in the record in order. Recursively unrolls + // records bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); // Send events - void SendEvent(EventHandlerPtr ev, const int numvals, ...); - void SendEvent(EventHandlerPtr ev, list events); + void SendEvent(EventHandlerPtr ev, const int numvals, ...); + void SendEvent(EventHandlerPtr ev, list events); - // Call predicate function and return result + // Call predicate function and return result. bool CallPred(Func* pred_func, const int numvals, ...); - // get a hashkey for a set of threading::Values + // Get a hashkey for a set of threading::Values. HashKey* HashValues(const int num_elements, const threading::Value* const *vals); - // Get the memory used by a specific value + // Get the memory used by a specific value. int GetValueLength(const threading::Value* val); - // Copies the raw data in a specific threading::Value to position sta + + // Copies the raw data in a specific threading::Value to position + // startpos. int CopyValue(char *data, const int startpos, const threading::Value* val); - // Convert Threading::Value to an internal Bro Type (works also with Records) + // Convert Threading::Value to an internal Bro Type (works also with + // Records). Val* ValueToVal(const threading::Value* val, BroType* request_type); - // Convert Threading::Value to an internal Bro List type + // Convert Threading::Value to an internal Bro List type. Val* ValueToIndexVal(int num_fields, const RecordType* type, const threading::Value* const *vals); - // Converts a threading::value to a record type. mostly used by ValueToVal - RecordVal* ValueToRecordVal(const threading::Value* const *vals, RecordType *request_type, int* position); + // Converts a threading::value to a record type. Mostly used by + // ValueToVal. + RecordVal* ValueToRecordVal(const threading::Value* const *vals, RecordType *request_type, int* position); + Val* RecordValToIndexVal(RecordVal *r); - - // Converts a Bro ListVal to a RecordVal given the record type + + // Converts a Bro ListVal to a RecordVal given the record type. RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); Stream* FindStream(const string &name); Stream* FindStream(ReaderFrontend* reader); - enum StreamType { TABLE_FILTER, EVENT_FILTER }; - + enum StreamType { TABLE_STREAM, EVENT_STREAM }; + map readers; }; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index c625301383..328e0bc535 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -15,7 +15,7 @@ public: : threading::OutputMessage("Put", reader), val(val) {} - virtual bool Process() + virtual bool Process() { input_mgr->Put(Object(), val); return true; @@ -31,7 +31,7 @@ public: : threading::OutputMessage("Delete", reader), val(val) {} - virtual bool Process() + virtual bool Process() { return input_mgr->Delete(Object(), val); } @@ -45,7 +45,7 @@ public: ClearMessage(ReaderFrontend* reader) : threading::OutputMessage("Clear", reader) {} - virtual bool Process() + virtual bool Process() { input_mgr->Clear(Object()); return true; @@ -60,14 +60,14 @@ public: : threading::OutputMessage("SendEvent", reader), name(name), num_vals(num_vals), val(val) {} - virtual bool Process() + virtual bool Process() { bool success = input_mgr->SendEvent(name, num_vals, val); - if ( !success ) + if ( ! success ) reporter->Error("SendEvent for event %s failed", name.c_str()); - return true; // we do not want to die if sendEvent fails because the event did not return. + return true; // We do not want to die if sendEvent fails because the event did not return. } private: @@ -82,7 +82,7 @@ public: : threading::OutputMessage("SendEntry", reader), val(val) { } - virtual bool Process() + virtual bool Process() { input_mgr->SendEntry(Object(), val); return true; @@ -97,7 +97,7 @@ public: EndCurrentSendMessage(ReaderFrontend* reader) : threading::OutputMessage("EndCurrentSend", reader) {} - virtual bool Process() + virtual bool Process() { input_mgr->EndCurrentSend(Object()); return true; @@ -111,7 +111,7 @@ public: ReaderClosedMessage(ReaderFrontend* reader) : threading::OutputMessage("ReaderClosed", reader) {} - virtual bool Process() + virtual bool Process() { return input_mgr->RemoveStreamContinuation(Object()); } @@ -127,49 +127,46 @@ public: : threading::OutputMessage("Disable", writer) {} virtual bool Process() - { - Object()->SetDisable(); - return true; + { + Object()->SetDisable(); + return true; } }; ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() { - buf = 0; - buf_len = 1024; disabled = true; // disabled will be set correcty in init. - frontend = arg_frontend; SetName(frontend->Name()); } -ReaderBackend::~ReaderBackend() - { +ReaderBackend::~ReaderBackend() + { } -void ReaderBackend::Put(Value* *val) +void ReaderBackend::Put(Value* *val) { SendOut(new PutMessage(frontend, val)); } -void ReaderBackend::Delete(Value* *val) +void ReaderBackend::Delete(Value* *val) { SendOut(new DeleteMessage(frontend, val)); } -void ReaderBackend::Clear() +void ReaderBackend::Clear() { SendOut(new ClearMessage(frontend)); } -void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) +void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) { SendOut(new SendEventMessage(frontend, name, num_vals, vals)); - } + } -void ReaderBackend::EndCurrentSend() +void ReaderBackend::EndCurrentSend() { SendOut(new EndCurrentSendMessage(frontend)); } @@ -179,19 +176,19 @@ void ReaderBackend::SendEntry(Value* *vals) SendOut(new SendEntryMessage(frontend, vals)); } -bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, - const threading::Field* const* arg_fields) +bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, + const threading::Field* const* arg_fields) { source = arg_source; SetName("InputReader/"+source); num_fields = arg_num_fields; - fields = arg_fields; + fields = arg_fields; // disable if DoInit returns error. int success = DoInit(arg_source, mode, arg_num_fields, arg_fields); - if ( !success ) + if ( ! success ) { Error("Init failed"); DisableFrontend(); @@ -202,30 +199,30 @@ bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, return success; } -void ReaderBackend::Close() +void ReaderBackend::Close() { DoClose(); disabled = true; DisableFrontend(); SendOut(new ReaderClosedMessage(frontend)); - if ( fields != 0 ) + if ( fields != 0 ) { - for ( unsigned int i = 0; i < num_fields; i++ ) + for ( unsigned int i = 0; i < num_fields; i++ ) delete(fields[i]); - delete[] (fields); + delete [] (fields); fields = 0; } } -bool ReaderBackend::Update() +bool ReaderBackend::Update() { - if ( disabled ) + if ( disabled ) return false; bool success = DoUpdate(); - if ( !success ) + if ( ! success ) DisableFrontend(); return success; @@ -233,8 +230,9 @@ bool ReaderBackend::Update() void ReaderBackend::DisableFrontend() { - disabled = true; - // we also set disabled here, because there still may be other messages queued and we will dutifully ignore these from now + // We also set disabled here, because there still may be other + // messages queued and we will dutifully ignore these from now. + disabled = true; SendOut(new DisableMessage(frontend)); } @@ -244,9 +242,9 @@ bool ReaderBackend::DoHeartbeat(double network_time, double current_time) return true; } -TransportProto ReaderBackend::StringToProto(const string &proto) +TransportProto ReaderBackend::StringToProto(const string &proto) { - if ( proto == "unknown" ) + if ( proto == "unknown" ) return TRANSPORT_UNKNOWN; else if ( proto == "tcp" ) return TRANSPORT_TCP; @@ -261,8 +259,8 @@ TransportProto ReaderBackend::StringToProto(const string &proto) } -// more or less verbose copy from IPAddr.cc -- which uses reporter -Value::addr_t ReaderBackend::StringToAddr(const string &s) +// More or less verbose copy from IPAddr.cc -- which uses reporter. +Value::addr_t ReaderBackend::StringToAddr(const string &s) { Value::addr_t val; @@ -270,9 +268,9 @@ Value::addr_t ReaderBackend::StringToAddr(const string &s) { val.family = IPv4; - if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) + if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) { - Error(Fmt("Bad addres: %s", s.c_str())); + Error(Fmt("Bad address: %s", s.c_str())); memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); } @@ -283,7 +281,7 @@ Value::addr_t ReaderBackend::StringToAddr(const string &s) val.family = IPv6; if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) { - Error(Fmt("Bad IP address: %s", s.c_str())); + Error(Fmt("Bad address: %s", s.c_str())); memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); } } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index b4d9101bc8..ae8437b08c 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -12,13 +12,13 @@ namespace input { class ReaderFrontend; /** - * Base class for reader implementation. When the input:Manager creates a - * new input stream, it instantiates a ReaderFrontend. That then in turn - * creates a ReaderBackend of the right type. The frontend then forwards - * message over the backend as its methods are called. + * Base class for reader implementation. When the input:Manager creates a new + * input stream, it instantiates a ReaderFrontend. That then in turn creates + * a ReaderBackend of the right type. The frontend then forwards messages + * over the backend as its methods are called. * - * All of this methods must be called only from the corresponding child - * thread (the constructor is the one exception.) + * All methods must be called only from the corresponding child thread (the + * constructor is the one exception.) */ class ReaderBackend : public threading::MsgThread { public: @@ -27,54 +27,51 @@ public: * * @param frontend The frontend reader that created this backend. The * *only* purpose of this value is to be passed back via messages as - * a argument to callbacks. One must not otherwise access the + * an argument to callbacks. One must not otherwise access the * frontend, it's running in a different thread. - * - * @param frontend pointer to the reader frontend - */ + */ ReaderBackend(ReaderFrontend* frontend); - + /** * Destructor. - */ + */ virtual ~ReaderBackend(); /** * One-time initialization of the reader to define the input source. * - * @param arg_source A string left to the interpretation of the reader - * implementation; it corresponds to the value configured on the - * script-level for the input stream. + * @param arg_source A string left to the interpretation of the + * reader implementation; it corresponds to the value configured on + * the script-level for the input stream. * - * @param num_fields The number of log fields for the stream. + * @param fields An array of size \a num_fields with the input + * fields. The method takes ownership of the array. * - * @param fields An array of size \a num_fields with the log fields. - * The methods takes ownership of the array. - * - * @param mode the opening mode for the input source + * @param mode The opening mode for the input source as one of the + * Input::Mode script constants. * - * @param arg_num_fields number of fields contained in \a fields + * @param arg_num_fields Number of fields contained in \a fields. * - * @param fields the types and names of the fields to be retrieved - * from the input source + * @param fields The types and names of the fields to be retrieved + * from the input source. * * @return False if an error occured. */ bool Init(string arg_source, int mode, int arg_num_fields, const threading::Field* const* fields); /** - * Finishes reading from this input stream in a regular fashion. Must not be - * called if an error has been indicated earlier. After calling this, - * no further reading from the stream can be performed + * Finishes reading from this input stream in a regular fashion. Must + * not be called if an error has been indicated earlier. After + * calling this, no further reading from the stream can be performed. * * @return False if an error occured. */ void Close(); /** - * Force trigger an update of the input stream. - * The action that will be taken depends on the current read mode and the - * individual input backend + * Force trigger an update of the input stream. The action that will + * be taken depends on the current read mode and the individual input + * backend. * * An backend can choose to ignore this. * @@ -84,16 +81,17 @@ public: /** * Disables the frontend that has instantiated this backend. Once - * disabled,the frontend will not send any further message over. + * disabled, the frontend will not send any further message over. */ - void DisableFrontend(); - + void DisableFrontend(); + protected: - // Methods that have to be overwritten by the individual readers - + // Methods that have to be overwritten by the individual readers + /** - * Reader-specific intialization method. Note that data may only be - * read from the input source after the Start function has been called. + * Reader-specific intialization method. Note that data may only be + * read from the input source after the Init() function has been + * called. * * A reader implementation must override this method. If it returns * false, it will be assumed that a fatal error has occured that @@ -105,39 +103,39 @@ protected: /** * Reader-specific method implementing input finalization at - * termination. + * termination. * * A reader implementation must override this method but it can just - * ignore calls if an input source must not be closed. + * ignore calls if an input source can't actually be closed. * - * After the method is called, the writer will be deleted. If an error occurs - * during shutdown, an implementation should also call Error() to indicate what - * happened. - */ + * After the method is called, the writer will be deleted. If an + * error occurs during shutdown, an implementation should also call + * Error() to indicate what happened. + */ virtual void DoClose() = 0; /** - * Reader-specific method implementing the forced update trigger + * Reader-specific method implementing the forced update trigger. * - * A reader implementation must override this method but it can just ignore - * calls, if a forced update does not fit the input source or the current input - * reading mode. + * A reader implementation must override this method but it can just + * ignore calls if a forced update does not fit the input source or + * the current input reading mode. * - * If it returns false, it will be assumed that a fatal error has occured - * that prevents the reader from further operation; it will then be - * disabled and eventually deleted. When returning false, an implementation - * should also call Error to indicate what happened. + * If it returns false, it will be assumed that a fatal error has + * occured that prevents the reader from further operation; it will + * then be disabled and eventually deleted. When returning false, an + * implementation should also call Error to indicate what happened. */ virtual bool DoUpdate() = 0; - + /** * Returns the input source as passed into the constructor. */ const string Source() const { return source; } /** - * Method allowing a reader to send a specified bro event. - * Vals must match the values expected by the bro event. + * Method allowing a reader to send a specified Bro event. Vals must + * match the values expected by the bro event. * * @param name name of the bro event to send * @@ -147,30 +145,33 @@ protected: */ void SendEvent(const string& name, const int num_vals, threading::Value* *vals); - // Content-sending-functions (simple mode). Including table-specific stuff that - // simply is not used if we have no table + // Content-sending-functions (simple mode). Including table-specific + // stuff that simply is not used if we have no table. + /** - * Method allowing a reader to send a list of values read for a specific stream - * back to the manager. + * Method allowing a reader to send a list of values read from a + * specific stream back to the manager in simple mode. * - * If the stream is a table stream, the values are inserted into the table; - * if it is an event stream, the event is raised. + * If the stream is a table stream, the values are inserted into the + * table; if it is an event stream, the event is raised. * * @param val list of threading::Values expected by the stream */ void Put(threading::Value* *val); /** - * Method allowing a reader to delete a specific value from a bro table. + * Method allowing a reader to delete a specific value from a Bro + * table. * - * If the receiving stream is an event stream, only a removed event is raised + * If the receiving stream is an event stream, only a removed event + * is raised. * * @param val list of threading::Values expected by the stream */ void Delete(threading::Value* *val); /** - * Method allowing a reader to clear a value from a bro table. + * Method allowing a reader to clear a Bro table. * * If the receiving stream is an event stream, this is ignored. * @@ -178,26 +179,25 @@ protected: void Clear(); // Content-sending-functions (tracking mode): Only changed lines are propagated. - /** - * Method allowing a reader to send a list of values read for a specific stream - * back to the manager. + * Method allowing a reader to send a list of values read from + * specific stream back to the manager in tracking mode. * - * If the stream is a table stream, the values are inserted into the table; - * if it is an event stream, the event is raised. + * If the stream is a table stream, the values are inserted into the + * table; if it is an event stream, the event is raised. * * @param val list of threading::Values expected by the stream */ void SendEntry(threading::Value* *vals); /** - * Method telling the manager, that the current list of entries sent by SendEntry - * is finished. - * - * For table streams, all entries that were not updated since the last EndCurrentSend - * will be deleted, because they are no longer present in the input source + * Method telling the manager, that the current list of entries sent + * by SendEntry is finished. * + * For table streams, all entries that were not updated since the + * last EndCurrentSend will be deleted, because they are no longer + * present in the input source */ void EndCurrentSend(); @@ -207,14 +207,14 @@ protected: * This method can be overridden but once must call * ReaderBackend::DoHeartbeat(). */ - virtual bool DoHeartbeat(double network_time, double current_time); + virtual bool DoHeartbeat(double network_time, double current_time); /** * Utility function for Readers - convert a string into a TransportProto * * @param proto the transport protocol */ - TransportProto StringToProto(const string &proto); + TransportProto StringToProto(const string &proto); /** * Utility function for Readers - convert a string into a Value::addr_t @@ -224,20 +224,16 @@ protected: threading::Value::addr_t StringToAddr(const string &addr); private: - // Frontend that instantiated us. This object must not be access from - // this class, it's running in a different thread! - ReaderFrontend* frontend; + // Frontend that instantiated us. This object must not be accessed + // from this class, it's running in a different thread! + ReaderFrontend* frontend; string source; - + bool disabled; - // For implementing Fmt(). - char* buf; - unsigned int buf_len; - unsigned int num_fields; - const threading::Field* const * fields; // raw mapping + const threading::Field* const * fields; // raw mapping }; } diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index f61fd357b9..75bb7fec50 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -12,11 +12,15 @@ namespace input { class InitMessage : public threading::InputMessage { public: - InitMessage(ReaderBackend* backend, const string source, const int mode, const int num_fields, const threading::Field* const* fields) + InitMessage(ReaderBackend* backend, const string source, const int mode, + const int num_fields, const threading::Field* const* fields) : threading::InputMessage("Init", backend), source(source), mode(mode), num_fields(num_fields), fields(fields) { } - virtual bool Process() { return Object()->Init(source, mode, num_fields, fields); } + virtual bool Process() + { + return Object()->Init(source, mode, num_fields, fields); + } private: const string source; @@ -46,7 +50,7 @@ public: }; -ReaderFrontend::ReaderFrontend(bro_int_t type) +ReaderFrontend::ReaderFrontend(bro_int_t type) { disabled = initialized = false; ty_name = ""; @@ -56,12 +60,12 @@ ReaderFrontend::ReaderFrontend(bro_int_t type) backend->Start(); } -ReaderFrontend::~ReaderFrontend() +ReaderFrontend::~ReaderFrontend() { } -void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, - const threading::Field* const* fields) +void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, + const threading::Field* const* fields) { if ( disabled ) return; @@ -73,14 +77,14 @@ void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, initialized = true; backend->SendIn(new InitMessage(backend, arg_source, mode, num_fields, fields)); - } + } -void ReaderFrontend::Update() +void ReaderFrontend::Update() { - if ( disabled ) + if ( disabled ) return; - if ( !initialized ) + if ( ! initialized ) { reporter->Error("Tried to call update on uninitialized reader"); return; @@ -89,12 +93,12 @@ void ReaderFrontend::Update() backend->SendIn(new UpdateMessage(backend)); } -void ReaderFrontend::Close() +void ReaderFrontend::Close() { - if ( disabled ) + if ( disabled ) return; - - if ( !initialized ) + + if ( ! initialized ) { reporter->Error("Tried to call finish on uninitialized reader"); return; diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index 88cf60804e..c18e22a064 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -29,14 +29,14 @@ public: * corresponding type. * * Frontends must only be instantiated by the main thread. - */ + */ ReaderFrontend(bro_int_t type); /** * Destructor. * * Frontends must only be destroyed by the main thread. - */ + */ virtual ~ReaderFrontend(); /** @@ -47,37 +47,39 @@ public: * sends a message back that will asynchronously call Disable(). * * See ReaderBackend::Init() for arguments. + * * This method must only be called from the main thread. - */ + */ void Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* fields); /** - * Force an update of the current input source. Actual action depends on - * the opening mode and on the input source. + * Force an update of the current input source. Actual action depends + * on the opening mode and on the input source. * * This method generates a message to the backend reader and triggers * the corresponding message there. + * * This method must only be called from the main thread. */ void Update(); /** - * Finalizes writing to this tream. + * Finalizes reading from this stream. * * This method generates a message to the backend reader and triggers - * the corresponding message there. - * This method must only be called from the main thread. - */ + * the corresponding message there. This method must only be called + * from the main thread. + */ void Close(); /** * Disables the reader frontend. From now on, all method calls that * would normally send message over to the backend, turn into no-ops. - * Note though that it does not stop the backend itself, use Finsh() + * Note though that it does not stop the backend itself, use Finish() * to do that as well (this method is primarily for use as callback * when the backend wants to disable the frontend). * - * Disabled frontend will eventually be discarded by the + * Disabled frontends will eventually be discarded by the * input::Manager. * * This method must only be called from the main thread. @@ -85,9 +87,10 @@ public: void SetDisable() { disabled = true; } /** - * Returns true if the reader frontend has been disabled with SetDisable(). + * Returns true if the reader frontend has been disabled with + * SetDisable(). */ - bool Disabled() { return disabled; } + bool Disabled() { return disabled; } /** * Returns a descriptive name for the reader, including the type of @@ -101,18 +104,21 @@ protected: friend class Manager; /** - * Returns the source as passed into the constructor + * Returns the source as passed into the constructor. */ - const string Source() const { return source; }; + const string& Source() const { return source; }; - string ty_name; // Name of the backend type. Set by the manager. + /** + * Returns the name of the backend's type. + */ + const string& TypeName() const { return ty_name; } private: - ReaderBackend* backend; // The backend we have instanatiated. + ReaderBackend* backend; // The backend we have instanatiated. string source; + string ty_name; // Backend type, set by manager. bool disabled; // True if disabled. - bool initialized; // True if initialized. - + bool initialized; // True if initialized. }; } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 8223d6e201..157ea90916 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -20,8 +20,7 @@ using namespace input::reader; using threading::Value; using threading::Field; - -FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) : name(arg_name), type(arg_type) { position = arg_position; @@ -29,8 +28,8 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int present = true; } -FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, - const TypeTag& arg_subtype, int arg_position) +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, + const TypeTag& arg_subtype, int arg_position) : name(arg_name), type(arg_type), subtype(arg_subtype) { position = arg_position; @@ -38,14 +37,14 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, present = true; } -FieldMapping::FieldMapping(const FieldMapping& arg) +FieldMapping::FieldMapping(const FieldMapping& arg) : name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present) { position = arg.position; secondary_position = arg.secondary_position; } -FieldMapping FieldMapping::subType() +FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } @@ -54,23 +53,23 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) { file = 0; - - separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), + separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), BifConst::InputAscii::separator->Len()); - if ( separator.size() != 1 ) + + if ( separator.size() != 1 ) Error("separator length has to be 1. Separator will be truncated."); set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), BifConst::InputAscii::set_separator->Len()); - if ( set_separator.size() != 1 ) + + if ( set_separator.size() != 1 ) Error("set_separator length has to be 1. Separator will be truncated."); - empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), + empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), BifConst::InputAscii::empty_field->Len()); - - unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), - BifConst::InputAscii::unset_field->Len()); + unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), + BifConst::InputAscii::unset_field->Len()); } Ascii::~Ascii() @@ -80,7 +79,7 @@ Ascii::~Ascii() void Ascii::DoClose() { - if ( file != 0 ) + if ( file != 0 ) { file->close(); delete(file); @@ -93,26 +92,26 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c fname = path; mode = arg_mode; mtime = 0; - + num_fields = arg_num_fields; fields = arg_fields; - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) + if ( (mode != MANUAL) && (mode != REREAD) && (mode != STREAM) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; - } + } file = new ifstream(path.c_str()); - if ( !file->is_open() ) + if ( ! file->is_open() ) { Error(Fmt("Init: cannot open %s", fname.c_str())); delete(file); file = 0; return false; } - - if ( ReadHeader(false) == false ) + + if ( ReadHeader(false) == false ) { Error(Fmt("Init: cannot open %s; headers are incorrect", fname.c_str())); file->close(); @@ -120,22 +119,22 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c file = 0; return false; } - + DoUpdate(); return true; } -bool Ascii::ReadHeader(bool useCached) +bool Ascii::ReadHeader(bool useCached) { // try to read the header line... string line; map ifields; - if ( !useCached ) + if ( ! useCached ) { - if ( !GetLine(line) ) + if ( ! GetLine(line) ) { Error("could not read first line"); return false; @@ -143,16 +142,17 @@ bool Ascii::ReadHeader(bool useCached) headerline = line; } - else + + else line = headerline; - + // construct list of field names. istringstream splitstream(line); int pos=0; - while ( splitstream ) + while ( splitstream ) { string s; - if ( !getline(splitstream, s, separator[0])) + if ( ! getline(splitstream, s, separator[0])) break; ifields[s] = pos; @@ -161,15 +161,15 @@ bool Ascii::ReadHeader(bool useCached) //printf("Updating fields from description %s\n", line.c_str()); columnMap.clear(); - - for ( unsigned int i = 0; i < num_fields; i++ ) + + for ( unsigned int i = 0; i < num_fields; i++ ) { const Field* field = fields[i]; - - map::iterator fit = ifields.find(field->name); - if ( fit == ifields.end() ) + + map::iterator fit = ifields.find(field->name); + if ( fit == ifields.end() ) { - if ( field->optional ) + if ( field->optional ) { // we do not really need this field. mark it as not present and always send an undef back. FieldMapping f(field->name, field->type, field->subtype, -1); @@ -178,38 +178,43 @@ bool Ascii::ReadHeader(bool useCached) continue; } - Error(Fmt("Did not find requested field %s in input data file %s.", field->name.c_str(), fname.c_str())); + Error(Fmt("Did not find requested field %s in input data file %s.", + field->name.c_str(), fname.c_str())); return false; } FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]); - if ( field->secondary_name != "" ) + + if ( field->secondary_name != "" ) { - map::iterator fit2 = ifields.find(field->secondary_name); - if ( fit2 == ifields.end() ) + map::iterator fit2 = ifields.find(field->secondary_name); + if ( fit2 == ifields.end() ) { - Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); + Error(Fmt("Could not find requested port type field %s in input data file.", + field->secondary_name.c_str())); return false; } + f.secondary_position = ifields[field->secondary_name]; } + columnMap.push_back(f); } - + // well, that seems to have worked... return true; } -bool Ascii::GetLine(string& str) +bool Ascii::GetLine(string& str) { while ( getline(*file, str) ) { - if ( str[0] != '#' ) + if ( str[0] != '#' ) return true; - if ( str.compare(0,8, "#fields\t") == 0 ) + if ( str.compare(0,8, "#fields\t") == 0 ) { str = str.substr(8); return true; @@ -220,14 +225,13 @@ bool Ascii::GetLine(string& str) } -Value* Ascii::EntryToVal(string s, FieldMapping field) +Value* Ascii::EntryToVal(string s, FieldMapping field) { - if ( s.compare(unset_field) == 0 ) // field is not set... return new Value(field.type, false); Value* val = new Value(field.type, true); - + switch ( field.type ) { case TYPE_ENUM: case TYPE_STRING: @@ -235,13 +239,14 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) break; case TYPE_BOOL: - if ( s == "T" ) + if ( s == "T" ) val->val.int_val = 1; - else if ( s == "F" ) + else if ( s == "F" ) val->val.int_val = 0; - else + else { - Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str())); + Error(Fmt("Field: %s Invalid value for boolean: %s", + field.name.c_str(), s.c_str())); return false; } break; @@ -266,13 +271,15 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) val->val.port_val.proto = TRANSPORT_UNKNOWN; break; - case TYPE_SUBNET: + case TYPE_SUBNET: { size_t pos = s.find("/"); - if ( pos == s.npos ) { + if ( pos == s.npos ) + { Error(Fmt("Invalid value for subnet: %s", s.c_str())); return false; - } + } + int width = atoi(s.substr(pos+1).c_str()); string addr = s.substr(0, pos); @@ -281,7 +288,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) break; } - case TYPE_ADDR: + case TYPE_ADDR: val->val.addr_val = StringToAddr(s); break; @@ -295,42 +302,42 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) // how many entries do we have... unsigned int length = 1; for ( unsigned int i = 0; i < s.size(); i++ ) - if ( s[i] == ',') length++; + if ( s[i] == ',' ) length++; unsigned int pos = 0; - - if ( s.compare(empty_field) == 0 ) + + if ( s.compare(empty_field) == 0 ) length = 0; Value** lvals = new Value* [length]; - if ( field.type == TYPE_TABLE ) + if ( field.type == TYPE_TABLE ) { val->val.set_val.vals = lvals; val->val.set_val.size = length; } - else if ( field.type == TYPE_VECTOR ) + + else if ( field.type == TYPE_VECTOR ) { val->val.vector_val.vals = lvals; val->val.vector_val.size = length; - } - else - { - assert(false); } + else + assert(false); + if ( length == 0 ) break; //empty istringstream splitstream(s); - while ( splitstream ) + while ( splitstream ) { string element; - if ( !getline(splitstream, element, set_separator[0]) ) + if ( ! getline(splitstream, element, set_separator[0]) ) break; - if ( pos >= length ) + if ( pos >= length ) { Error(Fmt("Internal error while parsing set. pos %d >= length %d." " Element: %s", pos, length, element.c_str())); @@ -338,18 +345,18 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) } Value* newval = EntryToVal(element, field.subType()); - if ( newval == 0 ) + if ( newval == 0 ) { Error("Error while reading set"); return 0; } + lvals[pos] = newval; pos++; } - - if ( pos != length ) + if ( pos != length ) { Error("Internal error while parsing set: did not find all elements"); return 0; @@ -358,24 +365,24 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) break; } - default: Error(Fmt("unsupported field format %d for %s", field.type, field.name.c_str())); return 0; - } + } return val; } // read the entire file and send appropriate thingies back to InputMgr -bool Ascii::DoUpdate() +bool Ascii::DoUpdate() { switch ( mode ) { case REREAD: + { // check if the file has changed struct stat sb; - if ( stat(fname.c_str(), &sb) == -1 ) + if ( stat(fname.c_str(), &sb) == -1 ) { Error(Fmt("Could not get stat for %s", fname.c_str())); return false; @@ -388,54 +395,58 @@ bool Ascii::DoUpdate() // file changed. reread. // fallthrough + } + case MANUAL: case STREAM: - - // dirty, fix me. (well, apparently after trying seeking, etc + { + // dirty, fix me. (well, apparently after trying seeking, etc // - this is not that bad) - if ( file && file->is_open() ) + if ( file && file->is_open() ) { - if ( mode == STREAM ) + if ( mode == STREAM ) { file->clear(); // remove end of file evil bits - if ( !ReadHeader(true) ) + if ( !ReadHeader(true) ) return false; // header reading failed break; } file->close(); } + file = new ifstream(fname.c_str()); - if ( !file->is_open() ) + if ( !file->is_open() ) { Error(Fmt("cannot open %s", fname.c_str())); return false; } - - if ( ReadHeader(false) == false ) + if ( ReadHeader(false) == false ) { return false; } break; + } + default: assert(false); } string line; - while ( GetLine(line ) ) + while ( GetLine(line ) ) { // split on tabs istringstream splitstream(line); map stringfields; int pos = 0; - while ( splitstream ) + while ( splitstream ) { string s; - if ( !getline(splitstream, s, separator[0]) ) + if ( ! getline(splitstream, s, separator[0]) ) break; stringfields[pos] = s; @@ -444,7 +455,6 @@ bool Ascii::DoUpdate() pos--; // for easy comparisons of max element. - Value** fields = new Value*[num_fields]; int fpos = 0; @@ -453,33 +463,34 @@ bool Ascii::DoUpdate() fit++ ) { - if ( ! fit->present ) + if ( ! fit->present ) { // add non-present field fields[fpos] = new Value((*fit).type, false); fpos++; continue; } - + assert(fit->position >= 0 ); - if ( (*fit).position > pos || (*fit).secondary_position > pos ) + if ( (*fit).position > pos || (*fit).secondary_position > pos ) { - Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); + Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", + line.c_str(), pos, (*fit).position, (*fit).secondary_position)); return false; } Value* val = EntryToVal(stringfields[(*fit).position], *fit); - if ( val == 0 ) + if ( val == 0 ) { Error("Could not convert String value to Val"); return false; } - - if ( (*fit).secondary_position != -1 ) + + if ( (*fit).secondary_position != -1 ) { // we have a port definition :) - assert(val->type == TYPE_PORT ); + assert(val->type == TYPE_PORT ); // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); @@ -493,31 +504,33 @@ bool Ascii::DoUpdate() //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); assert ( (unsigned int) fpos == num_fields ); - if ( mode == STREAM ) + if ( mode == STREAM ) Put(fields); else SendEntry(fields); } - if ( mode != STREAM ) + if ( mode != STREAM ) EndCurrentSend(); - + return true; } bool Ascii::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - + switch ( mode ) { case MANUAL: // yay, we do nothing :) break; + case REREAD: case STREAM: - Update(); // call update and not DoUpdate, because update + Update(); // call update and not DoUpdate, because update // checks disabled. break; + default: assert(false); } diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index e5f3070724..e5540c5467 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -14,73 +14,57 @@ namespace input { namespace reader { struct FieldMapping { string name; TypeTag type; - // internal type for sets and vectors - TypeTag subtype; + TypeTag subtype; // internal type for sets and vectors int position; - // for ports: pos of the second field - int secondary_position; + int secondary_position; // for ports: pos of the second field bool present; - FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); - FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); FieldMapping(const FieldMapping& arg); FieldMapping() { position = -1; secondary_position = -1; } FieldMapping subType(); - //bool IsEmpty() { return position == -1; } }; - class Ascii : public ReaderBackend { public: - Ascii(ReaderFrontend* frontend); - ~Ascii(); - - static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Ascii(frontend); } - + Ascii(ReaderFrontend* frontend); + ~Ascii(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Ascii(frontend); } + protected: - virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); - virtual void DoClose(); - virtual bool DoUpdate(); private: - virtual bool DoHeartbeat(double network_time, double current_time); - unsigned int num_fields; - - const threading::Field* const * fields; // raw mapping - - // map columns in the file to columns to send back to the manager - vector columnMap; - bool ReadHeader(bool useCached); + bool GetLine(string& str); threading::Value* EntryToVal(string s, FieldMapping type); - bool GetLine(string& str); - + unsigned int num_fields; + const threading::Field* const *fields; // raw mapping + ifstream* file; string fname; + int mode; + time_t mtime; - // Options set from the script-level. - string separator; + // map columns in the file to columns to send back to the manager + vector columnMap; - string set_separator; - - string empty_field; - - string unset_field; - // keep a copy of the headerline to determine field locations when stream descriptions change string headerline; - int mode; - - time_t mtime; - + // Options set from the script-level. + string separator; + string set_separator; + string empty_field; + string unset_field; }; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index 29f0070fec..c6cc1649eb 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -23,15 +23,14 @@ using threading::Field; Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) { - multiplication_factor = double(BifConst::InputBenchmark::factor); - autospread = double(BifConst::InputBenchmark::autospread); + multiplication_factor = double(BifConst::InputBenchmark::factor); + autospread = double(BifConst::InputBenchmark::autospread); spread = int(BifConst::InputBenchmark::spread); add = int(BifConst::InputBenchmark::addfactor); autospread_time = 0; stopspreadat = int(BifConst::InputBenchmark::stopspreadat); timedspread = double(BifConst::InputBenchmark::timedspread); heart_beat_interval = double(BifConst::Threading::heart_beat_interval); - } Benchmark::~Benchmark() @@ -46,15 +45,15 @@ void Benchmark::DoClose() bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) { mode = arg_mode; - + num_fields = arg_num_fields; fields = arg_fields; num_lines = atoi(path.c_str()); - + if ( autospread != 0.0 ) autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) + if ( (mode != MANUAL) && (mode != REREAD) && (mode != STREAM) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; @@ -66,7 +65,7 @@ bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Fiel return true; } -string Benchmark::RandomString(const int len) +string Benchmark::RandomString(const int len) { string s(len, ' '); @@ -75,13 +74,13 @@ string Benchmark::RandomString(const int len) "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"; - for (int i = 0; i < len; ++i) - s[i] = values[rand() / (RAND_MAX / sizeof(values))]; + for (int i = 0; i < len; ++i) + s[i] = values[rand() / (RAND_MAX / sizeof(values))]; return s; } -double Benchmark::CurrTime() +double Benchmark::CurrTime() { struct timeval tv; assert ( gettimeofday(&tv, 0) >= 0 ); @@ -91,56 +90,57 @@ double Benchmark::CurrTime() // read the entire file and send appropriate thingies back to InputMgr -bool Benchmark::DoUpdate() +bool Benchmark::DoUpdate() { - int linestosend = num_lines * heart_beat_interval; - for ( int i = 0; i < linestosend; i++ ) + int linestosend = num_lines * heart_beat_interval; + for ( int i = 0; i < linestosend; i++ ) { Value** field = new Value*[num_fields]; - for (unsigned int j = 0; j < num_fields; j++ ) + for (unsigned int j = 0; j < num_fields; j++ ) field[j] = EntryToVal(fields[j]->type, fields[j]->subtype); - if ( mode == STREAM ) + if ( mode == STREAM ) // do not do tracking, spread out elements over the second that we have... Put(field); - else + else SendEntry(field); - - if ( stopspreadat == 0 || num_lines < stopspreadat ) + + if ( stopspreadat == 0 || num_lines < stopspreadat ) { - if ( spread != 0 ) + if ( spread != 0 ) usleep(spread); - if ( autospread_time != 0 ) + if ( autospread_time != 0 ) usleep( autospread_time ); } - if ( timedspread != 0.0 ) + if ( timedspread != 0.0 ) { double diff; - do + do diff = CurrTime() - heartbeatstarttime; - while ( diff/heart_beat_interval < i/(linestosend + while ( diff/heart_beat_interval < i/(linestosend + (linestosend * timedspread) ) ); } } - if ( mode != STREAM ) + if ( mode != STREAM ) EndCurrentSend(); return true; } -threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) +threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { Value* val = new Value(type, true); // basically construct something random from the fields that we want. - + switch ( type ) { case TYPE_ENUM: assert(false); // no enums, please. + case TYPE_STRING: val->val.string_val = new string(RandomString(10)); break; @@ -172,14 +172,14 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) val->val.port_val.proto = TRANSPORT_UNKNOWN; break; - case TYPE_SUBNET: + case TYPE_SUBNET: { val->val.subnet_val.prefix = StringToAddr("192.168.17.1"); val->val.subnet_val.length = 16; } break; - case TYPE_ADDR: + case TYPE_ADDR: val->val.addr_val = StringToAddr("192.168.17.1"); break; @@ -195,26 +195,26 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) Value** lvals = new Value* [length]; - if ( type == TYPE_TABLE ) + if ( type == TYPE_TABLE ) { val->val.set_val.vals = lvals; val->val.set_val.size = length; - } - else if ( type == TYPE_VECTOR ) + } + else if ( type == TYPE_VECTOR ) { val->val.vector_val.vals = lvals; val->val.vector_val.size = length; - } - else + } + else assert(false); if ( length == 0 ) break; //empty - for ( unsigned int pos = 0; pos < length; pos++ ) + for ( unsigned int pos = 0; pos < length; pos++ ) { Value* newval = EntryToVal(subtype, TYPE_ENUM); - if ( newval == 0 ) + if ( newval == 0 ) { Error("Error while reading set"); return 0; @@ -229,7 +229,7 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) default: Error(Fmt("unsupported field format %d", type)); return 0; - } + } return val; @@ -247,9 +247,10 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) case MANUAL: // yay, we do nothing :) break; + case REREAD: case STREAM: - if ( multiplication_factor != 1 || add != 0 ) + if ( multiplication_factor != 1 || add != 0 ) { // we have to document at what time we changed the factor to what value. Value** v = new Value*[2]; @@ -261,10 +262,10 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) SendEvent("lines_changed", 2, v); } - if ( autospread != 0.0 ) + if ( autospread != 0.0 ) // because executing this in every loop is apparently too expensive. autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); - + Update(); // call update and not DoUpdate, because update actually checks disabled. SendEvent("HeartbeatDone", 0, 0); @@ -275,4 +276,3 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) return true; } - diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index b791dabe21..ec14dc6567 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -3,41 +3,37 @@ #ifndef INPUT_READERS_BENCHMARK_H #define INPUT_READERS_BENCHMARK_H - #include "../ReaderBackend.h" namespace input { namespace reader { +/** + * A benchmark reader to measure performance of the input framework. + */ class Benchmark : public ReaderBackend { public: - Benchmark(ReaderFrontend* frontend); - ~Benchmark(); - - static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Benchmark(frontend); } - + Benchmark(ReaderFrontend* frontend); + ~Benchmark(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Benchmark(frontend); } + protected: - virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); - virtual void DoClose(); - virtual bool DoUpdate(); private: - virtual bool DoHeartbeat(double network_time, double current_time); - unsigned int num_fields; - double CurrTime(); - - const threading::Field* const * fields; // raw mapping - + string RandomString(const int len); threading::Value* EntryToVal(TypeTag Type, TypeTag subtype); + unsigned int num_fields; + const threading::Field* const * fields; // raw mapping + int mode; int num_lines; - double multiplication_factor; int spread; double autospread; @@ -47,9 +43,6 @@ private: double heartbeatstarttime; double timedspread; double heart_beat_interval; - - string RandomString(const int len); - }; diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index ce0b4f8a5f..6538da070b 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -28,8 +28,10 @@ Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend) file = 0; in = 0; - separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(), BifConst::InputRaw::record_separator->Len()); - if ( separator.size() != 1 ) + separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(), + BifConst::InputRaw::record_separator->Len()); + + if ( separator.size() != 1 ) Error("separator length has to be 1. Separator will be truncated."); } @@ -40,57 +42,56 @@ Raw::~Raw() void Raw::DoClose() { - if ( file != 0 ) + if ( file != 0 ) { Close(); } } -bool Raw::Open() +bool Raw::Open() { - if ( execute ) + if ( execute ) { file = popen(fname.c_str(), "r"); - if ( file == NULL ) + if ( file == NULL ) { Error(Fmt("Could not execute command %s", fname.c_str())); return false; } } - else + else { file = fopen(fname.c_str(), "r"); - if ( file == NULL ) + if ( file == NULL ) { Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } } - + + // This is defined in input/fdstream.h in = new boost::fdistream(fileno(file)); - if ( execute && mode == STREAM ) - { + if ( execute && mode == STREAM ) fcntl(fileno(file), F_SETFL, O_NONBLOCK); - } return true; } bool Raw::Close() { - if ( file == NULL ) + if ( file == NULL ) { InternalError(Fmt("Trying to close closed file for stream %s", fname.c_str())); return false; } - if ( execute ) + if ( execute ) { delete(in); pclose(file); - } - else + } + else { delete(in); fclose(file); @@ -114,13 +115,13 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con num_fields = arg_num_fields; fields = arg_fields; - if ( path.length() == 0 ) + if ( path.length() == 0 ) { Error("No source path provided"); return false; } - - if ( arg_num_fields != 1 ) + + if ( arg_num_fields != 1 ) { Error("Filter for raw reader contains more than one field. " "Filters for the raw reader may only contain exactly one string field. " @@ -128,7 +129,7 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con return false; } - if ( fields[0]->type != TYPE_STRING ) + if ( fields[0]->type != TYPE_STRING ) { Error("Filter for raw reader contains a field that is not of type string."); return false; @@ -136,30 +137,32 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con // do Initialization char last = path[path.length()-1]; - if ( last == '|' ) + if ( last == '|' ) { execute = true; fname = path.substr(0, fname.length() - 1); - if ( ( mode != MANUAL ) && ( mode != STREAM ) ) { - Error(Fmt("Unsupported read mode %d for source %s in execution mode", mode, fname.c_str())); + if ( (mode != MANUAL) && (mode != STREAM) ) { + Error(Fmt("Unsupported read mode %d for source %s in execution mode", + mode, fname.c_str())); return false; - } - + } + result = Open(); } else { execute = false; - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) + if ( (mode != MANUAL) && (mode != REREAD) && (mode != STREAM) ) { - Error(Fmt("Unsupported read mode %d for source %s", mode, fname.c_str())); + Error(Fmt("Unsupported read mode %d for source %s", + mode, fname.c_str())); return false; } - result = Open(); + result = Open(); } - if ( result == false ) + if ( result == false ) return result; #ifdef DEBUG @@ -176,80 +179,78 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con } -bool Raw::GetLine(string& str) +bool Raw::GetLine(string& str) { - if ( in->peek() == std::iostream::traits_type::eof() ) + if ( in->peek() == std::iostream::traits_type::eof() ) return false; - if ( in->eofbit == true || in->failbit == true ) + if ( in->eofbit == true || in->failbit == true ) return false; - while ( getline(*in, str, separator[0]) ) - return true; - - return false; + return getline(*in, str, separator[0]); } - // read the entire file and send appropriate thingies back to InputMgr -bool Raw::DoUpdate() +bool Raw::DoUpdate() { - if ( firstrun ) + if ( firstrun ) firstrun = false; + else { switch ( mode ) { - case REREAD: + case REREAD: + { + // check if the file has changed + struct stat sb; + if ( stat(fname.c_str(), &sb) == -1 ) { - // check if the file has changed - struct stat sb; - if ( stat(fname.c_str(), &sb) == -1 ) - { - Error(Fmt("Could not get stat for %s", fname.c_str())); - return false; - } - - if ( sb.st_mtime <= mtime ) - // no change - return true; - - mtime = sb.st_mtime; - // file changed. reread. - - // fallthrough + Error(Fmt("Could not get stat for %s", fname.c_str())); + return false; } - case MANUAL: - case STREAM: - if ( mode == STREAM && file != NULL && in != NULL ) - { - //fpurge(file); - in->clear(); // remove end of file evil bits - break; - } - Close(); - if ( !Open() ) - return false; + if ( sb.st_mtime <= mtime ) + // no change + return true; + mtime = sb.st_mtime; + // file changed. reread. + // + // fallthrough + } + + case MANUAL: + case STREAM: + if ( mode == STREAM && file != NULL && in != NULL ) + { + //fpurge(file); + in->clear(); // remove end of file evil bits break; - default: - assert(false); + } + Close(); + if ( ! Open() ) + return false; + + break; + + default: + assert(false); } } string line; - while ( GetLine(line) ) + while ( GetLine(line) ) { assert (num_fields == 1); - + Value** fields = new Value*[1]; // filter has exactly one text field. convert to it. Value* val = new Value(TYPE_STRING, true); val->val.string_val = new string(line); fields[0] = val; - + Put(fields); } @@ -260,7 +261,6 @@ bool Raw::DoUpdate() return true; } - bool Raw::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); @@ -269,10 +269,11 @@ bool Raw::DoHeartbeat(double network_time, double current_time) case MANUAL: // yay, we do nothing :) break; + case REREAD: case STREAM: - Update(); // call update and not DoUpdate, because update - // checks disabled. + Update(); // call update and not DoUpdate, because update + // checks disabled. break; default: assert(false); diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index 9f575bb89c..3fa09309b0 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -10,51 +10,44 @@ namespace input { namespace reader { +/** + * A reader that returns a file (or the output of a command) as a single + * blob. + */ class Raw : public ReaderBackend { public: - Raw(ReaderFrontend* frontend); - ~Raw(); - - static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); } - + Raw(ReaderFrontend* frontend); + ~Raw(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); } + protected: - virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); - virtual void DoClose(); - virtual bool DoUpdate(); private: - virtual bool DoHeartbeat(double network_time, double current_time); + bool Open(); bool Close(); - bool GetLine(string& str); - + + unsigned int num_fields; + const threading::Field* const * fields; // raw mapping + istream* in; - FILE* file; - string fname; - - // Options set from the script-level. - string separator; - int mode; bool execute; bool firstrun; - time_t mtime; - - unsigned int num_fields; - - const threading::Field* const * fields; // raw mapping + // Options set from the script-level. + string separator; }; - } } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out index 8611b35dd3..a38f3fce84 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out @@ -6,4 +6,4 @@ print outfile, s; close(outfile); }] Input::EVENT_NEW - 8 ../input.log +8 ../input.log diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro index 6d07a9bf29..6df28d08ea 100644 --- a/testing/btest/scripts/base/frameworks/input/executeraw.bro +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -1,6 +1,7 @@ # # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: cat out.tmp | sed 's/^ *//g' >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -31,7 +32,7 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string) { event bro_init() { - outfile = open ("../out"); + outfile = open ("../out.tmp"); Input::add_event([$source="wc -l ../input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]); Input::remove("input"); } From f4864c69af58c153700924774092b85c7405eaa9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 29 May 2012 09:21:16 -0700 Subject: [PATCH 141/149] fix another memory lead (when updating tables). Adjust twotables testcase - now it is faster. Shorten the output -- because of threading, the results did not always come out in the same order (it depends on which thread manages to sneak in the results into the queue earlier). --- src/input/Manager.cc | 3 +- .../out | 231 ++---------------- .../base/frameworks/input/twotables.bro | 21 +- 3 files changed, 41 insertions(+), 214 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 6cae5e2f34..56d7d82ce6 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1123,6 +1123,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) val = stream->tab->Lookup(idx); assert(val != 0); predidx = ListValToRecordVal(idx, stream->itype, &startpos); + Unref(idx); ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); } @@ -1594,7 +1595,7 @@ RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, (*position)++; } - rec->Assign(i, fieldVal); + rec->Assign(i, fieldVal->Ref()); } return rec; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out index 41d9438da0..e9e03add3a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out @@ -29,68 +29,6 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ -[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}] -Type -Input::EVENT_NEW -Left -[i=-42] -Right -[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] ==========SERVERS============ { [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ @@ -119,68 +57,6 @@ BB }, vc=[10, 20, 30], ve=[]] } ============EVENT============ -Description -[source=../input2.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh2, destination={ -[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE 2============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}] -Type -Input::EVENT_NEW -Left -[i=-43] -Right -[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] ==========SERVERS============ { [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ @@ -239,87 +115,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ -[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}] -Type -Input::EVENT_NEW -Left -[i=-44] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] ============EVENT============ -Description -Input::EVENT_REMOVED -Type -[i=-42] -Left -[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -Right ==========SERVERS============ { [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ @@ -347,3 +143,30 @@ BB }, vc=[10, 20, 30], ve=[]] } +done +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro index 6f18e0e939..1413275e63 100644 --- a/testing/btest/scripts/base/frameworks/input/twotables.bro +++ b/testing/btest/scripts/base/frameworks/input/twotables.bro @@ -64,14 +64,14 @@ global try: count; event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { print outfile, "============EVENT============"; - print outfile, "Description"; - print outfile, description; - print outfile, "Type"; - print outfile, tpe; - print outfile, "Left"; - print outfile, left; - print outfile, "Right"; - print outfile, right; +# print outfile, "Description"; +# print outfile, description; +# print outfile, "Type"; +# print outfile, tpe; +# print outfile, "Left"; +# print outfile, left; +# print outfile, "Right"; +# print outfile, right; } event bro_init() @@ -105,9 +105,12 @@ event Input::update_finished(name: string, source: string) { print outfile, servers; try = try + 1; - if ( try == 5 ) { + if ( try == 3 ) { print outfile, "done"; + print outfile, servers; close(outfile); Input::remove("input"); + Input::remove("input2"); + terminate(); } } From 1416d5404dca9d0e54c5c270176bb9a4ee56db3f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 29 May 2012 10:35:56 -0700 Subject: [PATCH 142/149] and another small memory leak when using streaming reads. --- src/input/Manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 56d7d82ce6..9bf885072b 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -836,7 +836,6 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu idxval = ValueToVal(vals[0], type->FieldType(0)); position = 1; } - else { ListVal *l = new ListVal(TYPE_ANY); @@ -1283,7 +1282,6 @@ int Manager::PutTable(Stream* i, const Value* const *vals) else if ( stream->num_val_fields == 1 && stream->want_record == 0 ) valval = ValueToVal(vals[position], stream->rtype->FieldType(0)); - else valval = ValueToRecordVal(vals, stream->rtype, &position); @@ -1377,6 +1375,8 @@ int Manager::PutTable(Stream* i, const Value* const *vals) else // no predicates or other stuff stream->tab->Assign(idxval, valval); + Unref(idxval); // not consumed by assign + return stream->num_idx_fields + stream->num_val_fields; } From 0c5afc59f79099fa1874cbe96e3bb74b7df693ad Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 29 May 2012 14:51:45 -0500 Subject: [PATCH 143/149] Improve script debugger backtrace and print commands. Stack trace context descriptions are no longer limited to 1024 chars and better error messages are relayed when the arguments to print commands fail to parse (e.g. an "unknown identifier" was given). --- src/Debug.cc | 18 +++++++++++++++--- src/DebugCmds.cc | 6 +++--- src/parse.y | 7 +++++-- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/Debug.cc b/src/Debug.cc index ea9c52f77e..a1e2000bea 100644 --- a/src/Debug.cc +++ b/src/Debug.cc @@ -721,7 +721,6 @@ static char* get_prompt(bool reset_counter = false) string get_context_description(const Stmt* stmt, const Frame* frame) { - char buf[1024]; ODesc d; const BroFunc* func = frame->GetFunction(); @@ -739,10 +738,14 @@ string get_context_description(const Stmt* stmt, const Frame* frame) loc.last_line = 0; } - safe_snprintf(buf, sizeof(buf), "In %s at %s:%d", + size_t buf_size = strlen(d.Description()) + strlen(loc.filename) + 1024; + char* buf = new char[buf_size]; + safe_snprintf(buf, buf_size, "In %s at %s:%d", d.Description(), loc.filename, loc.last_line); - return string(buf); + string retval(buf); + delete [] buf; + return retval; } int dbg_handle_debug_input() @@ -924,6 +927,8 @@ bool post_execute_stmt(Stmt* stmt, Frame* f, Val* result, stmt_flow_type* flow) // Evaluates the given expression in the context of the currently selected // frame. Returns the resulting value, or nil if none (or there was an error). Expr* g_curr_debug_expr = 0; +const char* g_curr_debug_error = 0; +bool in_debug = false; // ### fix this hardwired access to external variables etc. struct yy_buffer_state; @@ -969,6 +974,10 @@ Val* dbg_eval_expr(const char* expr) Val* result = 0; if ( yyparse() ) { + if ( g_curr_debug_error ) + debug_msg("Parsing expression '%s' failed: %s\n", expr, g_curr_debug_error); + else + debug_msg("Parsing expression '%s' failed\n", expr); if ( g_curr_debug_expr ) { delete g_curr_debug_expr; @@ -983,6 +992,9 @@ Val* dbg_eval_expr(const char* expr) delete g_curr_debug_expr; g_curr_debug_expr = 0; + delete [] g_curr_debug_error; + g_curr_debug_error = 0; + in_debug = false; return result; } diff --git a/src/DebugCmds.cc b/src/DebugCmds.cc index 1d3b9dd220..bfb4d6ecc8 100644 --- a/src/DebugCmds.cc +++ b/src/DebugCmds.cc @@ -553,7 +553,8 @@ int dbg_cmd_print(DebugCmd cmd, const vector& args) for ( int i = 0; i < int(args.size()); ++i ) { expr += args[i]; - expr += " "; + if ( i < int(args.size()) - 1 ) + expr += " "; } Val* val = dbg_eval_expr(expr.c_str()); @@ -566,8 +567,7 @@ int dbg_cmd_print(DebugCmd cmd, const vector& args) } else { - // ### Print something? - // debug_msg("\n"); + debug_msg("\n"); } return 1; diff --git a/src/parse.y b/src/parse.y index f78003f08b..6875f07668 100644 --- a/src/parse.y +++ b/src/parse.y @@ -112,13 +112,14 @@ bool is_export = false; // true if in an export {} block * (obviously not reentrant). */ extern Expr* g_curr_debug_expr; +extern bool in_debug; +extern const char* g_curr_debug_error; #define YYLTYPE yyltype Expr* bro_this = 0; int in_init = 0; int in_record = 0; -bool in_debug = false; bool resolving_global_ID = false; bool defining_global_ID = false; @@ -249,7 +250,6 @@ bro: TOK_DEBUG { in_debug = true; } expr { g_curr_debug_expr = $3; - in_debug = false; } ; @@ -1685,6 +1685,9 @@ int yyerror(const char msg[]) strcat(msgbuf, "\nDocumentation mode is enabled: " "remember to check syntax of ## style comments\n"); + if ( in_debug ) + g_curr_debug_error = copy_string(msg); + reporter->Error("%s", msgbuf); return 0; From 0aecca979e830d0ee8f6524c4dee3fe83cfc3c4c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 29 May 2012 17:29:11 -0500 Subject: [PATCH 144/149] Remove unnecessary assert in ICMP analyzer (addresses #822). The ICMP/ICMPv6 analyzers function correctly when full packets have not been captured, but everything up to and including the ICMP header is there (e.g. the functions that inspect ICMP error message context correctly check the caplen to see if more info can be extracted). The "Should have been caught earlier already." comment may have referred to NetSessions::CheckHeaderTrunc, which works as intended to catch cases where the ICMP header is not there in full, but then the assert was still not correctly formulated for that... Also changed the ICMP checksum calculation to not occur when the full packet has not been captured, which seems consistent with what the UDP analysis does. --- src/ICMP.cc | 4 +--- testing/btest/Baseline/core.truncation/output | 8 ++++++++ testing/btest/Traces/trunc/icmp-header-trunc.pcap | Bin 0 -> 136 bytes .../btest/Traces/trunc/icmp-payload-trunc.pcap | Bin 0 -> 408 bytes testing/btest/core/truncation.test | 13 +++++++++++++ 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 testing/btest/Traces/trunc/icmp-header-trunc.pcap create mode 100644 testing/btest/Traces/trunc/icmp-payload-trunc.pcap diff --git a/src/ICMP.cc b/src/ICMP.cc index 05a6b67dff..b06c6440e1 100644 --- a/src/ICMP.cc +++ b/src/ICMP.cc @@ -49,9 +49,7 @@ void ICMP_Analyzer::DeliverPacket(int len, const u_char* data, const struct icmp* icmpp = (const struct icmp*) data; - assert(caplen >= len); // Should have been caught earlier already. - - if ( ! ignore_checksums ) + if ( ! ignore_checksums && caplen >= len ) { int chksum = 0; diff --git a/testing/btest/Baseline/core.truncation/output b/testing/btest/Baseline/core.truncation/output index f3d64b8b28..95d9073648 100644 --- a/testing/btest/Baseline/core.truncation/output +++ b/testing/btest/Baseline/core.truncation/output @@ -22,3 +22,11 @@ #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer #types time string addr port addr port string string bool string 1334094648.590126 - - - - - truncated_IP - F bro +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer +#types time string addr port addr port string string bool string +1338328954.078361 - - - - - internally_truncated_header - F bro diff --git a/testing/btest/Traces/trunc/icmp-header-trunc.pcap b/testing/btest/Traces/trunc/icmp-header-trunc.pcap new file mode 100644 index 0000000000000000000000000000000000000000..5765cf288605f05f9344d27334bb26258b467ebf GIT binary patch literal 136 zcmca|c+)~A1{MYw`2U}Qff2~5azE-XX~f8&0c0nEBn57FTzU^;Ffed1xH2$=L>^&a zaA4HF#Rb%GfI*v!gW=}w%=;jH^IMVhL9~E%L-g&M>%j1X@zQ^g9*|xJhKE3X04E?J Al>h($ literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/trunc/icmp-payload-trunc.pcap b/testing/btest/Traces/trunc/icmp-payload-trunc.pcap new file mode 100644 index 0000000000000000000000000000000000000000..13607dd50cd5344a6d6706b56d8070474a5e3cee GIT binary patch literal 408 zcmca|c+)~A1{MYw`2U}Qff2}AYkt%}p^J+l0LV@PNebNVxbz;xU|`^2aAjZ!d3%6? z!GST{kqfB7tM;KL2ZKuQ&CLu9{zuK%GH|uqb8vET^YHTV3kV7ci-?MeOGrvd%gD;f zD<~={tEhs^-!v1+e6Yz7TOj7o*vi0Q!MMc-WH!hS1_m7x%-;YspE(7|d<^&J&jq<( zj1gq^0S0X@4hGxzk0I{gz`&J!3%~n8=C^M{G9SbJITsulelRMW1(^+Tzbgsm0{}2B BQ#$|v literal 0 HcmV?d00001 diff --git a/testing/btest/core/truncation.test b/testing/btest/core/truncation.test index ee8bdd5bf9..3406879183 100644 --- a/testing/btest/core/truncation.test +++ b/testing/btest/core/truncation.test @@ -6,4 +6,17 @@ # @TEST-EXEC: cat weird.log >> output # @TEST-EXEC: bro -r $TRACES/trunc/ip6-ext-trunc.pcap # @TEST-EXEC: cat weird.log >> output + +# If an ICMP packet's payload is truncated due to too small snaplen, +# the checksum calculation is bypassed (and Bro doesn't crash, of course). + +# @TEST-EXEC: rm -f weird.log +# @TEST-EXEC: bro -r $TRACES/trunc/icmp-payload-trunc.pcap +# @TEST-EXEC: test ! -e weird.log + +# If an ICMP packet has the ICMP header truncated due to too small snaplen, +# an internally_truncated_header weird gets generated. + +# @TEST-EXEC: bro -r $TRACES/trunc/icmp-header-trunc.pcap +# @TEST-EXEC: cat weird.log >> output # @TEST-EXEC: btest-diff output From 65b50ab2da9537428355059a5c640298dd8acefa Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 30 May 2012 10:16:05 -0700 Subject: [PATCH 145/149] another small memory leak in ascii reader: on re-read istream instance was re-created but not freed before. --- src/input/readers/Ascii.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 157ea90916..275b3a9e67 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -413,6 +413,8 @@ bool Ascii::DoUpdate() break; } file->close(); + delete(file); + file = 0; } file = new ifstream(fname.c_str()); From fc907c0090fbceed5dd64385c38d09bb88502acf Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 30 May 2012 16:38:08 -0700 Subject: [PATCH 146/149] A set of input framework refactoring, cleanup, and polishing. --- scripts/base/frameworks/input/main.bro | 3 - src/input/Manager.cc | 22 +++++- src/input/ReaderBackend.cc | 7 +- src/input/ReaderBackend.h | 92 +++++++++++++++++++------- src/input/ReaderFrontend.cc | 6 +- src/input/ReaderFrontend.h | 4 +- src/input/readers/Ascii.cc | 61 +++++++---------- src/input/readers/Ascii.h | 14 ++-- src/input/readers/Benchmark.cc | 37 +++-------- src/input/readers/Benchmark.h | 9 +-- src/input/readers/Raw.cc | 47 +++++-------- src/input/readers/Raw.h | 15 ++--- src/threading/Manager.cc | 11 +-- src/threading/Manager.h | 2 - 14 files changed, 162 insertions(+), 168 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 7a372dc120..f5df72473f 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -117,9 +117,6 @@ export { module Input; -#global streams: table[string] of Filter; -# ^ change to set containing the names - function add_table(description: Input::TableDescription) : bool { return __create_table_stream(description); diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 9bf885072b..bc79a2390b 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -74,7 +74,7 @@ public: string source; bool removed; - int mode; + ReaderMode mode; StreamType stream_type; // to distinguish between event and table streams @@ -299,7 +299,25 @@ bool Manager::CreateStream(Stream* info, RecordVal* description) Unref(sourceval); EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); - info->mode = mode->InternalInt(); + + switch ( mode->InternalInt() ) + { + case 0: + info->mode = MODE_MANUAL; + break; + + case 1: + info->mode = MODE_REREAD; + break; + + case 2: + info->mode = MODE_STREAM; + break; + + default: + reporter->InternalError("unknown reader mode"); + } + Unref(mode); info->reader = reader_obj; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 328e0bc535..43cbf8dfc1 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -176,15 +176,16 @@ void ReaderBackend::SendEntry(Value* *vals) SendOut(new SendEntryMessage(frontend, vals)); } -bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, +bool ReaderBackend::Init(string arg_source, ReaderMode arg_mode, const int arg_num_fields, const threading::Field* const* arg_fields) { source = arg_source; - SetName("InputReader/"+source); - + mode = arg_mode; num_fields = arg_num_fields; fields = arg_fields; + SetName("InputReader/"+source); + // disable if DoInit returns error. int success = DoInit(arg_source, mode, arg_num_fields, arg_fields); diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index ae8437b08c..8b5e7d674b 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -4,11 +4,32 @@ #define INPUT_READERBACKEND_H #include "BroString.h" -#include "../threading/SerialTypes.h" + +#include "threading/SerialTypes.h" #include "threading/MsgThread.h" namespace input { +/** + * The modes a reader can be in. + */ +enum ReaderMode { + /** + * TODO Bernhard. + */ + MODE_MANUAL, + + /** + * TODO Bernhard. + */ + MODE_REREAD, + + /** + * TODO Bernhard. + */ + MODE_STREAM +}; + class ReaderFrontend; /** @@ -40,24 +61,20 @@ public: /** * One-time initialization of the reader to define the input source. * - * @param arg_source A string left to the interpretation of the + * @param source A string left to the interpretation of the * reader implementation; it corresponds to the value configured on * the script-level for the input stream. * - * @param fields An array of size \a num_fields with the input - * fields. The method takes ownership of the array. + * @param mode The opening mode for the input source. * - * @param mode The opening mode for the input source as one of the - * Input::Mode script constants. - * - * @param arg_num_fields Number of fields contained in \a fields. + * @param num_fields Number of fields contained in \a fields. * * @param fields The types and names of the fields to be retrieved * from the input source. * * @return False if an error occured. */ - bool Init(string arg_source, int mode, int arg_num_fields, const threading::Field* const* fields); + bool Init(string source, ReaderMode mode, int num_fields, const threading::Field* const* fields); /** * Finishes reading from this input stream in a regular fashion. Must @@ -98,8 +115,15 @@ protected: * prevents the reader from further operation; it will then be * disabled and eventually deleted. When returning false, an * implementation should also call Error() to indicate what happened. + * + * Arguments are the same as Init(). + * + * Note that derived classes don't need to store the values passed in + * here if other methods need them to; the \a ReaderBackend class + * provides accessor methods to get them later, and they are passed + * in here only for convinience. */ - virtual bool DoInit(string arg_sources, int mode, int arg_num_fields, const threading::Field* const* fields) = 0; + virtual bool DoInit(string path, ReaderMode mode, int arg_num_fields, const threading::Field* const* fields) = 0; /** * Reader-specific method implementing input finalization at @@ -129,10 +153,25 @@ protected: virtual bool DoUpdate() = 0; /** - * Returns the input source as passed into the constructor. + * Returns the input source as passed into Init()/. */ const string Source() const { return source; } + /** + * Returns the reader mode as passed into Init(). + */ + const ReaderMode Mode() const { return mode; } + + /** + * Returns the number of log fields as passed into Init(). + */ + unsigned int NumFields() const { return num_fields; } + + /** + * Returns the log fields as passed into Init(). + */ + const threading::Field* const * Fields() const { return fields; } + /** * Method allowing a reader to send a specified Bro event. Vals must * match the values expected by the bro event. @@ -145,8 +184,8 @@ protected: */ void SendEvent(const string& name, const int num_vals, threading::Value* *vals); - // Content-sending-functions (simple mode). Including table-specific - // stuff that simply is not used if we have no table. + // Content-sending-functions (simple mode). Include table-specific + // functionality that simply is not used if we have no table. /** * Method allowing a reader to send a list of values read from a @@ -155,9 +194,10 @@ protected: * If the stream is a table stream, the values are inserted into the * table; if it is an event stream, the event is raised. * - * @param val list of threading::Values expected by the stream + * @param val Array of threading::Values expected by the stream. The + * array must have exactly NumEntries() elements. */ - void Put(threading::Value* *val); + void Put(threading::Value** val); /** * Method allowing a reader to delete a specific value from a Bro @@ -166,9 +206,10 @@ protected: * If the receiving stream is an event stream, only a removed event * is raised. * - * @param val list of threading::Values expected by the stream + * @param val Array of threading::Values expected by the stream. The + * array must have exactly NumEntries() elements. */ - void Delete(threading::Value* *val); + void Delete(threading::Value** val); /** * Method allowing a reader to clear a Bro table. @@ -187,9 +228,10 @@ protected: * If the stream is a table stream, the values are inserted into the * table; if it is an event stream, the event is raised. * - * @param val list of threading::Values expected by the stream + * @param val Array of threading::Values expected by the stream. The + * array must have exactly NumEntries() elements. */ - void SendEntry(threading::Value* *vals); + void SendEntry(threading::Value** vals); /** * Method telling the manager, that the current list of entries sent @@ -210,14 +252,16 @@ protected: virtual bool DoHeartbeat(double network_time, double current_time); /** - * Utility function for Readers - convert a string into a TransportProto + * Convert a string into a TransportProto. This is just a utility + * function for Readers. * * @param proto the transport protocol */ TransportProto StringToProto(const string &proto); /** - * Utility function for Readers - convert a string into a Value::addr_t + * Convert a string into a Value::addr_t. This is just a utility + * function for Readers. * * @param addr containing an ipv4 or ipv6 address */ @@ -229,11 +273,11 @@ private: ReaderFrontend* frontend; string source; - - bool disabled; - + ReaderMode mode; unsigned int num_fields; const threading::Field* const * fields; // raw mapping + + bool disabled; }; } diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index 75bb7fec50..d85a227577 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -12,7 +12,7 @@ namespace input { class InitMessage : public threading::InputMessage { public: - InitMessage(ReaderBackend* backend, const string source, const int mode, + InitMessage(ReaderBackend* backend, const string source, ReaderMode mode, const int num_fields, const threading::Field* const* fields) : threading::InputMessage("Init", backend), source(source), mode(mode), num_fields(num_fields), fields(fields) { } @@ -24,7 +24,7 @@ public: private: const string source; - const int mode; + const ReaderMode mode; const int num_fields; const threading::Field* const* fields; }; @@ -64,7 +64,7 @@ ReaderFrontend::~ReaderFrontend() { } -void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, +void ReaderFrontend::Init(string arg_source, ReaderMode mode, const int num_fields, const threading::Field* const* fields) { if ( disabled ) diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index c18e22a064..0de4e7c3dc 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -6,6 +6,8 @@ #include "../threading/MsgThread.h" #include "../threading/SerialTypes.h" +#include "ReaderBackend.h" + namespace input { class Manager; @@ -50,7 +52,7 @@ public: * * This method must only be called from the main thread. */ - void Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* fields); + void Init(string arg_source, ReaderMode mode, const int arg_num_fields, const threading::Field* const* fields); /** * Force an update of the current input source. Actual action depends diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 157ea90916..b5f81c8732 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -8,10 +8,6 @@ #include "../../threading/SerialTypes.h" -#define MANUAL 0 -#define REREAD 1 -#define STREAM 2 - #include #include #include @@ -87,25 +83,14 @@ void Ascii::DoClose() } } -bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) +bool Ascii::DoInit(string path, ReaderMode mode, int num_fields, const Field* const* fields) { - fname = path; - mode = arg_mode; mtime = 0; - num_fields = arg_num_fields; - fields = arg_fields; - - if ( (mode != MANUAL) && (mode != REREAD) && (mode != STREAM) ) - { - Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); - return false; - } - file = new ifstream(path.c_str()); if ( ! file->is_open() ) { - Error(Fmt("Init: cannot open %s", fname.c_str())); + Error(Fmt("Init: cannot open %s", path.c_str())); delete(file); file = 0; return false; @@ -113,7 +98,7 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c if ( ReadHeader(false) == false ) { - Error(Fmt("Init: cannot open %s; headers are incorrect", fname.c_str())); + Error(Fmt("Init: cannot open %s; headers are incorrect", path.c_str())); file->close(); delete(file); file = 0; @@ -162,9 +147,9 @@ bool Ascii::ReadHeader(bool useCached) //printf("Updating fields from description %s\n", line.c_str()); columnMap.clear(); - for ( unsigned int i = 0; i < num_fields; i++ ) + for ( unsigned int i = 0; i < NumFields(); i++ ) { - const Field* field = fields[i]; + const Field* field = Fields()[i]; map::iterator fit = ifields.find(field->name); if ( fit == ifields.end() ) @@ -179,7 +164,7 @@ bool Ascii::ReadHeader(bool useCached) } Error(Fmt("Did not find requested field %s in input data file %s.", - field->name.c_str(), fname.c_str())); + field->name.c_str(), Source().c_str())); return false; } @@ -377,14 +362,14 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) // read the entire file and send appropriate thingies back to InputMgr bool Ascii::DoUpdate() { - switch ( mode ) { - case REREAD: + switch ( Mode() ) { + case MODE_REREAD: { // check if the file has changed struct stat sb; - if ( stat(fname.c_str(), &sb) == -1 ) + if ( stat(Source().c_str(), &sb) == -1 ) { - Error(Fmt("Could not get stat for %s", fname.c_str())); + Error(Fmt("Could not get stat for %s", Source().c_str())); return false; } @@ -397,14 +382,14 @@ bool Ascii::DoUpdate() // fallthrough } - case MANUAL: - case STREAM: + case MODE_MANUAL: + case MODE_STREAM: { // dirty, fix me. (well, apparently after trying seeking, etc // - this is not that bad) if ( file && file->is_open() ) { - if ( mode == STREAM ) + if ( Mode() == MODE_STREAM ) { file->clear(); // remove end of file evil bits if ( !ReadHeader(true) ) @@ -415,10 +400,10 @@ bool Ascii::DoUpdate() file->close(); } - file = new ifstream(fname.c_str()); + file = new ifstream(Source().c_str()); if ( !file->is_open() ) { - Error(Fmt("cannot open %s", fname.c_str())); + Error(Fmt("cannot open %s", Source().c_str())); return false; } @@ -455,7 +440,7 @@ bool Ascii::DoUpdate() pos--; // for easy comparisons of max element. - Value** fields = new Value*[num_fields]; + Value** fields = new Value*[NumFields()]; int fpos = 0; for ( vector::iterator fit = columnMap.begin(); @@ -502,15 +487,15 @@ bool Ascii::DoUpdate() } //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); - assert ( (unsigned int) fpos == num_fields ); + assert ( (unsigned int) fpos == NumFields() ); - if ( mode == STREAM ) + if ( Mode() == MODE_STREAM ) Put(fields); else SendEntry(fields); } - if ( mode != STREAM ) + if ( Mode () != MODE_STREAM ) EndCurrentSend(); return true; @@ -520,13 +505,13 @@ bool Ascii::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - switch ( mode ) { - case MANUAL: + switch ( Mode() ) { + case MODE_MANUAL: // yay, we do nothing :) break; - case REREAD: - case STREAM: + case MODE_REREAD: + case MODE_STREAM: Update(); // call update and not DoUpdate, because update // checks disabled. break; diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index e5540c5467..a15acc29ee 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -10,7 +10,7 @@ namespace input { namespace reader { -// Description for input field mapping +// Description for input field mapping. struct FieldMapping { string name; TypeTag type; @@ -27,6 +27,9 @@ struct FieldMapping { FieldMapping subType(); }; +/** + * Reader for structured ASCII files. + */ class Ascii : public ReaderBackend { public: Ascii(ReaderFrontend* frontend); @@ -35,23 +38,18 @@ public: static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Ascii(frontend); } protected: - virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); + virtual bool DoInit(string path, ReaderMode mode, int arg_num_fields, const threading::Field* const* fields); virtual void DoClose(); virtual bool DoUpdate(); + virtual bool DoHeartbeat(double network_time, double current_time); private: - virtual bool DoHeartbeat(double network_time, double current_time); bool ReadHeader(bool useCached); bool GetLine(string& str); threading::Value* EntryToVal(string s, FieldMapping type); - unsigned int num_fields; - const threading::Field* const *fields; // raw mapping - ifstream* file; - string fname; - int mode; time_t mtime; // map columns in the file to columns to send back to the manager diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index c6cc1649eb..5644f26cb3 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -5,10 +5,6 @@ #include "../../threading/SerialTypes.h" -#define MANUAL 0 -#define REREAD 1 -#define STREAM 2 - #include #include #include @@ -19,8 +15,6 @@ using namespace input::reader; using threading::Value; using threading::Field; - - Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) { multiplication_factor = double(BifConst::InputBenchmark::factor); @@ -42,23 +36,13 @@ void Benchmark::DoClose() { } -bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) +bool Benchmark::DoInit(string path, ReaderMode mode, int num_fields, const Field* const* fields) { - mode = arg_mode; - - num_fields = arg_num_fields; - fields = arg_fields; num_lines = atoi(path.c_str()); if ( autospread != 0.0 ) autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); - if ( (mode != MANUAL) && (mode != REREAD) && (mode != STREAM) ) - { - Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); - return false; - } - heartbeatstarttime = CurrTime(); DoUpdate(); @@ -95,11 +79,11 @@ bool Benchmark::DoUpdate() int linestosend = num_lines * heart_beat_interval; for ( int i = 0; i < linestosend; i++ ) { - Value** field = new Value*[num_fields]; - for (unsigned int j = 0; j < num_fields; j++ ) - field[j] = EntryToVal(fields[j]->type, fields[j]->subtype); + Value** field = new Value*[NumFields()]; + for (unsigned int j = 0; j < NumFields(); j++ ) + field[j] = EntryToVal(Fields()[j]->type, Fields()[j]->subtype); - if ( mode == STREAM ) + if ( Mode() == MODE_STREAM ) // do not do tracking, spread out elements over the second that we have... Put(field); else @@ -125,7 +109,7 @@ bool Benchmark::DoUpdate() } - if ( mode != STREAM ) + if ( Mode() != MODE_STREAM ) EndCurrentSend(); return true; @@ -243,13 +227,13 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) num_lines += add; heartbeatstarttime = CurrTime(); - switch ( mode ) { - case MANUAL: + switch ( Mode() ) { + case MODE_MANUAL: // yay, we do nothing :) break; - case REREAD: - case STREAM: + case MODE_REREAD: + case MODE_STREAM: if ( multiplication_factor != 1 || add != 0 ) { // we have to document at what time we changed the factor to what value. @@ -270,6 +254,7 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) SendEvent("HeartbeatDone", 0, 0); break; + default: assert(false); } diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index ec14dc6567..2bb23ee17a 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -18,21 +18,16 @@ public: static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Benchmark(frontend); } protected: - virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); + virtual bool DoInit(string path, ReaderMode mode, int arg_num_fields, const threading::Field* const* fields); virtual void DoClose(); virtual bool DoUpdate(); - -private: virtual bool DoHeartbeat(double network_time, double current_time); +private: double CurrTime(); string RandomString(const int len); threading::Value* EntryToVal(TypeTag Type, TypeTag subtype); - unsigned int num_fields; - const threading::Field* const * fields; // raw mapping - - int mode; int num_lines; double multiplication_factor; int spread; diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index 6538da070b..c0da4969aa 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -9,10 +9,6 @@ #include "../../threading/SerialTypes.h" #include "../fdstream.h" -#define MANUAL 0 -#define REREAD 1 -#define STREAM 2 - #include #include #include @@ -48,7 +44,7 @@ void Raw::DoClose() } } -bool Raw::Open() +bool Raw::OpenInput() { if ( execute ) { @@ -72,13 +68,13 @@ bool Raw::Open() // This is defined in input/fdstream.h in = new boost::fdistream(fileno(file)); - if ( execute && mode == STREAM ) + if ( execute && Mode() == MODE_STREAM ) fcntl(fileno(file), F_SETFL, O_NONBLOCK); return true; } -bool Raw::Close() +bool Raw::CloseInput() { if ( file == NULL ) { @@ -103,25 +99,21 @@ bool Raw::Close() return true; } -bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) +bool Raw::DoInit(string path, ReaderMode mode, int num_fields, const Field* const* fields) { fname = path; - mode = arg_mode; mtime = 0; execute = false; firstrun = true; bool result; - num_fields = arg_num_fields; - fields = arg_fields; - if ( path.length() == 0 ) { Error("No source path provided"); return false; } - if ( arg_num_fields != 1 ) + if ( num_fields != 1 ) { Error("Filter for raw reader contains more than one field. " "Filters for the raw reader may only contain exactly one string field. " @@ -142,7 +134,7 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con execute = true; fname = path.substr(0, fname.length() - 1); - if ( (mode != MANUAL) && (mode != STREAM) ) { + if ( (mode != MODE_MANUAL) && (mode != MODE_STREAM) ) { Error(Fmt("Unsupported read mode %d for source %s in execution mode", mode, fname.c_str())); return false; @@ -152,13 +144,6 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con } else { execute = false; - if ( (mode != MANUAL) && (mode != REREAD) && (mode != STREAM) ) - { - Error(Fmt("Unsupported read mode %d for source %s", - mode, fname.c_str())); - return false; - } - result = Open(); } @@ -198,8 +183,8 @@ bool Raw::DoUpdate() else { - switch ( mode ) { - case REREAD: + switch ( Mode() ) { + case MODE_REREAD: { // check if the file has changed struct stat sb; @@ -219,9 +204,9 @@ bool Raw::DoUpdate() // fallthrough } - case MANUAL: - case STREAM: - if ( mode == STREAM && file != NULL && in != NULL ) + case MODE_MANUAL: + case MODE_STREAM: + if ( Mode() == MODE_STREAM && file != NULL && in != NULL ) { //fpurge(file); in->clear(); // remove end of file evil bits @@ -242,7 +227,7 @@ bool Raw::DoUpdate() string line; while ( GetLine(line) ) { - assert (num_fields == 1); + assert (NumFields() == 1); Value** fields = new Value*[1]; @@ -265,13 +250,13 @@ bool Raw::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - switch ( mode ) { - case MANUAL: + switch ( Mode() ) { + case MODE_MANUAL: // yay, we do nothing :) break; - case REREAD: - case STREAM: + case MODE_REREAD: + case MODE_STREAM: Update(); // call update and not DoUpdate, because update // checks disabled. break; diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index 3fa09309b0..76c9125544 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -22,24 +22,19 @@ public: static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); } protected: - virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); + virtual bool DoInit(string path, ReaderMode mode, int arg_num_fields, const threading::Field* const* fields); virtual void DoClose(); virtual bool DoUpdate(); - -private: virtual bool DoHeartbeat(double network_time, double current_time); - bool Open(); - bool Close(); +private: + bool OpenInput(); + bool CloseInput(); bool GetLine(string& str); - unsigned int num_fields; - const threading::Field* const * fields; // raw mapping - + string fname; // Sources with a potential " |" removed. istream* in; FILE* file; - string fname; - int mode; bool execute; bool firstrun; time_t mtime; diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 267d793e06..6071e70271 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -12,9 +12,6 @@ Manager::Manager() next_beat = 0; terminating = false; idle = true; - - heart_beat_interval = double(BifConst::Threading::heart_beat_interval); - DBG_LOG(DBG_THREADING, "Heart beat interval set to %f", heart_beat_interval); } Manager::~Manager() @@ -61,12 +58,6 @@ void Manager::KillThreads() void Manager::AddThread(BasicThread* thread) { - if ( heart_beat_interval == 0 ) { - // Sometimes initialization does not seem to work from constructor. - heart_beat_interval = double(BifConst::Threading::heart_beat_interval); - DBG_LOG(DBG_THREADING, "Heart beat interval set to %f", heart_beat_interval); - } - DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str()); all_threads.push_back(thread); idle = false; @@ -107,7 +98,7 @@ void Manager::Process() if ( network_time && (network_time > next_beat || ! next_beat) ) { do_beat = true; - next_beat = ::network_time + heart_beat_interval; + next_beat = ::network_time + BifConst::Threading::heart_beat_interval; } did_process = false; diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 14c5893214..1afd115da0 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -126,8 +126,6 @@ protected: virtual const char* Tag() { return "threading::Manager"; } private: - int heart_beat_interval; - typedef std::list all_thread_list; all_thread_list all_threads; From 14ea7801767bf10c859adcb6539b0425e2286aab Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 30 May 2012 16:40:49 -0700 Subject: [PATCH 147/149] And now it even compiles after my earlier changes. --- src/input/readers/Raw.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index c0da4969aa..f6e61a906e 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -39,9 +39,7 @@ Raw::~Raw() void Raw::DoClose() { if ( file != 0 ) - { - Close(); - } + CloseInput(); } bool Raw::OpenInput() @@ -140,11 +138,13 @@ bool Raw::DoInit(string path, ReaderMode mode, int num_fields, const Field* cons return false; } - result = Open(); + result = OpenInput(); - } else { + } + else + { execute = false; - result = Open(); + result = OpenInput(); } if ( result == false ) @@ -213,8 +213,8 @@ bool Raw::DoUpdate() break; } - Close(); - if ( ! Open() ) + CloseInput(); + if ( ! OpenInput() ) return false; break; From f34ebb7b60e3131b5f93cf84d627c84701e93168 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 30 May 2012 19:12:43 -0700 Subject: [PATCH 148/149] Updating submodule(s). [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- aux/btest | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index cb9c0177fc..b68469fc5d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.0-571 | 2012-05-30 19:12:43 -0700 + + * Updating submodule(s). + 2.0-570 | 2012-05-30 19:08:18 -0700 * A new input framework enables scripts to read in external data diff --git a/VERSION b/VERSION index dea7c2bf67..b3387dc11b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-570 +2.0-571 diff --git a/aux/btest b/aux/btest index 3ee8d4b323..4697bf4c80 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 3ee8d4b3232d74ed7bd475819193ad3a4055e2f5 +Subproject commit 4697bf4c8046a3ab7d5e00e926c5db883cb44664 From be0316ee29979532a4f0fa6df1b97e10613006bd Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 30 May 2012 19:26:43 -0700 Subject: [PATCH 149/149] Fixes for running tests in parallel. --- testing/btest/istate/bro-ipv6-socket.bro | 2 +- testing/btest/istate/broccoli-ipv6-socket.bro | 2 +- testing/btest/istate/broccoli-ssl.bro | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/btest/istate/bro-ipv6-socket.bro b/testing/btest/istate/bro-ipv6-socket.bro index ae77a42c54..b339bf4487 100644 --- a/testing/btest/istate/bro-ipv6-socket.bro +++ b/testing/btest/istate/bro-ipv6-socket.bro @@ -1,4 +1,4 @@ -# @TEST-GROUP: comm +# @TEST-SERIALIZE: comm # # @TEST-REQUIRES: ifconfig | grep -q -E "inet6 ::1|inet6 addr: ::1" # diff --git a/testing/btest/istate/broccoli-ipv6-socket.bro b/testing/btest/istate/broccoli-ipv6-socket.bro index e36ac9e9f7..21067c1b23 100644 --- a/testing/btest/istate/broccoli-ipv6-socket.bro +++ b/testing/btest/istate/broccoli-ipv6-socket.bro @@ -1,4 +1,4 @@ -# @TEST-GROUP: comm +# @TEST-SERIALIZE: comm # # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib # @TEST-REQUIRES: ifconfig | grep -q -E "inet6 ::1|inet6 addr: ::1" diff --git a/testing/btest/istate/broccoli-ssl.bro b/testing/btest/istate/broccoli-ssl.bro index 61401c483a..4465cd1bb3 100644 --- a/testing/btest/istate/broccoli-ssl.bro +++ b/testing/btest/istate/broccoli-ssl.bro @@ -1,4 +1,4 @@ -# @TEST-GROUP: comm +# @TEST-SERIALIZE: comm # # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib #