From ca17a1cf46ce50a84038eecfa5da401351777d4b Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 16:18:54 -0800 Subject: [PATCH 01/42] make logging framework send the protocol to the writer. for use in future writers, that have a special type for port, which includes the protocol. --- src/LogMgr.cc | 32 +++++++++++++++++++++++++++++--- src/LogMgr.h | 4 ++++ src/LogWriterAscii.cc | 5 ++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 0b706f6417..8873b22b46 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -114,6 +114,10 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } + +// if ( type == TYPE_PORT && present ) +// delete val.port_val.proto; + } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -186,9 +190,12 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Read(&val.uint_val, "uint"); + case TYPE_PORT: + val.port_val.proto = new string; + return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -301,9 +308,11 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Write(val.uint_val, "uint"); + case TYPE_PORT: + return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -1062,6 +1071,22 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } +string LogMgr::TransportProtoToString(TransportProto p) { + switch ( p ) { + case TRANSPORT_UNKNOWN: + return "unknown"; + case TRANSPORT_TCP: + return "tcp"; + case TRANSPORT_UDP: + return "udp"; + case TRANSPORT_ICMP: + return "icmp"; + } + + assert(false); + return ""; +} + LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1093,7 +1118,8 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; case TYPE_PORT: - lval->val.uint_val = val->AsPortVal()->Port(); + lval->val.port_val.port = val->AsPortVal()->Port(); + lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..52acd04be5 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -34,10 +34,12 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; + struct port_t { bro_uint_t port; string* proto; }; union _val { bro_int_t int_val; bro_uint_t uint_val; + port_t port_val; uint32 addr_val[NUM_ADDR_WORDS]; subnet_type subnet_val; double double_val; @@ -132,6 +134,8 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); + string TransportProtoToString(TransportProto p); + vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9b1fda3b62..c449c1a788 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -169,10 +169,13 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: desc->Add(val->val.uint_val); break; + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + case TYPE_SUBNET: desc->Add(dotted_addr(val->val.subnet_val.net)); desc->Add("/"); From a0da991030836d53bc669b64ad3de9b4dba34070 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:56:26 -0800 Subject: [PATCH 02/42] memleak fix. --- src/LogMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 8873b22b46..729979b4ef 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -115,8 +115,8 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } -// if ( type == TYPE_PORT && present ) -// delete val.port_val.proto; + if ( type == TYPE_PORT && present ) + delete val.port_val.proto; } From 4b3cc95f7206d61614a0193508acc0f60828e3df Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 7 Dec 2011 12:43:15 -0800 Subject: [PATCH 03/42] send enum instead of string --- src/LogMgr.cc | 50 ++++++++++++++++++++++++++------------------------ src/LogMgr.h | 4 +--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 729979b4ef..58581e2943 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -115,9 +115,6 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } - if ( type == TYPE_PORT && present ) - delete val.port_val.proto; - } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -192,9 +189,30 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNTER: return fmt->Read(&val.uint_val, "uint"); - case TYPE_PORT: - val.port_val.proto = new string; - return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_PORT: { + int proto; + if ( ! (fmt->Read(&val.port_val.port, "port") && fmt->Read(&proto, "proto") ) ) { + return false; + } + + switch (proto) { + case 0: + val.port_val.proto = TRANSPORT_UNKNOWN; + break; + case 1: + val.port_val.proto = TRANSPORT_TCP; + break; + case 2: + val.port_val.proto = TRANSPORT_UDP; + break; + case 3: + val.port_val.proto = TRANSPORT_ICMP; + break; + default: + return false; + } + return true; + } case TYPE_SUBNET: { @@ -311,7 +329,7 @@ bool LogVal::Write(SerializationFormat* fmt) const return fmt->Write(val.uint_val, "uint"); case TYPE_PORT: - return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); case TYPE_SUBNET: { @@ -1071,22 +1089,6 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } -string LogMgr::TransportProtoToString(TransportProto p) { - switch ( p ) { - case TRANSPORT_UNKNOWN: - return "unknown"; - case TRANSPORT_TCP: - return "tcp"; - case TRANSPORT_UDP: - return "udp"; - case TRANSPORT_ICMP: - return "icmp"; - } - - assert(false); - return ""; -} - LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1119,7 +1121,7 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) case TYPE_PORT: lval->val.port_val.port = val->AsPortVal()->Port(); - lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); + lval->val.port_val.proto = val->AsPortVal()->PortType(); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 52acd04be5..8c2c8250f8 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -34,7 +34,7 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; - struct port_t { bro_uint_t port; string* proto; }; + struct port_t { bro_uint_t port; TransportProto proto; }; union _val { bro_int_t int_val; @@ -134,8 +134,6 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); - string TransportProtoToString(TransportProto p); - vector streams; // Indexed by stream enum. }; From e4e770d47517ad6b9c730a7132037feb265ffe30 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 26 Jan 2012 17:47:36 -0800 Subject: [PATCH 04/42] Threaded logging framework. This is based on Gilbert's code but I ended up refactoring it quite a bit. That's why I didn't do a direct merge but started with a new branch and copied things over to adapt. It looks quite a bit different now as I tried to generalize things a bit more to also support the Input Framework. The larger changes code are: - Moved all logging code into subdirectory src/logging/. Code here is in namespace "logging". - Moved all threading code into subdirectory src/threading/. Code here is in namespace "threading". - Introduced a central thread manager that tracks threads and is in charge of termination and (eventually) statistics. - Refactored logging independent threading code into base classes BasicThread and MsgThread. The former encapsulates all the pthread code with simple start/stop methods and provides a single Run() method to override. The latter is derived from BasicThread and adds bi-directional message passing between main and child threads. The hope is that the Input Framework can reuse this part quite directly. - A log writer is now split into a general WriterFrontend (LogEmissary in Gilbert's code) and a type-specific WriterBackend. Specific writers are implemented by deriving from the latter. (The plugin interface is almost unchanged compared to the 2.0 version.). Frontend and backend communicate via MsgThread's message passing. - MsgThread (and thus WriterBackend) has a Heartbeat() method that a thread can override to execute code on a regular basis. It's triggered roughly once a second by the main thread. - Integration into "the rest of Bro". Threads can send messages to the reporter and do debugging output; they are hooked into the I/O loop for sending messages back; and there's a new debugging stream "threading" that logs, well, threading activity. This all seems to work for the most part, but it's not done yet. TODO list: - Not all tests pass yet. In particular, diffs for the external tests seem to indicate some memory problem (no crashes, just an occasional weird character). - Only tested in --enable-debug mode. - Only tested on Linux. - Needs leak check. - Each log write is currently a single inter-thread message. Bring Gilbert's bulk writes back. - Code needs further cleanup. - Document the class API. - Document the internal structure of the logging framework. - Check for robustness: live traffic, aborting, signals, etc. - Add thread statistics to profile.log (most of the code is there). - Customize the OS-visible thread names on platforms that support it. --- src/Attr.cc | 4 +- src/CMakeLists.txt | 19 +- src/DebugLogger.cc | 2 +- src/DebugLogger.h | 1 + src/LogWriter.cc | 158 ---------- src/LogWriterNone.cc | 16 - src/LogWriterNone.h | 30 -- src/RemoteSerializer.cc | 22 +- src/RemoteSerializer.h | 15 +- src/logging.bif | 3 +- src/{LogMgr.cc => logging/Manager.cc} | 289 +++++++++--------- src/{LogMgr.h => logging/Manager.h} | 75 +++-- src/logging/WriterBackend.cc | 161 ++++++++++ src/{LogWriter.h => logging/WriterBackend.h} | 123 ++++---- src/logging/WriterFrontend.cc | 175 +++++++++++ src/logging/WriterFrontend.h | 66 ++++ .../writers/Ascii.cc} | 41 +-- .../writers/Ascii.h} | 34 ++- src/logging/writers/None.cc | 19 ++ src/logging/writers/None.h | 35 +++ src/main.cc | 20 +- src/threading/BasicThread.cc | 129 ++++++++ src/threading/BasicThread.h | 63 ++++ src/threading/Manager.cc | 104 +++++++ src/threading/Manager.h | 52 ++++ src/threading/MsgThread.cc | 285 +++++++++++++++++ src/threading/MsgThread.h | 157 ++++++++++ src/threading/Queue.h | 150 +++++++++ 28 files changed, 1745 insertions(+), 503 deletions(-) delete mode 100644 src/LogWriter.cc delete mode 100644 src/LogWriterNone.cc delete mode 100644 src/LogWriterNone.h rename src/{LogMgr.cc => logging/Manager.cc} (87%) rename src/{LogMgr.h => logging/Manager.h} (68%) create mode 100644 src/logging/WriterBackend.cc rename src/{LogWriter.h => logging/WriterBackend.h} (70%) create mode 100644 src/logging/WriterFrontend.cc create mode 100644 src/logging/WriterFrontend.h rename src/{LogWriterAscii.cc => logging/writers/Ascii.cc} (89%) rename src/{LogWriterAscii.h => logging/writers/Ascii.h} (57%) create mode 100644 src/logging/writers/None.cc create mode 100644 src/logging/writers/None.h create mode 100644 src/threading/BasicThread.cc create mode 100644 src/threading/BasicThread.h create mode 100644 src/threading/Manager.cc create mode 100644 src/threading/Manager.h create mode 100644 src/threading/MsgThread.cc create mode 100644 src/threading/MsgThread.h create mode 100644 src/threading/Queue.h diff --git a/src/Attr.cc b/src/Attr.cc index aed9165182..b877250f52 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -5,7 +5,7 @@ #include "Attr.h" #include "Expr.h" #include "Serializer.h" -#include "LogMgr.h" +#include "logging/Manager.h" const char* attr_name(attr_tag t) { @@ -416,7 +416,7 @@ void Attributes::CheckAttr(Attr* a) break; case ATTR_LOG: - if ( ! LogVal::IsCompatibleType(type) ) + if ( ! logging::Value::IsCompatibleType(type) ) Error("&log applied to a type that cannot be logged"); break; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0e29082db3..61a4847b70 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -213,6 +213,8 @@ binpac_target(syslog.pac ######################################################################## ## bro target +find_package (Threads) + # This macro stores associated headers for any C/C++ source files given # as arguments (past _var) as a list in the CMake variable named "_var". macro(COLLECT_HEADERS _var) @@ -334,10 +336,6 @@ set(bro_SRCS IRC.cc List.cc Reporter.cc - LogMgr.cc - LogWriter.cc - LogWriterAscii.cc - LogWriterNone.cc Login.cc MIME.cc NCP.cc @@ -409,6 +407,17 @@ set(bro_SRCS PacketDumper.cc strsep.c modp_numtoa.c + + threading/BasicThread.cc + threading/Manager.cc + threading/MsgThread.cc + + logging/Manager.cc + logging/WriterBackend.cc + logging/WriterFrontend.cc + logging/writers/Ascii.cc + logging/writers/None.cc + ${dns_SRCS} ${openssl_SRCS} ) @@ -421,7 +430,7 @@ add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}") add_executable(bro ${bro_SRCS} ${bro_HEADERS}) -target_link_libraries(bro ${brodeps}) +target_link_libraries(bro ${brodeps} ${CMAKE_THREAD_LIBS_INIT}) install(TARGETS bro DESTINATION bin) install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base) diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index d60fdd70c8..c41a0552c6 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -15,7 +15,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, - { "logging", 0, false } + { "logging", 0, false }, { "threading", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index a2dece5b3c..71e21bfa26 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -24,6 +24,7 @@ enum DebugStream { DBG_DPD, // Dynamic application detection framework DBG_TM, // Time-machine packet input via Brocolli DBG_LOGGING, // Logging streams + DBG_THREADING, // Threading system NUM_DBGS // Has to be last }; diff --git a/src/LogWriter.cc b/src/LogWriter.cc deleted file mode 100644 index 8584a0b0b5..0000000000 --- a/src/LogWriter.cc +++ /dev/null @@ -1,158 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "util.h" -#include "LogWriter.h" - -LogWriter::LogWriter() - { - buf = 0; - buf_len = 1024; - buffering = true; - disabled = false; - } - -LogWriter::~LogWriter() - { - if ( buf ) - free(buf); - - for(int i = 0; i < num_fields; ++i) - delete fields[i]; - - delete [] fields; - } - -bool LogWriter::Init(string arg_path, int arg_num_fields, - const LogField* const * arg_fields) - { - path = arg_path; - num_fields = arg_num_fields; - fields = arg_fields; - - if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) - { - disabled = true; - return false; - } - - return true; - } - -bool LogWriter::Write(int arg_num_fields, LogVal** vals) - { - // Double-check that the arguments match. If we get this from remote, - // something might be mixed up. - if ( num_fields != arg_num_fields ) - { - DBG_LOG(DBG_LOGGING, "Number of fields don't match in LogWriter::Write() (%d vs. %d)", - arg_num_fields, num_fields); - - DeleteVals(vals); - return false; - } - - for ( int i = 0; i < num_fields; ++i ) - { - if ( vals[i]->type != fields[i]->type ) - { - DBG_LOG(DBG_LOGGING, "Field type doesn't match in LogWriter::Write() (%d vs. %d)", - vals[i]->type, fields[i]->type); - DeleteVals(vals); - return false; - } - } - - bool result = DoWrite(num_fields, fields, vals); - - DeleteVals(vals); - - if ( ! result ) - disabled = true; - - return result; - } - -bool LogWriter::SetBuf(bool enabled) - { - if ( enabled == buffering ) - // No change. - return true; - - buffering = enabled; - - if ( ! DoSetBuf(enabled) ) - { - disabled = true; - return false; - } - - return true; - } - -bool LogWriter::Rotate(string rotated_path, double open, - double close, bool terminating) - { - if ( ! DoRotate(rotated_path, open, close, terminating) ) - { - disabled = true; - return false; - } - - return true; - } - -bool LogWriter::Flush() - { - if ( ! DoFlush() ) - { - disabled = true; - return false; - } - - return true; - } - -void LogWriter::Finish() - { - DoFinish(); - } - -const char* LogWriter::Fmt(const char* format, ...) - { - if ( ! buf ) - buf = (char*) malloc(buf_len); - - va_list al; - va_start(al, format); - int n = safe_vsnprintf(buf, buf_len, format, al); - va_end(al); - - if ( (unsigned int) n >= buf_len ) - { // Not enough room, grow the buffer. - buf_len = n + 32; - buf = (char*) realloc(buf, buf_len); - - // Is it portable to restart? - va_start(al, format); - n = safe_vsnprintf(buf, buf_len, format, al); - va_end(al); - } - - return buf; - } - -void LogWriter::Error(const char *msg) - { - log_mgr->Error(this, msg); - } - -void LogWriter::DeleteVals(LogVal** vals) - { - log_mgr->DeleteVals(num_fields, vals); - } - -bool LogWriter::FinishedRotation(string new_name, string old_name, double open, - double close, bool terminating) - { - return log_mgr->FinishedRotation(this, new_name, old_name, open, close, terminating); - } diff --git a/src/LogWriterNone.cc b/src/LogWriterNone.cc deleted file mode 100644 index 592772afdb..0000000000 --- a/src/LogWriterNone.cc +++ /dev/null @@ -1,16 +0,0 @@ - -#include "LogWriterNone.h" - -bool LogWriterNone::DoRotate(string rotated_path, double open, - double close, bool terminating) - { - if ( ! FinishedRotation(string("/dev/null"), Path(), open, close, terminating)) - { - Error(Fmt("error rotating %s", Path().c_str())); - return false; - } - - return true; - } - - diff --git a/src/LogWriterNone.h b/src/LogWriterNone.h deleted file mode 100644 index 3811a19469..0000000000 --- a/src/LogWriterNone.h +++ /dev/null @@ -1,30 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. -// -// Dummy log writer that just discards everything (but still pretends to rotate). - -#ifndef LOGWRITERNONE_H -#define LOGWRITERNONE_H - -#include "LogWriter.h" - -class LogWriterNone : public LogWriter { -public: - LogWriterNone() {} - ~LogWriterNone() {}; - - static LogWriter* Instantiate() { return new LogWriterNone; } - -protected: - virtual bool DoInit(string path, int num_fields, - const LogField* const * fields) { return true; } - - virtual bool DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals) { return true; } - virtual bool DoSetBuf(bool enabled) { return true; } - virtual bool DoRotate(string rotated_path, double open, double close, - bool terminating); - virtual bool DoFlush() { return true; } - virtual void DoFinish() {} -}; - -#endif diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index b72a6dcc1a..a75812b42b 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -183,8 +183,8 @@ #include "Sessions.h" #include "File.h" #include "Conn.h" -#include "LogMgr.h" #include "Reporter.h" +#include "logging/Manager.h" extern "C" { #include "setsignal.h" @@ -2476,7 +2476,7 @@ bool RemoteSerializer::ProcessRemotePrint() return true; } -bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields) +bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields) { loop_over_list(peers, i) { @@ -2486,7 +2486,7 @@ bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string return true; } -bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields) +bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields) { SetErrorDescr("logging"); @@ -2540,7 +2540,7 @@ error: return false; } -bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals) +bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals) { loop_over_list(peers, i) { @@ -2550,7 +2550,7 @@ bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, i return true; } -bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals) +bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals) { if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) return false; @@ -2641,7 +2641,7 @@ bool RemoteSerializer::ProcessLogCreateWriter() EnumVal* id_val = 0; EnumVal* writer_val = 0; - LogField** fields = 0; + logging::Field** fields = 0; BinarySerializationFormat fmt; fmt.StartRead(current_args->data, current_args->len); @@ -2658,11 +2658,11 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( ! success ) goto error; - fields = new LogField* [num_fields]; + fields = new logging::Field* [num_fields]; for ( int i = 0; i < num_fields; i++ ) { - fields[i] = new LogField; + fields[i] = new logging::Field; if ( ! fields[i]->Read(&fmt) ) goto error; } @@ -2703,7 +2703,7 @@ bool RemoteSerializer::ProcessLogWrite() // Unserialize one entry. EnumVal* id_val = 0; EnumVal* writer_val = 0; - LogVal** vals = 0; + logging::Value** vals = 0; int id, writer; string path; @@ -2717,11 +2717,11 @@ bool RemoteSerializer::ProcessLogWrite() if ( ! success ) goto error; - vals = new LogVal* [num_fields]; + vals = new logging::Value* [num_fields]; for ( int i = 0; i < num_fields; i++ ) { - vals[i] = new LogVal; + vals[i] = new logging::Value; if ( ! vals[i]->Read(&fmt) ) goto error; } diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index b64fdcbe66..ba0bde7d41 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -14,8 +14,11 @@ // FIXME: Change this to network byte order class IncrementalSendTimer; -class LogField; -class LogVal; + +namespace logging { + class Field; + class Value; +} // This class handles the communication done in Bro's main loop. class RemoteSerializer : public Serializer, public IOSource { @@ -99,13 +102,13 @@ public: bool SendPrintHookEvent(BroFile* f, const char* txt, size_t len); // Send a request to create a writer on a remote side. - bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields); + bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields); // Broadcasts a request to create a writer. - bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields); + bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields); // Broadcast a log entry to everybody interested. - bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals); + bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals); // Synchronzizes time with all connected peers. Returns number of // current sync-point, or -1 on error. @@ -300,7 +303,7 @@ protected: bool SendID(SerialInfo* info, Peer* peer, const ID& id); bool SendCapabilities(Peer* peer); bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p); - bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals); + bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals); void UnregisterHandlers(Peer* peer); void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0); diff --git a/src/logging.bif b/src/logging.bif index 31e1bebacd..c8960b4e38 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -3,8 +3,9 @@ module Log; %%{ -#include "LogMgr.h" #include "NetVar.h" + +#include "logging/Manager.h" %%} type Filter: record; diff --git a/src/LogMgr.cc b/src/logging/Manager.cc similarity index 87% rename from src/LogMgr.cc rename to src/logging/Manager.cc index 28e9a2ac1f..09c5030fdc 100644 --- a/src/LogMgr.cc +++ b/src/logging/Manager.cc @@ -2,33 +2,38 @@ #include -#include "LogMgr.h" -#include "Event.h" -#include "EventHandler.h" -#include "NetVar.h" -#include "Net.h" +#include "../Event.h" +#include "../EventHandler.h" +#include "../NetVar.h" +#include "../Net.h" -#include "LogWriterAscii.h" -#include "LogWriterNone.h" +#include "Manager.h" +#include "WriterFrontend.h" +#include "WriterBackend.h" + +#include "writers/Ascii.h" +#include "writers/None.h" + +using namespace logging; // Structure describing a log writer type. -struct LogWriterDefinition { +struct WriterDefinition { bro_int_t type; // The type. const char *name; // Descriptive name for error messages. bool (*init)(); // An optional one-time initialization function. - LogWriter* (*factory)(); // A factory function creating instances. + WriterBackend* (*factory)(); // A factory function creating instances. }; // Static table defining all availabel log writers. -LogWriterDefinition log_writers[] = { - { BifEnum::Log::WRITER_NONE, "None", 0, LogWriterNone::Instantiate }, - { BifEnum::Log::WRITER_ASCII, "Ascii", 0, LogWriterAscii::Instantiate }, +WriterDefinition log_writers[] = { + { BifEnum::Log::WRITER_NONE, "None", 0, writer::None::Instantiate }, + { BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate }, // End marker, don't touch. - { BifEnum::Log::WRITER_DEFAULT, "None", 0, (LogWriter* (*)())0 } + { BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)())0 } }; -struct LogMgr::Filter { +struct Manager::Filter { string name; EnumVal* id; Func* pred; @@ -42,7 +47,7 @@ struct LogMgr::Filter { Func* postprocessor; int num_fields; - LogField** fields; + Field** fields; // Vector indexed by field number. Each element is a list of record // indices defining a path leading to the value across potential @@ -52,16 +57,16 @@ struct LogMgr::Filter { ~Filter(); }; -struct LogMgr::WriterInfo { +struct Manager::WriterInfo { EnumVal* type; double open_time; Timer* rotation_timer; double interval; Func* postprocessor; - LogWriter* writer; + WriterFrontend* writer; }; -struct LogMgr::Stream { +struct Manager::Stream { EnumVal* id; bool enabled; string name; @@ -78,7 +83,7 @@ struct LogMgr::Stream { ~Stream(); }; -bool LogField::Read(SerializationFormat* fmt) +bool Field::Read(SerializationFormat* fmt) { int t; int st; @@ -90,12 +95,12 @@ bool LogField::Read(SerializationFormat* fmt) return success; } -bool LogField::Write(SerializationFormat* fmt) const +bool Field::Write(SerializationFormat* fmt) const { return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); } -LogVal::~LogVal() +Value::~Value() { if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC) && present ) @@ -118,7 +123,7 @@ LogVal::~LogVal() } } -bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) +bool Value::IsCompatibleType(BroType* t, bool atomic_only) { if ( ! t ) return false; @@ -169,7 +174,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) return false; } -bool LogVal::Read(SerializationFormat* fmt) +bool Value::Read(SerializationFormat* fmt) { int ty; @@ -249,11 +254,11 @@ bool LogVal::Read(SerializationFormat* fmt) if ( ! fmt->Read(&val.set_val.size, "set_size") ) return false; - val.set_val.vals = new LogVal* [val.set_val.size]; + val.set_val.vals = new Value* [val.set_val.size]; for ( int i = 0; i < val.set_val.size; ++i ) { - val.set_val.vals[i] = new LogVal; + val.set_val.vals[i] = new Value; if ( ! val.set_val.vals[i]->Read(fmt) ) return false; @@ -267,11 +272,11 @@ bool LogVal::Read(SerializationFormat* fmt) if ( ! fmt->Read(&val.vector_val.size, "vector_size") ) return false; - val.vector_val.vals = new LogVal* [val.vector_val.size]; + val.vector_val.vals = new Value* [val.vector_val.size]; for ( int i = 0; i < val.vector_val.size; ++i ) { - val.vector_val.vals[i] = new LogVal; + val.vector_val.vals[i] = new Value; if ( ! val.vector_val.vals[i]->Read(fmt) ) return false; @@ -281,13 +286,13 @@ bool LogVal::Read(SerializationFormat* fmt) } default: - reporter->InternalError("unsupported type %s in LogVal::Write", type_name(type)); + reporter->InternalError("unsupported type %s in Value::Write", type_name(type)); } return false; } -bool LogVal::Write(SerializationFormat* fmt) const +bool Value::Write(SerializationFormat* fmt) const { if ( ! (fmt->Write((int)type, "type") && fmt->Write(present, "present")) ) @@ -382,13 +387,13 @@ bool LogVal::Write(SerializationFormat* fmt) const } default: - reporter->InternalError("unsupported type %s in LogVal::REad", type_name(type)); + reporter->InternalError("unsupported type %s in Value::REad", type_name(type)); } return false; } -LogMgr::Filter::~Filter() +Manager::Filter::~Filter() { for ( int i = 0; i < num_fields; ++i ) delete fields[i]; @@ -398,7 +403,7 @@ LogMgr::Filter::~Filter() Unref(path_val); } -LogMgr::Stream::~Stream() +Manager::Stream::~Stream() { Unref(columns); @@ -421,17 +426,64 @@ LogMgr::Stream::~Stream() delete *f; } -LogMgr::LogMgr() +Manager::Manager() { } -LogMgr::~LogMgr() +Manager::~Manager() { for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) delete *s; } -LogMgr::Stream* LogMgr::FindStream(EnumVal* id) +WriterBackend* Manager::CreateBackend(bro_int_t type) + { + WriterDefinition* ld = log_writers; + + while ( true ) + { + if ( ld->type == BifEnum::Log::WRITER_DEFAULT ) + { + reporter->Error("unknow writer when creating writer"); + return 0; + } + + if ( ld->type == type ) + break; + + if ( ! ld->factory ) + // Oops, we can't instantiate this guy. + return 0; + + // If the writer has an init function, call it. + if ( ld->init ) + { + if ( (*ld->init)() ) + // Clear the init function so that we won't + // call it again later. + ld->init = 0; + else + // Init failed, disable by deleting factory + // function. + ld->factory = 0; + + DBG_LOG(DBG_LOGGING, "failed to init writer class %s", + ld->name); + + return false; + } + + ++ld; + } + + assert(ld->factory); + + WriterBackend* backend = (*ld->factory)(); + assert(backend); + return backend; + } + +Manager::Stream* Manager::FindStream(EnumVal* id) { unsigned int idx = id->AsEnum(); @@ -441,7 +493,7 @@ LogMgr::Stream* LogMgr::FindStream(EnumVal* id) return streams[idx]; } -LogMgr::WriterInfo* LogMgr::FindWriter(LogWriter* writer) +Manager::WriterInfo* Manager::FindWriter(WriterFrontend* writer) { for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) { @@ -460,7 +512,7 @@ LogMgr::WriterInfo* LogMgr::FindWriter(LogWriter* writer) return 0; } -void LogMgr::RemoveDisabledWriters(Stream* stream) +void Manager::RemoveDisabledWriters(Stream* stream) { list disabled; @@ -468,6 +520,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) { if ( j->second && j->second->writer->Disabled() ) { + j->second->writer->Stop(); delete j->second; disabled.push_back(j->first); } @@ -477,7 +530,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) +bool Manager::CreateStream(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); @@ -497,7 +550,7 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) if ( ! (columns->FieldDecl(i)->FindAttr(ATTR_LOG)) ) continue; - if ( ! LogVal::IsCompatibleType(columns->FieldType(i)) ) + if ( ! Value::IsCompatibleType(columns->FieldType(i)) ) { reporter->Error("type of field '%s' is not support for logging output", columns->FieldName(i)); @@ -569,7 +622,7 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) return true; } -bool LogMgr::EnableStream(EnumVal* id) +bool Manager::EnableStream(EnumVal* id) { Stream* stream = FindStream(id); @@ -585,7 +638,7 @@ bool LogMgr::EnableStream(EnumVal* id) return true; } -bool LogMgr::DisableStream(EnumVal* id) +bool Manager::DisableStream(EnumVal* id) { Stream* stream = FindStream(id); @@ -602,7 +655,7 @@ bool LogMgr::DisableStream(EnumVal* id) } // Helper for recursive record field unrolling. -bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, +bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices) { for ( int i = 0; i < rt->NumFields(); ++i ) @@ -696,9 +749,9 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, filter->indices.push_back(new_indices); - filter->fields = (LogField**) + filter->fields = (Field**) realloc(filter->fields, - sizeof(LogField) * ++filter->num_fields); + sizeof(Field) * ++filter->num_fields); if ( ! filter->fields ) { @@ -706,14 +759,14 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, return false; } - LogField* field = new LogField(); + Field* field = new Field(); field->name = new_path; field->type = t->Tag(); - if ( field->type == TYPE_TABLE ) + if ( field->type == TYPE_TABLE ) { field->subtype = t->AsSetType()->Indices()->PureType()->Tag(); - } - else if ( field->type == TYPE_VECTOR ) + } + else if ( field->type == TYPE_VECTOR ) { field->subtype = t->AsVectorType()->YieldType()->Tag(); } @@ -723,7 +776,7 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, return true; } -bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) +bool Manager::AddFilter(EnumVal* id, RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); @@ -819,7 +872,7 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) for ( int i = 0; i < filter->num_fields; i++ ) { - LogField* field = filter->fields[i]; + Field* field = filter->fields[i]; DBG_LOG(DBG_LOGGING, " field %10s: %s", field->name.c_str(), type_name(field->type)); } @@ -828,12 +881,12 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) return true; } -bool LogMgr::RemoveFilter(EnumVal* id, StringVal* name) +bool Manager::RemoveFilter(EnumVal* id, StringVal* name) { return RemoveFilter(id, name->AsString()->CheckString()); } -bool LogMgr::RemoveFilter(EnumVal* id, string name) +bool Manager::RemoveFilter(EnumVal* id, string name) { Stream* stream = FindStream(id); if ( ! stream ) @@ -860,7 +913,7 @@ bool LogMgr::RemoveFilter(EnumVal* id, string name) return true; } -bool LogMgr::Write(EnumVal* id, RecordVal* columns) +bool Manager::Write(EnumVal* id, RecordVal* columns) { bool error = false; @@ -980,7 +1033,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(filter->writer->AsEnum(), path)); - LogWriter* writer = 0; + WriterFrontend* writer = 0; if ( w != stream->writers.end() ) // We know this writer already. @@ -990,12 +1043,12 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) { // No, need to create one. - // Copy the fields for LogWriter::Init() as it will take - // ownership. - LogField** arg_fields = new LogField*[filter->num_fields]; + // Copy the fields for WriterFrontend::Init() as it + // will take ownership. + Field** arg_fields = new Field*[filter->num_fields]; for ( int j = 0; j < filter->num_fields; ++j ) - arg_fields[j] = new LogField(*filter->fields[j]); + arg_fields[j] = new Field(*filter->fields[j]); if ( filter->remote ) remote_serializer->SendLogCreateWriter(stream->id, @@ -1034,7 +1087,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) if ( filter->local || filter->remote ) { - LogVal** vals = RecordToFilterVals(stream, filter, columns); + Value** vals = RecordToFilterVals(stream, filter, columns); if ( filter->remote ) remote_serializer->SendLogWrite(stream->id, @@ -1045,11 +1098,9 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) if ( filter->local ) { - assert(writer); - // Write takes ownership of vals. - if ( ! writer->Write(filter->num_fields, vals) ) - error = true; + assert(writer); + writer->Write(filter->num_fields, vals); } else @@ -1072,15 +1123,15 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } -LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) +Value* Manager::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) ty = val->Type(); if ( ! val ) - return new LogVal(ty->Tag(), false); + return new Value(ty->Tag(), false); - LogVal* lval = new LogVal(ty->Tag()); + Value* lval = new Value(ty->Tag()); switch ( lval->type ) { case TYPE_BOOL: @@ -1160,7 +1211,7 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) set = new ListVal(TYPE_INT); lval->val.set_val.size = set->Length(); - lval->val.set_val.vals = new LogVal* [lval->val.set_val.size]; + lval->val.set_val.vals = new Value* [lval->val.set_val.size]; for ( int i = 0; i < lval->val.set_val.size; i++ ) lval->val.set_val.vals[i] = ValToLogVal(set->Index(i)); @@ -1174,7 +1225,7 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) VectorVal* vec = val->AsVectorVal(); lval->val.vector_val.size = vec->Size(); lval->val.vector_val.vals = - new LogVal* [lval->val.vector_val.size]; + new Value* [lval->val.vector_val.size]; for ( int i = 0; i < lval->val.vector_val.size; i++ ) { @@ -1193,10 +1244,10 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) return lval; } -LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, +Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns) { - LogVal** vals = new LogVal*[filter->num_fields]; + Value** vals = new Value*[filter->num_fields]; for ( int i = 0; i < filter->num_fields; ++i ) { @@ -1215,7 +1266,7 @@ LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, if ( ! val ) { // Value, or any of its parents, is not set. - vals[i] = new LogVal(filter->fields[i]->type, false); + vals[i] = new Value(filter->fields[i]->type, false); break; } } @@ -1227,8 +1278,8 @@ LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, return vals; } -LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, LogField** fields) +WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, + int num_fields, Field** fields) { Stream* stream = FindStream(id); @@ -1244,56 +1295,10 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, // return it. return w->second->writer; - // Need to instantiate a new writer. + WriterFrontend* writer_obj = new WriterFrontend(writer->AsEnum()); + assert(writer_obj); - LogWriterDefinition* ld = log_writers; - - while ( true ) - { - if ( ld->type == BifEnum::Log::WRITER_DEFAULT ) - { - reporter->Error("unknow writer when creating writer"); - return 0; - } - - if ( ld->type == writer->AsEnum() ) - break; - - if ( ! ld->factory ) - // Oops, we can't instantiate this guy. - return 0; - - // If the writer has an init function, call it. - if ( ld->init ) - { - if ( (*ld->init)() ) - // Clear the init function so that we won't - // call it again later. - ld->init = 0; - else - // Init failed, disable by deleting factory - // function. - ld->factory = 0; - - DBG_LOG(DBG_LOGGING, "failed to init writer class %s", - ld->name); - - return false; - } - - ++ld; - } - - assert(ld->factory); - LogWriter* writer_obj = (*ld->factory)(); - - if ( ! writer_obj->Init(path, num_fields, fields) ) - { - DBG_LOG(DBG_LOGGING, "failed to init instance of writer %s", - ld->name); - - return 0; - } + writer_obj->Init(path, num_fields, fields); WriterInfo* winfo = new WriterInfo; winfo->type = writer->Ref()->AsEnumVal(); @@ -1338,16 +1343,17 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, return writer_obj; } -void LogMgr::DeleteVals(int num_fields, LogVal** vals) +void Manager::DeleteVals(int num_fields, Value** vals) { + // Note this code is duplicated in WriterBackend::DeleteVals(). for ( int i = 0; i < num_fields; i++ ) delete vals[i]; delete [] vals; } -bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, - LogVal** vals) +bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, + Value** vals) { Stream* stream = FindStream(id); @@ -1357,7 +1363,7 @@ bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, #ifdef DEBUG ODesc desc; id->Describe(&desc); - DBG_LOG(DBG_LOGGING, "unknown stream %s in LogMgr::Write()", + DBG_LOG(DBG_LOGGING, "unknown stream %s in Manager::Write()", desc.Description()); #endif DeleteVals(num_fields, vals); @@ -1379,23 +1385,24 @@ bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, #ifdef DEBUG ODesc desc; id->Describe(&desc); - DBG_LOG(DBG_LOGGING, "unknown writer %s in LogMgr::Write()", + DBG_LOG(DBG_LOGGING, "unknown writer %s in Manager::Write()", desc.Description()); #endif DeleteVals(num_fields, vals); return false; } - bool success = (w->second ? w->second->writer->Write(num_fields, vals) : true); + if ( w->second ) + w->second->writer->Write(num_fields, vals); DBG_LOG(DBG_LOGGING, - "Wrote pre-filtered record to path '%s' on stream '%s' [%s]", - path.c_str(), stream->name.c_str(), (success ? "ok" : "error")); + "Wrote pre-filtered record to path '%s' on stream '%s'", + path.c_str(), stream->name.c_str()); - return success; + return true; } -void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) +void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer) { for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) { @@ -1410,7 +1417,7 @@ void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) if ( ! i->second ) continue; - LogWriter* writer = i->second->writer; + WriterFrontend* writer = i->second->writer; EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer); remote_serializer->SendLogCreateWriter(peer, (*s)->id, @@ -1422,7 +1429,7 @@ void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) } } -bool LogMgr::SetBuf(EnumVal* id, bool enabled) +bool Manager::SetBuf(EnumVal* id, bool enabled) { Stream* stream = FindStream(id); if ( ! stream ) @@ -1440,7 +1447,7 @@ bool LogMgr::SetBuf(EnumVal* id, bool enabled) return true; } -bool LogMgr::Flush(EnumVal* id) +bool Manager::Flush(EnumVal* id) { Stream* stream = FindStream(id); if ( ! stream ) @@ -1461,7 +1468,7 @@ bool LogMgr::Flush(EnumVal* id) return true; } -void LogMgr::Error(LogWriter* writer, const char* msg) +void Manager::Error(WriterFrontend* writer, const char* msg) { reporter->Error("error with writer for %s: %s", writer->Path().c_str(), msg); @@ -1470,7 +1477,7 @@ void LogMgr::Error(LogWriter* writer, const char* msg) // Timer which on dispatching rotates the filter. class RotationTimer : public Timer { public: - RotationTimer(double t, LogMgr::WriterInfo* arg_winfo, bool arg_rotate) + RotationTimer(double t, Manager::WriterInfo* arg_winfo, bool arg_rotate) : Timer(t, TIMER_ROTATE) { winfo = arg_winfo; @@ -1482,7 +1489,7 @@ public: void Dispatch(double t, int is_expire); protected: - LogMgr::WriterInfo* winfo; + Manager::WriterInfo* winfo; bool rotate; }; @@ -1506,7 +1513,7 @@ void RotationTimer::Dispatch(double t, int is_expire) } } -void LogMgr::InstallRotationTimer(WriterInfo* winfo) +void Manager::InstallRotationTimer(WriterInfo* winfo) { if ( terminating ) return; @@ -1548,7 +1555,7 @@ void LogMgr::InstallRotationTimer(WriterInfo* winfo) } } -void LogMgr::Rotate(WriterInfo* winfo) +void Manager::Rotate(WriterInfo* winfo) { DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f", winfo->writer->Path().c_str(), network_time); @@ -1568,7 +1575,7 @@ void LogMgr::Rotate(WriterInfo* winfo) winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating); } -bool LogMgr::FinishedRotation(LogWriter* writer, string new_name, string old_name, +bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name, double open, double close, bool terminating) { DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s", diff --git a/src/LogMgr.h b/src/logging/Manager.h similarity index 68% rename from src/LogMgr.h rename to src/logging/Manager.h index 3eaba360d5..7fa2c271db 100644 --- a/src/LogMgr.h +++ b/src/logging/Manager.h @@ -2,24 +2,28 @@ // // A class managing log writers and filters. -#ifndef LOGMGR_H -#define LOGMGR_H +#ifndef LOGGING_MANAGER_H +#define LOGGING_MANAGER_H -#include "Val.h" -#include "EventHandler.h" -#include "RemoteSerializer.h" +#include "../Val.h" +#include "../EventHandler.h" +#include "../RemoteSerializer.h" class SerializationFormat; +class RemoteSerializer; +class RotationTimer; + +namespace logging { // Description of a log field. -struct LogField { +struct Field { string name; TypeTag type; // inner type of sets TypeTag subtype; - LogField() { subtype = TYPE_VOID; } - LogField(const LogField& other) + Field() { subtype = TYPE_VOID; } + Field(const Field& other) : name(other.name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. @@ -28,13 +32,13 @@ struct LogField { }; // Values as logged by a writer. -struct LogVal { +struct Value { TypeTag type; bool present; // False for unset fields. // The following union is a subset of BroValUnion, including only the // types we can log directly. - struct set_t { bro_int_t size; LogVal** vals; }; + struct set_t { bro_int_t size; Value** vals; }; typedef set_t vec_t; union _val { @@ -48,9 +52,9 @@ struct LogVal { vec_t vector_val; } val; - LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) + Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) : type(arg_type), present(arg_present) {} - ~LogVal(); + ~Value(); // (Un-)serialize. bool Read(SerializationFormat* fmt); @@ -61,17 +65,17 @@ struct LogVal { static bool IsCompatibleType(BroType* t, bool atomic_only=false); private: - LogVal(const LogVal& other) { } + Value(const Value& other) { } }; -class LogWriter; -class RemoteSerializer; -class RotationTimer; +class WriterBackend; +class WriterFrontend; +class RotationFinishedMessage; -class LogMgr { +class Manager { public: - LogMgr(); - ~LogMgr(); + Manager(); + ~Manager(); // These correspond to the BiFs visible on the scripting layer. The // actual BiFs just forward here. @@ -86,19 +90,24 @@ public: bool Flush(EnumVal* id); // Flushes all writers.. protected: - friend class LogWriter; - friend class RemoteSerializer; - friend class RotationTimer; + friend class WriterFrontend; + friend class RotationFinishedMessage; + friend class ::RemoteSerializer; + friend class ::RotationTimer; + + // Instantiates a new WriterBackend of the given type (note that + // doing so creates a new thread!). + WriterBackend* CreateBackend(bro_int_t type); //// Function also used by the RemoteSerializer. // Takes ownership of fields. - LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, LogField** fields); + WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path, + int num_fields, Field** fields); // Takes ownership of values.. bool Write(EnumVal* id, EnumVal* writer, string path, - int num_fields, LogVal** vals); + int num_fields, Value** vals); // Announces all instantiated writers to peer. void SendAllWritersTo(RemoteSerializer::PeerID peer); @@ -106,14 +115,14 @@ protected: //// Functions safe to use by writers. // Signals that a file has been rotated. - bool FinishedRotation(LogWriter* writer, string new_name, string old_name, + bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, double open, double close, bool terminating); // Reports an error for the given writer. - void Error(LogWriter* writer, const char* msg); + void Error(WriterFrontend* writer, const char* msg); // Deletes the values as passed into Write(). - void DeleteVals(int num_fields, LogVal** vals); + void DeleteVals(int num_fields, Value** vals); private: struct Filter; @@ -123,20 +132,22 @@ private: bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); - LogVal** RecordToFilterVals(Stream* stream, Filter* filter, + Value** RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns); - LogVal* ValToLogVal(Val* val, BroType* ty = 0); + Value* ValToLogVal(Val* val, BroType* ty = 0); Stream* FindStream(EnumVal* id); void RemoveDisabledWriters(Stream* stream); void InstallRotationTimer(WriterInfo* winfo); void Rotate(WriterInfo* info); Filter* FindFilter(EnumVal* id, StringVal* filter); - WriterInfo* FindWriter(LogWriter* writer); + WriterInfo* FindWriter(WriterFrontend* writer); vector streams; // Indexed by stream enum. }; -extern LogMgr* log_mgr; +} + +extern logging::Manager* log_mgr; #endif diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc new file mode 100644 index 0000000000..095490edc4 --- /dev/null +++ b/src/logging/WriterBackend.cc @@ -0,0 +1,161 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "util.h" + +#include "WriterBackend.h" +#include "WriterFrontend.h" + +// Messages sent from backend to frontend (i.e., "OutputMessages"). + +namespace logging { + +class RotationFinishedMessage : public threading::OutputMessage +{ +public: + RotationFinishedMessage(WriterFrontend* writer, string new_name, string old_name, + double open, double close, bool terminating) + : threading::OutputMessage("RotationFinished", writer), + new_name(new_name), old_name(old_name), open(open), + close(close), terminating(terminating) { } + + virtual bool Process() + { + return log_mgr->FinishedRotation(Object(), new_name, old_name, open, close, terminating); + } + +private: + string new_name; + string old_name; + double open; + double close; + bool terminating; +}; + +class DisableMessage : public threading::OutputMessage +{ +public: + DisableMessage(WriterFrontend* writer) + : threading::OutputMessage("Disable", writer) {} + + virtual bool Process() { Object()->SetDisable(); return true; } +}; + +} + +// Backend methods. + +using namespace logging; + +WriterBackend::WriterBackend(const string& name) : MsgThread(name) + { + path = ""; + num_fields = 0; + fields = 0; + buffering = true; + } + +WriterBackend::~WriterBackend() + { + if ( fields ) + { + for(int i = 0; i < num_fields; ++i) + delete fields[i]; + + delete [] fields; + } + } + +void WriterBackend::DeleteVals(Value** vals) + { + // Note this code is duplicated in Manager::DeleteVals(). + for ( int i = 0; i < num_fields; i++ ) + delete vals[i]; + + delete [] vals; + } + +bool WriterBackend::FinishedRotation(WriterFrontend* writer, string new_name, string old_name, + double open, double close, bool terminating) + { + SendOut(new RotationFinishedMessage(writer, new_name, old_name, open, close, terminating)); + return true; + } + +bool WriterBackend::Init(string arg_path, int arg_num_fields, + const Field* const * arg_fields) + { + path = arg_path; + num_fields = arg_num_fields; + fields = arg_fields; + + if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) + return false; + + return true; + } + +bool WriterBackend::Write(int arg_num_fields, Value** vals) + { + // Double-check that the arguments match. If we get this from remote, + // something might be mixed up. + if ( num_fields != arg_num_fields ) + { + +#ifdef DEBUG + const char* msg = Fmt("Number of fields don't match in WriterBackend::Write() (%d vs. %d)", + arg_num_fields, num_fields); + Debug(DBG_LOGGING, msg); +#endif + + DeleteVals(vals); + return false; + } + + for ( int i = 0; i < num_fields; ++i ) + { + if ( vals[i]->type != fields[i]->type ) + { +#ifdef DEBUG + const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)", + vals[i]->type, fields[i]->type); + Debug(DBG_LOGGING, msg); +#endif + + DeleteVals(vals); + return false; + } + } + + bool result = DoWrite(num_fields, fields, vals); + + DeleteVals(vals); + + return result; + } + +bool WriterBackend::SetBuf(bool enabled) + { + if ( enabled == buffering ) + // No change. + return true; + + buffering = enabled; + + return DoSetBuf(enabled); + } + +bool WriterBackend::Rotate(WriterFrontend* writer, string rotated_path, + double open, double close, bool terminating) + { + return DoRotate(writer, rotated_path, open, close, terminating); + } + +bool WriterBackend::Flush() + { + return DoFlush(); + } + +bool WriterBackend::Finish() + { + return DoFinish(); + } diff --git a/src/LogWriter.h b/src/logging/WriterBackend.h similarity index 70% rename from src/LogWriter.h rename to src/logging/WriterBackend.h index 1d2f9fa4b2..d1e4634e6d 100644 --- a/src/LogWriter.h +++ b/src/logging/WriterBackend.h @@ -1,32 +1,22 @@ // See the file "COPYING" in the main distribution directory for copyright. // -// Interface API for a log writer backend. The LogMgr creates a separate -// writer instance of pair of (writer type, output path). -// -// Note thay classes derived from LogWriter must be fully thread-safe and not -// use any non-thread-safe Bro functionality (which includes almost -// everything ...). In particular, do not use fmt() but LogWriter::Fmt()!. -// -// The one exception to this rule is the constructor: it is guaranteed to be -// executed inside the main thread and can thus in particular access global -// script variables. +// Bridge class between main process and writer threads. -#ifndef LOGWRITER_H -#define LOGWRITER_H +#ifndef LOGGING_WRITERBACKEND_H +#define LOGGING_WRITERBACKEND_H -#include "LogMgr.h" -#include "BroString.h" +#include "Manager.h" -class LogWriter { +#include "threading/MsgThread.h" + +namespace logging { + +// The backend runs in its own thread, separate from the main process. +class WriterBackend : public threading::MsgThread +{ public: - LogWriter(); - virtual ~LogWriter(); - - //// Interface methods to interact with the writer. Note that these - //// methods are not necessarily thread-safe and must be called only - //// from the main thread (which will typically mean only from the - //// LogMgr). In particular, they must not be called from the - //// writer's derived implementation. + WriterBackend(const string& name); + virtual ~WriterBackend(); // One-time initialization of the writer to define the logged fields. // Interpretation of "path" is left to the writer, and will be @@ -37,18 +27,18 @@ public: // // The new instance takes ownership of "fields", and will delete them // when done. - bool Init(string path, int num_fields, const LogField* const * fields); + bool Init(string path, int num_fields, const Field* const * fields); // Writes one log entry. The method takes ownership of "vals" and // will return immediately after queueing the write request, which is // potentially before output has actually been written out. // - // num_fields and the types of the LogVals must match what was passed + // num_fields and the types of the Values must match what was passed // to Init(). // // Returns false if an error occured, in which case the writer must // not be used any further. - bool Write(int num_fields, LogVal** vals); + bool Write(int num_fields, Value** vals); // Sets the buffering status for the writer, if the writer supports // that. (If not, it will be ignored). @@ -60,12 +50,12 @@ public: // Triggers rotation, if the writer supports that. (If not, it will // be ignored). - bool Rotate(string rotated_path, double open, double close, bool terminating); + bool Rotate(WriterFrontend* writer, string rotated_path, double open, double close, bool terminating); // Finishes writing to this logger regularly. Must not be called if // an error has been indicated earlier. After calling this, no // further writing must be performed. - void Finish(); + bool Finish(); //// Thread-safe methods that may be called from the writer //// implementation. @@ -73,24 +63,43 @@ public: // The following methods return the information as passed to Init(). const string Path() const { return path; } int NumFields() const { return num_fields; } - const LogField* const * Fields() const { return fields; } + const Field* const * Fields() const { return fields; } + + // Returns the current buffering state. + bool IsBuf() { return buffering; } + + // Signals to the log manager that a file has been rotated. + // + // writer: The frontend writer that triggered the rotation. This must + // be the value passed into DoRotate(). + // + // new_name: The filename of the rotated file. old_name: The filename + // of the origina file. + // + // open/close: The timestamps when the original file was opened and + // closed, respectively. + // + // terminating: True if rotation request occured due to the main Bro + // process shutting down. + bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, + double open, double close, bool terminating); protected: // Methods for writers to override. If any of these returs false, it // will be assumed that a fatal error has occured that prevents the // writer from further operation. It will then be disabled and - // deleted. When return false, the writer should also report the + // deleted. When returning false, the writer should also report the // error via Error(). Note that even if a writer does not support the // functionality for one these methods (like rotation), it must still // return true if that is not to be considered a fatal error. // // Called once for initialization of the writer. virtual bool DoInit(string path, int num_fields, - const LogField* const * fields) = 0; + const Field* const * fields) = 0; // Called once per log entry to record. - virtual bool DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals) = 0; + virtual bool DoWrite(int num_fields, const Field* const * fields, + Value** vals) = 0; // Called when the buffering status for this writer is changed. If // buffering is disabled, the writer should attempt to write out @@ -119,6 +128,11 @@ protected: // RotationDone() to signal the log manager that potential // postprocessors can now run. // + // "writer" is the frontend writer that triggered the rotation. The + // *only* purpose of this value is to be passed into + // FinishedRotation() once done. You must not otherwise access the + // frontend, it's running in a different thread. + // // "rotate_path" reflects the path to where the rotated output is to // be moved, with specifics depending on the writer. It should // generally be interpreted in a way consistent with that of "path" @@ -135,52 +149,31 @@ protected: // // A writer may ignore rotation requests if it doesn't fit with its // semantics (but must still return true in that case). - virtual bool DoRotate(string rotated_path, double open, double close, - bool terminating) = 0; + virtual bool DoRotate(WriterFrontend* writer, string rotated_path, + double open, double close, bool terminating) = 0; // Called once on termination. Not called when any of the other // methods has previously signaled an error, i.e., executing this // method signals a regular shutdown of the writer. - virtual void DoFinish() = 0; + virtual bool DoFinish() = 0; - //// Methods for writers to use. These are thread-safe. - - // A thread-safe version of fmt(). - const char* Fmt(const char* format, ...); - - // Returns the current buffering state. - bool IsBuf() { return buffering; } - - // Reports an error to the user. - void Error(const char *msg); - - // Signals to the log manager that a file has been rotated. - // - // new_name: The filename of the rotated file. old_name: The filename - // of the origina file. - // - // open/close: The timestamps when the original file was opened and - // closed, respectively. - // - // terminating: True if rotation request occured due to the main Bro - // process shutting down. - bool FinishedRotation(string new_name, string old_name, double open, - double close, bool terminating); + // Triggered by regular heartbeat messages from the main process. + virtual bool DoHeartbeat(double network_time, double current_time) { return true; }; private: - friend class LogMgr; + friend class Manager; // When an error occurs, we call this method to set a flag marking - // the writer as disabled. The LogMgr will check the flag later and + // the writer as disabled. The Manager will check the flag later and // remove the writer. bool Disabled() { return disabled; } // Deletes the values as passed into Write(). - void DeleteVals(LogVal** vals); + void DeleteVals(Value** vals); string path; int num_fields; - const LogField* const * fields; + const Field* const * fields; bool buffering; bool disabled; @@ -189,4 +182,8 @@ private: unsigned int buf_len; }; + +} + #endif + diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc new file mode 100644 index 0000000000..92c93c1c56 --- /dev/null +++ b/src/logging/WriterFrontend.cc @@ -0,0 +1,175 @@ + +#include "WriterFrontend.h" +#include "WriterBackend.h" + +namespace logging { + +// Messages sent from frontend to backend (i.e., "InputMessages"). + +class InitMessage : public threading::InputMessage +{ +public: + InitMessage(WriterBackend* backend, const string path, const int num_fields, const Field* const *fields) + : threading::InputMessage("Init", backend), + path(path), num_fields(num_fields), fields(fields) { } + + virtual bool Process() { return Object()->Init(path, num_fields, fields); } + +private: + const string path; + const int num_fields; + const Field * const* fields; +}; + +class RotateMessage : public threading::InputMessage +{ +public: + RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const string rotated_path, const double open, + const double close, const bool terminating) + : threading::InputMessage("Rotate", backend), + frontend(frontend), + rotated_path(rotated_path), open(open), + close(close), terminating(terminating) { } + + virtual bool Process() { return Object()->Rotate(frontend, rotated_path, open, close, terminating); } + +private: + WriterFrontend* frontend; + const string rotated_path; + const double open; + const double close; + const bool terminating; +}; + +class WriteMessage : public threading::InputMessage +{ +public: + WriteMessage(WriterBackend* backend, const int num_fields, Value **vals) + : threading::InputMessage("Write", backend), + num_fields(num_fields), fields(fields), vals(vals) {} + + virtual bool Process() { return Object()->Write(num_fields, vals); } + +private: + int num_fields; + Field* const* fields; + Value **vals; +}; + +class SetBufMessage : public threading::InputMessage +{ +public: + SetBufMessage(WriterBackend* backend, const bool enabled) + : threading::InputMessage("SetBuf", backend), + enabled(enabled) { } + + virtual bool Process() { return Object()->SetBuf(enabled); } + +private: + const bool enabled; +}; + +class FlushMessage : public threading::InputMessage +{ +public: + FlushMessage(WriterBackend* backend) + : threading::InputMessage("Flush", backend) {} + + virtual bool Process() { return Object()->Flush(); } +}; + +class FinishMessage : public threading::InputMessage +{ +public: + FinishMessage(WriterBackend* backend) + : threading::InputMessage("Finish", backend) {} + + virtual bool Process() { return Object()->Finish(); } +}; + +} + +// Frontend methods. + +using namespace logging; + +WriterFrontend::WriterFrontend(bro_int_t type) + { + disabled = initialized = false; + backend = log_mgr->CreateBackend(type); + + assert(backend); + backend->Start(); + } + +WriterFrontend::~WriterFrontend() + { + } + +void WriterFrontend::Stop() + { + SetDisable(); + backend->Stop(); + } + +void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* const * arg_fields) + { + if ( disabled ) + return; + + if ( initialized ) + reporter->InternalError("writer initialize twice"); + + path = arg_path; + num_fields = arg_num_fields; + fields = arg_fields; + + initialized = true; + backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + } + +void WriterFrontend::Write(int num_fields, Value** vals) + { + if ( disabled ) + return; + + backend->SendIn(new WriteMessage(backend, num_fields, vals)); + } + +void WriterFrontend::SetBuf(bool enabled) + { + if ( disabled ) + return; + + backend->SendIn(new SetBufMessage(backend, enabled)); + } + +void WriterFrontend::Flush() + { + if ( disabled ) + return; + + backend->SendIn(new FlushMessage(backend)); + } + +void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating) + { + if ( disabled ) + return; + + backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); + } + +void WriterFrontend::Finish() + { + if ( disabled ) + return; + + backend->SendIn(new FinishMessage(backend)); + } + + + + + + diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h new file mode 100644 index 0000000000..1998429d38 --- /dev/null +++ b/src/logging/WriterFrontend.h @@ -0,0 +1,66 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Bridge class between main process and writer threads. + +#ifndef LOGGING_WRITERFRONTEND_H +#define LOGGING_WRITERFRONTEND_H + +#include "Manager.h" + +#include "threading/MsgThread.h" + +namespace logging { + +class WriterBackend; + +class WriterFrontend { +public: + WriterFrontend(bro_int_t type); + virtual ~WriterFrontend(); + + // Disables the writers and stop the backend thread. + void Stop(); + + // Interface methods to interact with the writer from the main thread + // (and only from the main thread), typicalli from the log manager. + // All these methods forward (via inter-thread messaging) to the + // corresponding methods of an internally created WriterBackend. See + // there for documentation. + // + // If any of these operations fails, the writer will be automatically + // (but asynchronoulsy) disabled. + + void Init(string path, int num_fields, const Field* const * fields); + void Write(int num_fields, Value** vals); + void SetBuf(bool enabled); + void Flush(); + void Rotate(string rotated_path, double open, double close, bool terminating); + void Finish(); + + // Calling this disable the writer. All methods calls will be no-ops + // from now on. The Manager will eventually remove disabled writers. + void SetDisable() { disabled = true; } + bool Disabled() { return disabled; } + + const string Path() const { return path; } + int NumFields() const { return num_fields; } + const Field* const * Fields() const { return fields; } + +protected: + friend class Manager; + + + WriterBackend* backend; + bool disabled; + bool initialized; + + string path; + int num_fields; + const Field* const * fields; +}; + +} + + + +#endif diff --git a/src/LogWriterAscii.cc b/src/logging/writers/Ascii.cc similarity index 89% rename from src/LogWriterAscii.cc rename to src/logging/writers/Ascii.cc index d2c1d91370..70f513be3b 100644 --- a/src/LogWriterAscii.cc +++ b/src/logging/writers/Ascii.cc @@ -3,10 +3,14 @@ #include #include -#include "LogWriterAscii.h" -#include "NetVar.h" +#include "../../NetVar.h" -LogWriterAscii::LogWriterAscii() +#include "Ascii.h" + +using namespace logging; +using namespace writer; + +Ascii::Ascii() : WriterBackend("Ascii") { file = 0; @@ -42,7 +46,7 @@ LogWriterAscii::LogWriterAscii() desc.AddEscapeSequence(separator, separator_len); } -LogWriterAscii::~LogWriterAscii() +Ascii::~Ascii() { if ( file ) fclose(file); @@ -54,7 +58,7 @@ LogWriterAscii::~LogWriterAscii() delete [] header_prefix; } -bool LogWriterAscii::WriteHeaderField(const string& key, const string& val) +bool Ascii::WriteHeaderField(const string& key, const string& val) { string str = string(header_prefix, header_prefix_len) + key + string(separator, separator_len) + val + "\n"; @@ -62,8 +66,8 @@ bool LogWriterAscii::WriteHeaderField(const string& key, const string& val) return (fwrite(str.c_str(), str.length(), 1, file) == 1); } -bool LogWriterAscii::DoInit(string path, int num_fields, - const LogField* const * fields) +bool Ascii::DoInit(string path, int num_fields, + const Field* const * fields) { if ( output_to_stdout ) path = "/dev/stdout"; @@ -108,7 +112,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, types += string(separator, separator_len); } - const LogField* field = fields[i]; + const Field* field = fields[i]; names += field->name; types += type_name(field->type); if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) ) @@ -131,17 +135,18 @@ write_error: return false; } -bool LogWriterAscii::DoFlush() +bool Ascii::DoFlush() { fflush(file); return true; } -void LogWriterAscii::DoFinish() +bool Ascii::DoFinish() { + return true; } -bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) +bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) { if ( ! val->present ) { @@ -281,8 +286,8 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) return true; } -bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals) +bool Ascii::DoWrite(int num_fields, const Field* const * fields, + Value** vals) { if ( ! file ) DoInit(Path(), NumFields(), Fields()); @@ -312,8 +317,8 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, return true; } -bool LogWriterAscii::DoRotate(string rotated_path, double open, - double close, bool terminating) +bool Ascii::DoRotate(WriterFrontend* writer, string rotated_path, double open, + double close, bool terminating) { // Don't rotate special files or if there's not one currently open. if ( ! file || IsSpecial(Path()) ) @@ -325,7 +330,7 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open, string nname = rotated_path + "." + LogExt(); rename(fname.c_str(), nname.c_str()); - if ( ! FinishedRotation(nname, fname, open, close, terminating) ) + if ( ! FinishedRotation(writer, nname, fname, open, close, terminating) ) { Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str())); return false; @@ -334,13 +339,13 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open, return true; } -bool LogWriterAscii::DoSetBuf(bool enabled) +bool Ascii::DoSetBuf(bool enabled) { // Nothing to do. return true; } -string LogWriterAscii::LogExt() +string Ascii::LogExt() { const char* ext = getenv("BRO_LOG_SUFFIX"); if ( ! ext ) ext = "log"; diff --git a/src/LogWriterAscii.h b/src/logging/writers/Ascii.h similarity index 57% rename from src/LogWriterAscii.h rename to src/logging/writers/Ascii.h index 72127c8b1f..37fcfef267 100644 --- a/src/LogWriterAscii.h +++ b/src/logging/writers/Ascii.h @@ -2,33 +2,35 @@ // // Log writer for delimiter-separated ASCII logs. -#ifndef LOGWRITERASCII_H -#define LOGWRITERASCII_H +#ifndef LOGGING_WRITER_ASCII_H +#define LOGGING_WRITER_ASCII_H -#include "LogWriter.h" +#include "../WriterBackend.h" -class LogWriterAscii : public LogWriter { +namespace logging { namespace writer { + +class Ascii : public WriterBackend { public: - LogWriterAscii(); - ~LogWriterAscii(); + Ascii(); + ~Ascii(); - static LogWriter* Instantiate() { return new LogWriterAscii; } + static WriterBackend* Instantiate() { return new Ascii; } static string LogExt(); protected: virtual bool DoInit(string path, int num_fields, - const LogField* const * fields); - virtual bool DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals); + const Field* const * fields); + virtual bool DoWrite(int num_fields, const Field* const * fields, + Value** vals); virtual bool DoSetBuf(bool enabled); - virtual bool DoRotate(string rotated_path, double open, double close, - bool terminating); + virtual bool DoRotate(WriterFrontend* writer, string rotated_path, + double open, double close, bool terminating); virtual bool DoFlush(); - virtual void DoFinish(); + virtual bool DoFinish(); private: bool IsSpecial(string path) { return path.find("/dev/") == 0; } - bool DoWriteOne(ODesc* desc, LogVal* val, const LogField* field); + bool DoWriteOne(ODesc* desc, Value* val, const Field* field); bool WriteHeaderField(const string& key, const string& value); FILE* file; @@ -55,4 +57,8 @@ private: int header_prefix_len; }; +} +} + + #endif diff --git a/src/logging/writers/None.cc b/src/logging/writers/None.cc new file mode 100644 index 0000000000..e419d88a6b --- /dev/null +++ b/src/logging/writers/None.cc @@ -0,0 +1,19 @@ + +#include "None.h" + +using namespace logging; +using namespace writer; + +bool None::DoRotate(WriterFrontend* writer, string rotated_path, + double open, double close, bool terminating) + { + if ( ! FinishedRotation(writer, string("/dev/null"), Path(), open, close, terminating)) + { + Error(Fmt("error rotating %s", Path().c_str())); + return false; + } + + return true; + } + + diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h new file mode 100644 index 0000000000..9b2ab6c698 --- /dev/null +++ b/src/logging/writers/None.h @@ -0,0 +1,35 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Dummy log writer that just discards everything (but still pretends to rotate). + +#ifndef LOGGING_WRITER_NONE_H +#define LOGGING_WRITER_NONE_H + +#include "../WriterBackend.h" + +namespace logging { namespace writer { + +class None : public WriterBackend { +public: + None() : WriterBackend("None") {} + ~None() {}; + + static WriterBackend* Instantiate() { return new None; } + +protected: + virtual bool DoInit(string path, int num_fields, + const Field* const * fields) { return true; } + + virtual bool DoWrite(int num_fields, const Field* const * fields, + Value** vals) { return true; } + virtual bool DoSetBuf(bool enabled) { return true; } + virtual bool DoRotate(WriterFrontend* writer, string rotated_path, + double open, double close, bool terminating); + virtual bool DoFlush() { return true; } + virtual bool DoFinish() { return true; } +}; + +} +} + +#endif diff --git a/src/main.cc b/src/main.cc index bcc0498123..58a23e6c80 100644 --- a/src/main.cc +++ b/src/main.cc @@ -29,7 +29,6 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "Event.h" #include "File.h" #include "Reporter.h" -#include "LogMgr.h" #include "Net.h" #include "NetVar.h" #include "Var.h" @@ -48,7 +47,10 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "DPM.h" #include "BroDoc.h" #include "Brofiler.h" -#include "LogWriterAscii.h" + +#include "threading/Manager.h" +#include "logging/Manager.h" +#include "logging/writers/Ascii.h" #include "binpac_bro.h" @@ -75,7 +77,8 @@ char* writefile = 0; name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; -LogMgr* log_mgr; +logging::Manager* log_mgr = 0; +threading::Manager* thread_mgr = 0; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -197,7 +200,7 @@ void usage() fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes()); fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake()); fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); - fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", LogWriterAscii::LogExt().c_str()); + fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str()); fprintf(stderr, " $BRO_PROFILER_FILE | Output file for script execution statistics (not set)\n"); exit(1); @@ -287,6 +290,8 @@ void terminate_bro() if ( remote_serializer ) remote_serializer->LogStats(); + thread_mgr->Terminate(); + delete timer_mgr; delete dns_mgr; delete persistence_serializer; @@ -299,6 +304,7 @@ void terminate_bro() delete remote_serializer; delete dpm; delete log_mgr; + delete thread_mgr; delete reporter; } @@ -661,7 +667,9 @@ int main(int argc, char** argv) set_processing_status("INITIALIZING", "main"); bro_start_time = current_time(true); + reporter = new Reporter(); + thread_mgr = new threading::Manager(); #ifdef DEBUG if ( debug_streams ) @@ -727,7 +735,7 @@ int main(int argc, char** argv) persistence_serializer = new PersistenceSerializer(); remote_serializer = new RemoteSerializer(); event_registry = new EventRegistry(); - log_mgr = new LogMgr(); + log_mgr = new logging::Manager(); if ( events_file ) event_player = new EventPlayer(events_file); @@ -1001,6 +1009,8 @@ int main(int argc, char** argv) have_pending_timers = ! reading_traces && timer_mgr->Size() > 0; + io_sources.Register(thread_mgr, true); + if ( io_sources.Size() > 0 || have_pending_timers ) { if ( profiling_logger ) diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc new file mode 100644 index 0000000000..273a192de3 --- /dev/null +++ b/src/threading/BasicThread.cc @@ -0,0 +1,129 @@ + +#include +#include + +#include "BasicThread.h" +#include "Manager.h" + +using namespace threading; + +BasicThread::BasicThread(const string& arg_name) + { + started = false; + terminating = false; + pthread = 0; + + buf = 0; + buf_len = 1024; + + char tmp[128]; + snprintf(tmp, sizeof(tmp), "%s@%p", arg_name.c_str(), this); + name = string(tmp); + + thread_mgr->AddThread(this); + } + +BasicThread::~BasicThread() + { + } + +const char* BasicThread::Fmt(const char* format, ...) + { + if ( ! buf ) + buf = (char*) malloc(buf_len); + + va_list al; + va_start(al, format); + int n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + + if ( (unsigned int) n >= buf_len ) + { // Not enough room, grow the buffer. + buf_len = n + 32; + buf = (char*) realloc(buf, buf_len); + + // Is it portable to restart? + va_start(al, format); + n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + } + + return buf; + } + +void BasicThread::Start() + { + if ( sem_init(&terminate, 0, 0) != 0 ) + reporter->FatalError("Cannot create terminate semaphore for thread %s", name.c_str()); + + if ( pthread_create(&pthread, 0, BasicThread::launcher, this) != 0 ) + reporter->FatalError("Cannot create thread %s", name.c_str()); + + DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str()); + + started = true; + + OnStart(); + } + +void BasicThread::Stop() + { + if ( ! started ) + return; + + if ( terminating ) + return; + + DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str()); + + // Signal that it's ok for the thread to exit now. + if ( sem_post(&terminate) != 0 ) + reporter->FatalError("Failure flagging terminate condition for thread %s", name.c_str()); + + terminating = true; + + OnStop(); + } + +void BasicThread::Join() + { + if ( ! started ) + return; + + if ( ! terminating ) + Stop(); + + DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str()); + + if ( pthread_join(pthread, 0) != 0 ) + reporter->FatalError("Failure joining thread %s", name.c_str()); + + sem_destroy(&terminate); + + DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str()); + + pthread = 0; + } + +void* BasicThread::launcher(void *arg) + { + BasicThread* thread = (BasicThread *)arg; + + // Block signals in thread. We handle signals only in the main + // process. + sigset_t mask_set; + sigfillset(&mask_set); + int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0); + assert(res == 0); // + + // Run thread's main function. + thread->Run(); + + // Wait until somebody actually wants us to terminate. + + if ( sem_wait(&thread->terminate) != 0 ) + reporter->FatalError("Failure flagging terminate condition for thread %s", thread->Name().c_str()); + + return 0; + } + diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h new file mode 100644 index 0000000000..30a11b4505 --- /dev/null +++ b/src/threading/BasicThread.h @@ -0,0 +1,63 @@ + +#ifndef THREADING_BASICTHREAD_H +#define THREADING_BASICTHREAD_H + +#include +#include + +#include "Queue.h" +#include "util.h" + +using namespace std; + +namespace threading { + +class Manager; + +class BasicThread +{ +public: + BasicThread(const string& name); // Managed by manager, must not delete otherwise. + virtual ~BasicThread(); + + const string& Name() const { return name; } + + void Start(); // Spawns the thread and enters Run(). + void Stop(); // Signals the thread to terminate. + + bool Terminating() const { return terminating; } + + // A thread-safe version of fmt(). + const char* Fmt(const char* format, ...); + +protected: + virtual void Run() = 0; + + virtual void OnStart() {} + virtual void OnStop() {} + +private: + friend class Manager; + + static void* launcher(void *arg); + + // Used from the ThreadMgr. + void Join(); // Waits until the thread has terminated and then joins it. + + bool started; // Set to to true once running. + bool terminating; // Set to to true to signal termination. + string name; + + pthread_t pthread; + sem_t terminate; + + // For implementing Fmt(). + char* buf; + unsigned int buf_len; +}; + +} + +extern threading::Manager* thread_mgr; + +#endif diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc new file mode 100644 index 0000000000..ed4d9cf623 --- /dev/null +++ b/src/threading/Manager.cc @@ -0,0 +1,104 @@ + +#include "Manager.h" + +using namespace threading; + +Manager::Manager() + { + DBG_LOG(DBG_THREADING, "Creating thread manager ..."); + + did_process = false; + next_beat = 0; + } + +Manager::~Manager() + { + if ( all_threads.size() ) + Terminate(); + } + +void Manager::Terminate() + { + DBG_LOG(DBG_THREADING, "Terminating thread manager ..."); + + // First process remaining thread output for the message threads. + do Process(); while ( did_process ); + + // Signal all to stop. + for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ ) + (*i)->Stop(); + + // Then join them all. + for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ ) + { + (*i)->Join(); + delete *i; + } + + all_threads.clear(); + msg_threads.clear(); + } + +void Manager::AddThread(BasicThread* thread) + { + DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str()); + all_threads.push_back(thread); + } + +void Manager::AddMsgThread(MsgThread* thread) + { + DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name().c_str()); + msg_threads.push_back(thread); + } + +void Manager::GetFds(int* read, int* write, int* except) + { + } + +double Manager::NextTimestamp(double* network_time) + { + if ( did_process || ! next_beat == 0 ) + // If we had something to process last time (or haven't had a + // chance to check yet), we want to check for more asap. + return timer_mgr->Time(); + + // Else we assume we don't have much to do at all and wait for the next heart beat. + return next_beat; + } + +void Manager::Process() + { + bool do_beat = (next_beat == 0 || network_time >= next_beat); + + did_process = false; + + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + { + MsgThread* t = *i; + + if ( do_beat ) + t->Heartbeat(); + + if ( ! t->HasOut() ) + continue; + + Message* msg = t->RetrieveOut(); + + if ( msg->Process() ) + did_process = true; + + else + { + string s = msg->Name() + " failed, terminating thread"; + reporter->Error(s.c_str()); + t->Stop(); + } + + delete msg; + } + + if ( do_beat ) + next_beat = network_time + HEART_BEAT_INTERVAL; + } + + diff --git a/src/threading/Manager.h b/src/threading/Manager.h new file mode 100644 index 0000000000..aa7292ee81 --- /dev/null +++ b/src/threading/Manager.h @@ -0,0 +1,52 @@ + +#ifndef THREADING_MANAGER_H +#define THREADING_MANAGER_H + +#include + +#include "IOSource.h" + +#include "BasicThread.h" +#include "MsgThread.h" + +namespace threading { + +class Manager : public IOSource +{ +public: + Manager(); + ~Manager(); + + void Terminate(); + +protected: + friend class BasicThread; + friend class MsgThread; + + void AddThread(BasicThread* thread); + void AddMsgThread(MsgThread* thread); + + // IOSource interface. + virtual void GetFds(int* read, int* write, int* except); + virtual double NextTimestamp(double* network_time); + virtual void Process(); + virtual const char* Tag() { return "threading::Manager"; } + +private: + static const int HEART_BEAT_INTERVAL = 1; + + typedef std::list all_thread_list; + all_thread_list all_threads; + + typedef std::list msg_thread_list; + msg_thread_list msg_threads; + + bool did_process; + double next_beat; +}; + +} + +extern threading::Manager* thread_mgr; + +#endif diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc new file mode 100644 index 0000000000..e2d81cf47f --- /dev/null +++ b/src/threading/MsgThread.cc @@ -0,0 +1,285 @@ + +#include "DebugLogger.h" + +#include "MsgThread.h" +#include "Manager.h" + +using namespace threading; + +static void strreplace(const string& s, const string& o, const string& n) + { + string r = s; + + while ( true ) + { + size_t i = r.find(o); + + if ( i == std::string::npos ) + break; + + r.replace(i, o.size(), n); + } + } + +namespace threading { + +// Standard messages. + +class TerminateMessage : public InputMessage +{ +public: + TerminateMessage(MsgThread* thread) : InputMessage("Terminate", thread) { } + + virtual bool Process() { return true; } +}; + +class ReporterMessage : public OutputMessage +{ +public: + enum Type { + INFO, WARNING, ERROR, FATAL_ERROR, FATAL_ERROR_WITH_CORE, + INTERNAL_WARNING, INTERNAL_ERROR + }; + + ReporterMessage(Type arg_type, MsgThread* thread, const string& arg_msg) + : OutputMessage("ReporterMessage", thread) + { type = arg_type; msg = arg_msg; } + + virtual bool Process(); + +private: + string msg; + Type type; +}; + +class HeartbeatMessage : public InputMessage +{ +public: + HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time) + : InputMessage("Heartbeat", thread) + { network_time = arg_network_time; current_time = arg_current_time; } + + virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); } + +private: + double network_time; + double current_time; +}; + +#ifdef DEBUG +class DebugMessage : public OutputMessage +{ +public: + DebugMessage(DebugStream arg_stream, MsgThread* thread, const string& arg_msg) + : OutputMessage("DebugMessage", thread) + { stream = arg_stream; msg = arg_msg; } + + virtual bool Process() + { + string s = Object()->Name() + ": " + msg; + strreplace(s, "%", "%%"); + debug_logger.Log(stream, s.c_str()); + return true; + } +private: + string msg; + DebugStream stream; +}; +#endif + +} + +// Methods. + +Message::~Message() + { + } + +bool ReporterMessage::Process() + { + string s = Object()->Name() + ": " + msg; + strreplace(s, "%", "%%"); + + const char* cmsg = s.c_str(); + + switch ( type ) { + + case INFO: + reporter->Info(cmsg); + break; + + case WARNING: + reporter->Warning(cmsg); + break; + + case ERROR: + reporter->Error(cmsg); + break; + + case FATAL_ERROR: + reporter->FatalError(cmsg); + break; + + case FATAL_ERROR_WITH_CORE: + reporter->FatalErrorWithCore(cmsg); + break; + + case INTERNAL_WARNING: + reporter->InternalWarning(cmsg); + break; + + case INTERNAL_ERROR : + reporter->InternalError(cmsg); + break; + + default: + reporter->InternalError("unknown ReporterMessage type %d", type); + } + + return true; + } + +MsgThread::MsgThread(const string& name) : BasicThread(name) + { + cnt_sent_in = cnt_sent_out = 0; + thread_mgr->AddMsgThread(this); + } + +void MsgThread::OnStop() + { + // This is to unblock the current queue read operation. + SendIn(new TerminateMessage(this), true); + } + +void MsgThread::Heartbeat() + { + SendIn(new HeartbeatMessage(this, network_time, current_time())); + } + +void MsgThread::Info(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::INFO, this, msg)); + } + +void MsgThread::Warning(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::WARNING, this, msg)); + } + +void MsgThread::Error(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::ERROR, this, msg)); + } + +void MsgThread::FatalError(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::FATAL_ERROR, this, msg)); + } + +void MsgThread::FatalErrorWithCore(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::FATAL_ERROR_WITH_CORE, this, msg)); + } + +void MsgThread::InternalWarning(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::INTERNAL_WARNING, this, msg)); + } + +void MsgThread::InternalError(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::INTERNAL_ERROR, this, msg)); + } + +#ifdef DEBUG + +void MsgThread::Debug(DebugStream stream, const char* msg) + { + SendOut(new DebugMessage(stream, this, msg)); + } + +#endif + +void MsgThread::SendIn(BasicInputMessage* msg, bool force) + { + if ( Terminating() && ! force ) + return; + + DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str()); + + queue_in.Put(msg); + ++cnt_sent_in; + } + + +void MsgThread::SendOut(BasicOutputMessage* msg, bool force) + { + if ( Terminating() && ! force ) + return; + + queue_out.Put(msg); + ++cnt_sent_out; + } + +BasicOutputMessage* MsgThread::RetrieveOut() + { + BasicOutputMessage* msg = queue_out.Get(); + assert(msg); + +#ifdef DEBUG + if ( msg->Name() != "DebugMessage" ) // Avoid recursion. + { + string s = Fmt("Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str()); + Debug(DBG_THREADING, s.c_str()); + } +#endif + + return msg; + } + +BasicInputMessage* MsgThread::RetrieveIn() + { + BasicInputMessage* msg = queue_in.Get(); + assert(msg); + +#ifdef DEBUG + string s = Fmt("Retrieved '%s' in %s", msg->Name().c_str(), Name().c_str()); + Debug(DBG_THREADING, s.c_str()); +#endif + + return msg; + } + +void MsgThread::Run() + { + while ( true ) + { + // When requested to terminate, we only do so when + // all input has been processed. + if ( Terminating() && ! queue_in.Ready() ) + break; + + BasicInputMessage* msg = RetrieveIn(); + + bool result = msg->Process(); + + if ( ! result ) + { + string s = msg->Name() + " failed, terminating thread"; + Error(s.c_str()); + Stop(); + break; + } + + delete msg; + } + } + +void MsgThread::GetStats(Stats* stats) + { + stats->sent_in = cnt_sent_in; + stats->sent_out = cnt_sent_out; + stats->pending_in = cnt_sent_in - queue_in.Size(); + stats->pending_out = cnt_sent_out - queue_out.Size(); + } + diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h new file mode 100644 index 0000000000..2e976c1773 --- /dev/null +++ b/src/threading/MsgThread.h @@ -0,0 +1,157 @@ + +#ifndef THREADING_MSGTHREAD_H +#define THREADING_MSGTHREAD_H + +#include + +#include "DebugLogger.h" + +#include "BasicThread.h" +#include "Queue.h" + +namespace threading { + +class BasicInputMessage; +class BasicOutputMessage; +class HeartbeatMessage; + +class MsgThread : public BasicThread +{ +public: + MsgThread(const string& name); + + void SendIn(BasicInputMessage* msg) { return SendIn(msg, false); } + void SendOut(BasicOutputMessage* msg) { return SendOut(msg, false); } + + BasicOutputMessage* RetrieveOut(); + + // Report an informational message, nothing that needs specific + // attention. + void Info(const char* msg); + + // Report a warning that may indicate a problem. + void Warning(const char* msg); + + // Report a non-fatal error. Processing proceeds normally after the error + // has been reported. + void Error(const char* msg); + + // Report a fatal error. Bro will terminate after the message has been + // reported. + void FatalError(const char* msg); + + // Report a fatal error. Bro will terminate after the message has been + // reported and always generate a core dump. + void FatalErrorWithCore(const char* msg); + + // Report about a potential internal problem. Bro will continue + // normally. + void InternalWarning(const char* msg); + + // Report an internal program error. Bro will terminate with a core + // dump after the message has been reported. + void InternalError(const char* msg); + +#ifdef DEBUG + // Records a debug message for the given stream. + void Debug(DebugStream stream, const char* msg); +#endif + + void Heartbeat(); + + struct Stats + { + uint64_t sent_in; + uint64_t sent_out; + uint64_t pending_in; + uint64_t pending_out; + }; + + void GetStats(Stats* stats); + +protected: + friend class HeartbeatMessage; + + virtual void Run(); + virtual void OnStop(); + + virtual bool DoHeartbeat(double network_time, double current_time) { return true; } + +private: + friend class Manager; + + BasicInputMessage* RetrieveIn(); + + void SendIn(BasicInputMessage* msg, bool force); + void SendOut(BasicOutputMessage* msg, bool force); + + bool HasIn() { return queue_in.Ready(); } + bool HasOut() { return queue_out.Ready(); } + + Queue_ queue_in; + Queue_ queue_out; + + uint64_t cnt_sent_in; + uint64_t cnt_sent_out; +}; + +class Message +{ +public: + virtual ~Message(); + + const string& Name() const { return name; } + + virtual bool Process() = 0; // Thread will be terminated if returngin false. + +protected: + Message(const string& arg_name) { name = arg_name; } + +private: + string name; +}; + +class BasicInputMessage : public Message +{ +protected: + BasicInputMessage(const string& name) : Message(name) {} +}; + +class BasicOutputMessage : public Message +{ +protected: + BasicOutputMessage(const string& name) : Message(name) {} +}; + +template +class InputMessage : public BasicInputMessage +{ +public: + O* Object() const { return object; } + +protected: + InputMessage(const string& name, O* arg_object) : BasicInputMessage(name) + { object = arg_object; } + +private: + O* object; +}; + +template +class OutputMessage : public BasicOutputMessage +{ +public: + O* Object() const { return object; } + +protected: + OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name) + { object = arg_object; } + +private: + O* object; +}; + +} + + +#endif diff --git a/src/threading/Queue.h b/src/threading/Queue.h new file mode 100644 index 0000000000..49859dc051 --- /dev/null +++ b/src/threading/Queue.h @@ -0,0 +1,150 @@ + +#ifndef THREADING_QUEUE_H +#define THREADING_QUEUE_H + +#include +#include +#include +#include + +#include "Reporter.h" + +namespace threading { + +/** + * Just a simple threaded queue wrapper class. Uses multiple queues and reads / writes in rotary fashion in an attempt to limit contention. + * Due to locking granularity, bulk put / get is no faster than single put / get as long as FIFO guarantee is required. + */ + +template +class Queue_ +{ +public: + Queue_(); + ~Queue_(); + + T Get(); + void Put(T data); + bool Ready(); + uint64_t Size(); + +private: + static const int NUM_QUEUES = 8; + + pthread_mutex_t mutex[NUM_QUEUES]; // Mutex protected shared accesses. + pthread_cond_t has_data[NUM_QUEUES]; // Signals when data becomes available + std::queue messages[NUM_QUEUES]; // Actually holds the queued messages + + int read_ptr; // Where the next operation will read from + int write_ptr; // Where the next operation will write to + uint64_t size; +}; + +inline static void safe_lock(pthread_mutex_t* mutex) + { + if ( pthread_mutex_lock(mutex) != 0 ) + reporter->FatalErrorWithCore("cannot lock mutex"); + } + +inline static void safe_unlock(pthread_mutex_t* mutex) + { + if ( pthread_mutex_unlock(mutex) != 0 ) + reporter->FatalErrorWithCore("cannot unlock mutex"); + } + +template +inline Queue_::Queue_() + { + read_ptr = 0; + write_ptr = 0; + + for( int i = 0; i < NUM_QUEUES; ++i ) + { + if ( pthread_cond_init(&has_data[i], NULL) != 0 ) + reporter->FatalError("cannot init queue condition variable"); + + if ( pthread_mutex_init(&mutex[i], NULL) != 0 ) + reporter->FatalError("cannot init queue mutex"); + } + } + +template +inline Queue_::~Queue_() + { + for( int i = 0; i < NUM_QUEUES; ++i ) + { + pthread_cond_destroy(&has_data[i]); + pthread_mutex_destroy(&mutex[i]); + } + } + +template +inline T Queue_::Get() + { + safe_lock(&mutex[read_ptr]); + + int old_read_ptr = read_ptr; + + if ( messages[read_ptr].empty() ) + pthread_cond_wait(&has_data[read_ptr], &mutex[read_ptr]); + + T data = messages[read_ptr].front(); + messages[read_ptr].pop(); + --size; + + read_ptr = (read_ptr + 1) % NUM_QUEUES; + + safe_unlock(&mutex[old_read_ptr]); + + return data; + } + +template +inline void Queue_::Put(T data) + { + safe_lock(&mutex[write_ptr]); + + int old_write_ptr = write_ptr; + + bool need_signal = messages[write_ptr].empty(); + + messages[write_ptr].push(data); + ++size; + + if ( need_signal ) + pthread_cond_signal(&has_data[write_ptr]); + + write_ptr = (write_ptr + 1) % NUM_QUEUES; + + safe_unlock(&mutex[old_write_ptr]); + } + + +template +inline bool Queue_::Ready() + { + safe_lock(&mutex[read_ptr]); + + bool ret = (messages[read_ptr].size()); + + safe_unlock(&mutex[read_ptr]); + + return ret; + } + +template +inline uint64_t Queue_::Size() + { + safe_lock(&mutex[read_ptr]); + + uint64_t s = size; + + safe_unlock(&mutex[read_ptr]); + + return s; + } + +} + +#endif + From 6cc29a78328cf463a4706e36b77d63ba256b9bd6 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 30 Jan 2012 12:12:14 -0800 Subject: [PATCH 05/42] make logging with threads compile on mac os and fix a couple of string literal warnings. --- src/net_util.cc | 2 +- src/threading/Manager.cc | 2 +- src/threading/MsgThread.cc | 20 +++++++++----------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/net_util.cc b/src/net_util.cc index c0bacc98b2..ebe0392e2a 100644 --- a/src/net_util.cc +++ b/src/net_util.cc @@ -322,7 +322,7 @@ const uint32* mask_addr(const uint32* a, uint32 top_bits_to_keep) if ( top_bits_to_keep == 0 || top_bits_to_keep > max_bits ) { - reporter->Error("bad address mask value %s", top_bits_to_keep); + reporter->Error("bad address mask value %u", top_bits_to_keep); return addr; } diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index ed4d9cf623..d963876755 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -90,7 +90,7 @@ void Manager::Process() else { string s = msg->Name() + " failed, terminating thread"; - reporter->Error(s.c_str()); + reporter->Error("%s", s.c_str()); t->Stop(); } diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index e2d81cf47f..d78c7533a3 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -28,7 +28,7 @@ namespace threading { class TerminateMessage : public InputMessage { public: - TerminateMessage(MsgThread* thread) : InputMessage("Terminate", thread) { } + TerminateMessage(MsgThread* thread) : InputMessage("Terminate", thread) { } virtual bool Process() { return true; } }; @@ -56,7 +56,7 @@ class HeartbeatMessage : public InputMessage { public: HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time) - : InputMessage("Heartbeat", thread) + : InputMessage("Heartbeat", thread) { network_time = arg_network_time; current_time = arg_current_time; } virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); } @@ -98,38 +98,36 @@ Message::~Message() bool ReporterMessage::Process() { string s = Object()->Name() + ": " + msg; - strreplace(s, "%", "%%"); - const char* cmsg = s.c_str(); switch ( type ) { case INFO: - reporter->Info(cmsg); + reporter->Info("%s", cmsg); break; case WARNING: - reporter->Warning(cmsg); + reporter->Warning("%s", cmsg); break; case ERROR: - reporter->Error(cmsg); + reporter->Error("%s", cmsg); break; case FATAL_ERROR: - reporter->FatalError(cmsg); + reporter->FatalError("%s", cmsg); break; case FATAL_ERROR_WITH_CORE: - reporter->FatalErrorWithCore(cmsg); + reporter->FatalErrorWithCore("%s", cmsg); break; case INTERNAL_WARNING: - reporter->InternalWarning(cmsg); + reporter->InternalWarning("%s", cmsg); break; case INTERNAL_ERROR : - reporter->InternalError(cmsg); + reporter->InternalError("%s", cmsg); break; default: From a428645b2a88f20c6e9f48c573e88b74c3c03398 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 31 Jan 2012 23:47:33 -0800 Subject: [PATCH 06/42] Documenting the threading/* classes. Also switching from semaphores to mutexes as the former don't seem to be fully supported on MacOS. --- src/threading/BasicThread.cc | 29 ++-- src/threading/BasicThread.h | 119 ++++++++++++--- src/threading/Manager.h | 63 +++++++- src/threading/MsgThread.cc | 56 +++---- src/threading/MsgThread.h | 282 +++++++++++++++++++++++++++++++---- src/threading/Queue.h | 56 +++++-- 6 files changed, 499 insertions(+), 106 deletions(-) diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 273a192de3..f7bd2afbcd 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -7,6 +7,8 @@ using namespace threading; +uint64_t BasicThread::thread_counter = 0; + BasicThread::BasicThread(const string& arg_name) { started = false; @@ -16,9 +18,7 @@ BasicThread::BasicThread(const string& arg_name) buf = 0; buf_len = 1024; - char tmp[128]; - snprintf(tmp, sizeof(tmp), "%s@%p", arg_name.c_str(), this); - name = string(tmp); + name = Fmt("%s@%d", arg_name.c_str(), ++thread_counter); thread_mgr->AddThread(this); } @@ -53,8 +53,15 @@ const char* BasicThread::Fmt(const char* format, ...) void BasicThread::Start() { - if ( sem_init(&terminate, 0, 0) != 0 ) - reporter->FatalError("Cannot create terminate semaphore for thread %s", name.c_str()); + if ( started ) + return; + + if ( pthread_mutex_init(&terminate, 0) != 0 ) + reporter->FatalError("Cannot create terminate mutex for thread %s", name.c_str()); + + // We use this like a binary semaphore and acquire it immediately. + if ( pthread_mutex_lock(&terminate) != 0 ) + reporter->FatalError("Cannot aquire terminate mutex for thread %s", name.c_str()); if ( pthread_create(&pthread, 0, BasicThread::launcher, this) != 0 ) reporter->FatalError("Cannot create thread %s", name.c_str()); @@ -76,8 +83,9 @@ void BasicThread::Stop() DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str()); - // Signal that it's ok for the thread to exit now. - if ( sem_post(&terminate) != 0 ) + // Signal that it's ok for the thread to exit now by unlocking the + // mutex. + if ( pthread_mutex_unlock(&terminate) != 0 ) reporter->FatalError("Failure flagging terminate condition for thread %s", name.c_str()); terminating = true; @@ -98,7 +106,7 @@ void BasicThread::Join() if ( pthread_join(pthread, 0) != 0 ) reporter->FatalError("Failure joining thread %s", name.c_str()); - sem_destroy(&terminate); + pthread_mutex_destroy(&terminate); DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str()); @@ -120,9 +128,8 @@ void* BasicThread::launcher(void *arg) thread->Run(); // Wait until somebody actually wants us to terminate. - - if ( sem_wait(&thread->terminate) != 0 ) - reporter->FatalError("Failure flagging terminate condition for thread %s", thread->Name().c_str()); + if ( pthread_mutex_lock(&thread->terminate) != 0 ) + reporter->FatalError("Failure acquiring terminate mutex at end of thread %s", thread->Name().c_str()); return 0; } diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index 30a11b4505..df5665c464 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -14,50 +14,133 @@ namespace threading { class Manager; +/** + * Base class for all threads. + * + * This class encapsulates all the OS-level thread handling. All thread + * instances are automatically added to the threading::Manager for management. The + * manager also takes care of deleting them (which must not be done + * manually). + */ class BasicThread { public: + /** + * Creates a new thread object. Instantiating the object does however + * not yet start the actual OS thread, that requires calling Start(). + * + * Only Bro's main thread may create new thread instances. + * + * @param name A descriptive name for thread the thread. This may + * show up in messages to the user. + */ BasicThread(const string& name); // Managed by manager, must not delete otherwise. - virtual ~BasicThread(); + /** + * Returns a descriptive name for the thread. This is the name passed + * into the constructor. + * + * This method is safe to call from any thread. + */ const string& Name() const { return name; } - void Start(); // Spawns the thread and enters Run(). - void Stop(); // Signals the thread to terminate. + /** + * Starts the thread. Calling this methods will spawn a new OS thread + * executing Run(). Note that one can't restart a thread after a + * Stop(), doing so will be ignored. + * + * Only Bro's main thread must call this method. + */ + void Start(); + /** + * Signals the thread to stop. The method lets Terminating() now + * return true. It does however not force the thread to terminate. + * It's up to the Run() method to to query Terminating() and exit + * eventually. + * + * Calling this method has no effect if Start() hasn't been executed + * yet. + * + * Only Bro's main thread must call this method. + */ + void Stop(); + + /** + * Returns true if Terminate() has been called. + * + * This method is safe to call from any thread. + */ bool Terminating() const { return terminating; } - // A thread-safe version of fmt(). + /** + * A version of fmt() that the thread can safely use. + * + * This is safe to call from Run() but must not be used from any + * other thread than the current one. + */ const char* Fmt(const char* format, ...); protected: - virtual void Run() = 0; - - virtual void OnStart() {} - virtual void OnStop() {} - -private: friend class Manager; + /** + * Entry point for the thread. This must be overridden by derived + * classes and will execute in a separate thread once Start() is + * called. The thread will not terminate before this method finishes. + * An implementation should regularly check Terminating() to see if + * exiting has been requested. + */ + virtual void Run() = 0; + + /** + * Executed with Start(). This is a hook into starting the thread. It + * will be called from Bro's main thread after the OS thread has been + * started. + */ + virtual void OnStart() {} + + /** + * Executed with Stop(). This is a hook into stopping the thread. It + * will be called from Bro's main thread after the thread has been + * signaled to stop. + */ + virtual void OnStop() {} + + /** + * Destructor. This will be called by the manager. + * + * Only Bro's main thread may delete thread instances. + * + */ + virtual ~BasicThread(); + + /** + * Waits until the thread's Run() method has finished and then joins + * it. This is called from the threading::Manager. + */ + void Join(); + +private: + // pthread entry function. static void* launcher(void *arg); - // Used from the ThreadMgr. - void Join(); // Waits until the thread has terminated and then joins it. - + string name; + pthread_t pthread; bool started; // Set to to true once running. bool terminating; // Set to to true to signal termination. - string name; - pthread_t pthread; - sem_t terminate; + // Used as a semaphore to tell the pthread thread when it may + // terminate. + pthread_mutex_t terminate; // For implementing Fmt(). char* buf; unsigned int buf_len; + + static uint64_t thread_counter; }; } -extern threading::Manager* thread_mgr; - #endif diff --git a/src/threading/Manager.h b/src/threading/Manager.h index aa7292ee81..2c4f88fa1e 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -11,25 +11,78 @@ namespace threading { +/** + * The thread manager coordinates all child threads. Once a BasicThread is + * instantitated, it gets addedd to the manager, which will delete it later + * once it has terminated. + * + * In addition to basic threads, the manager also provides additional + * functionality specific to MsgThread instances. In particular, it polls + * their outgoing message queue on a regular basis and feeds data sent into + * the rest of Bro. It also triggers the regular heartbeats. + */ class Manager : public IOSource { public: + /** + * Constructor. Only a single instance of the manager must be + * created. + */ Manager(); + + /** + * Destructir. + */ ~Manager(); + /** + * Terminates the manager's processor. The method signals all threads + * to terminates and wait for them to do so. It then joins them and + * returns to the caller. Afterwards, no more thread instances may be + * created. + */ void Terminate(); protected: friend class BasicThread; friend class MsgThread; + /** + * Registers a new basic thread with the manager. This is + * automatically called by the thread's constructor. + * + * @param thread The thread. + */ void AddThread(BasicThread* thread); + + /** + * Registers a new message thread with the manager. This is + * automatically called by the thread's constructor. This must be + * called \a in \a addition to AddThread(BasicThread* thread). The + * MsgThread constructor makes sure to do so. + * + * @param thread The thread. + */ void AddMsgThread(MsgThread* thread); - // IOSource interface. + /** + * Part of the IOSource interface. + */ virtual void GetFds(int* read, int* write, int* except); + + /** + * Part of the IOSource interface. + */ virtual double NextTimestamp(double* network_time); + + /** + * Part of the IOSource interface. + */ virtual void Process(); + + /** + * Part of the IOSource interface. + */ virtual const char* Tag() { return "threading::Manager"; } private: @@ -41,12 +94,16 @@ private: typedef std::list msg_thread_list; msg_thread_list msg_threads; - bool did_process; - double next_beat; + bool did_process; // True if the last Process() found some work to do. + double next_beat; // Timestamp when the next heartbeat will be sent. }; } +/** + * A singleton instance of the thread manager. All methods must only be + * called from Bro's main thread. + */ extern threading::Manager* thread_mgr; #endif diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index d78c7533a3..455c177df6 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -6,25 +6,13 @@ using namespace threading; -static void strreplace(const string& s, const string& o, const string& n) - { - string r = s; - - while ( true ) - { - size_t i = r.find(o); - - if ( i == std::string::npos ) - break; - - r.replace(i, o.size(), n); - } - } - namespace threading { -// Standard messages. +////// Messages. +// Signals child thread to terminate. This is actually a no-op; its only +// purpose is unblock the current read operation so that the child's Run() +// methods can check the termination status. class TerminateMessage : public InputMessage { public: @@ -33,6 +21,22 @@ public: virtual bool Process() { return true; } }; +/// Sends a heartbeat to the child thread. +class HeartbeatMessage : public InputMessage +{ +public: + HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time) + : InputMessage("Heartbeat", thread) + { network_time = arg_network_time; current_time = arg_current_time; } + + virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); } + +private: + double network_time; + double current_time; +}; + +// A message from the child to be passed on to the Reporter. class ReporterMessage : public OutputMessage { public: @@ -52,21 +56,8 @@ private: Type type; }; -class HeartbeatMessage : public InputMessage -{ -public: - HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time) - : InputMessage("Heartbeat", thread) - { network_time = arg_network_time; current_time = arg_current_time; } - - virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); } - -private: - double network_time; - double current_time; -}; - #ifdef DEBUG +// A debug message from the child to be passed on to the DebugLogger. class DebugMessage : public OutputMessage { public: @@ -77,8 +68,7 @@ public: virtual bool Process() { string s = Object()->Name() + ": " + msg; - strreplace(s, "%", "%%"); - debug_logger.Log(stream, s.c_str()); + debug_logger.Log(stream, "%s", s.c_str()); return true; } private: @@ -89,7 +79,7 @@ private: } -// Methods. +////// Methods. Message::~Message() { diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 2e976c1773..8f37041bb6 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -15,121 +15,332 @@ class BasicInputMessage; class BasicOutputMessage; class HeartbeatMessage; +/** + * A specialized thread that provides bi-directional message passing between + * Bro's main thread and the child thread. Messages are instances of + * BasicInputMessage and BasicOutputMessage for message sent \a to the child + * thread and received \a from the child thread, respectively. + * + * The thread's Run() method implements main loop that processes incoming + * messages until Terminating() indicates that execution should stop. Once + * that happens, the thread stops accepting any new messages, finishes + * processes all remaining ones still in the queue, and then exits. + */ class MsgThread : public BasicThread { public: + /** + * Constructor. It automatically registers the thread with the + * threading::Manager. + * + * Only Bro's main thread may instantiate a new thread. + * + * @param name A descriptive name. This is passed on to BasicThread(). + */ MsgThread(const string& name); + /** + * Sends a message to the child thread. The message will be proceesed + * once the thread has retrieved it from its incoming queue. + * + * Only the main thread may call this method. + * + * @param msg The message. + */ void SendIn(BasicInputMessage* msg) { return SendIn(msg, false); } + + /** + * Sends a message from the child thread to the main thread. + * + * Only the child thread may call this method. + * + * @param msg The mesasge. + */ void SendOut(BasicOutputMessage* msg) { return SendOut(msg, false); } - BasicOutputMessage* RetrieveOut(); - - // Report an informational message, nothing that needs specific - // attention. + /** + * Reports an informational message from the child thread. The main + * thread will pass this to the Reporter once received. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void Info(const char* msg); - // Report a warning that may indicate a problem. + /** + * Reports a warning from the child thread that may indicate a + * problem. The main thread will pass this to the Reporter once + * received. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void Warning(const char* msg); - // Report a non-fatal error. Processing proceeds normally after the error - // has been reported. + /** + * Reports a non-fatal error from the child thread. The main thread + * will pass this to the Reporter once received. Processing proceeds + * normally after the error has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void Error(const char* msg); - // Report a fatal error. Bro will terminate after the message has been - // reported. + /** + * Reports a fatal error from the child thread. The main thread will + * pass this to the Reporter once received. Bro will terminate after + * the message has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void FatalError(const char* msg); - // Report a fatal error. Bro will terminate after the message has been - // reported and always generate a core dump. + /** + * Reports a fatal error from the child thread. The main thread will + * pass this to the Reporter once received. Bro will terminate with a + * core dump after the message has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void FatalErrorWithCore(const char* msg); - // Report about a potential internal problem. Bro will continue - // normally. + /** + * Reports a potential internal problem from the child thread. The + * main thread will pass this to the Reporter once received. Bro will + * continue normally. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void InternalWarning(const char* msg); - // Report an internal program error. Bro will terminate with a core - // dump after the message has been reported. + /** + * Reports an internal program error from the child thread. The main + * thread will pass this to the Reporter once received. Bro will + * terminate with a core dump after the message has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void InternalError(const char* msg); #ifdef DEBUG - // Records a debug message for the given stream. + /** + * Records a debug message for the given stream from the child + * thread. The main thread will pass this to the DebugLogger once + * received. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void Debug(DebugStream stream, const char* msg); #endif - void Heartbeat(); - + /** + * Statistics about inter-thread communication. + */ struct Stats { - uint64_t sent_in; - uint64_t sent_out; - uint64_t pending_in; - uint64_t pending_out; + uint64_t sent_in; //! Number of messages sent to the child thread. + uint64_t sent_out; //! Number of messages sent from the child thread to the main thread + uint64_t pending_in; //! Number of messages sent to the child but not yet processed. + uint64_t pending_out; //! Number of messages sent from the child but not yet processed by the main thread. }; + /** + * Returns statistics about the inter-thread communication. + * + * @param stats A pointer to a structure that will be filled with + * current numbers. + */ void GetStats(Stats* stats); protected: + friend class Manager; friend class HeartbeatMessage; + /** + * Pops a message sent by the child from the child-to-main queue. + * + * This is method is called regularly by the threading::Manager. + * + * @return The message, wth ownership passed to caller. Returns null + * if the queue is empty. + */ + BasicOutputMessage* RetrieveOut(); + + /** + * Triggers a heartbeat message being sent to the client thread. + * + * This is method is called regularly by the threading::Manager. + */ + void Heartbeat(); + + /** + * Overriden from BasicThread. + * + */ virtual void Run(); virtual void OnStop(); virtual bool DoHeartbeat(double network_time, double current_time) { return true; } private: - friend class Manager; - + /** + * Pops a message sent by the main thread from the main-to-chold + * queue. + * + * Must only be called by the child thread. + * + * @return The message, wth ownership passed to caller. Returns null + * if the queue is empty. + */ BasicInputMessage* RetrieveIn(); + /** + * Queues a message for the child. + * + * Must only be called by the main thread. + * + * @param msg The message. + * + * @param force: If true, the message will be queued even when we're already + * Terminating(). Normally, the message would be discarded in that + * case. + */ void SendIn(BasicInputMessage* msg, bool force); + + /** + * Queues a message for the main thread. + * + * Must only be called by the child thread. + * + * @param msg The message. + * + * @param force: If true, the message will be queued even when we're already + * Terminating(). Normally, the message would be discarded in that + * case. + */ void SendOut(BasicOutputMessage* msg, bool force); + /** + * Returns true if there's at least one message pending for the child + * thread. + */ bool HasIn() { return queue_in.Ready(); } + + /** + * Returns true if there's at least one message pending for the main + * thread. + */ bool HasOut() { return queue_out.Ready(); } - Queue_ queue_in; - Queue_ queue_out; + Queue queue_in; + Queue queue_out; - uint64_t cnt_sent_in; - uint64_t cnt_sent_out; + uint64_t cnt_sent_in; // Counts message sent to child. + uint64_t cnt_sent_out; // Counts message sent by child. }; +/** + * Base class for all message between Bro's main process and a MsgThread. + */ class Message { public: + /** + * Destructor. + */ virtual ~Message(); + /** + * Returns a descriptive name for the message's general type. This is + * what's passed into the constructor and used mainly for debugging + * purposes. + */ const string& Name() const { return name; } + /** + * Callback that must be overriden for processing a message. + */ virtual bool Process() = 0; // Thread will be terminated if returngin false. protected: + /** + * Constructor. + * + * @param arg_name A descriptive name for the type of message. Used + * mainly for debugging purposes. + */ Message(const string& arg_name) { name = arg_name; } private: string name; }; +/** + * Base class for messages sent from Bro's main thread to a child MsgThread. + */ class BasicInputMessage : public Message { protected: + /** + * Constructor. + * + * @param name A descriptive name for the type of message. Used + * mainly for debugging purposes. + */ BasicInputMessage(const string& name) : Message(name) {} }; +/** + * Base class for messages sent from a child MsgThread to Bro's main thread. + */ class BasicOutputMessage : public Message { protected: + /** + * Constructor. + * + * @param name A descriptive name for the type of message. Used + * mainly for debugging purposes. + */ BasicOutputMessage(const string& name) : Message(name) {} }; +/** + * A paremeterized InputMessage that stores a pointer to an argument object. + * Normally, the objects will be used from the Process() callback. + */ template class InputMessage : public BasicInputMessage { public: + /** + * Returns the objects passed to the constructor. + */ O* Object() const { return object; } protected: + /** + * Constructor. + * + * @param name: A descriptive name for the type of message. Used + * mainly for debugging purposes. + * + * @param arg_object: An object to store with the message. + */ InputMessage(const string& name, O* arg_object) : BasicInputMessage(name) { object = arg_object; } @@ -137,13 +348,28 @@ private: O* object; }; +/** + * A paremeterized OututMessage that stores a pointer to an argument object. + * Normally, the objects will be used from the Process() callback. + */ template class OutputMessage : public BasicOutputMessage { public: + /** + * Returns the objects passed to the constructor. + */ O* Object() const { return object; } protected: + /** + * Constructor. + * + * @param name A descriptive name for the type of message. Used + * mainly for debugging purposes. + * + * @param arg_object An object to store with the message. + */ OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name) { object = arg_object; } diff --git a/src/threading/Queue.h b/src/threading/Queue.h index 49859dc051..add7019f9c 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -9,23 +9,53 @@ #include "Reporter.h" +#undef Queue // Defined elsewhere unfortunately. + namespace threading { /** - * Just a simple threaded queue wrapper class. Uses multiple queues and reads / writes in rotary fashion in an attempt to limit contention. - * Due to locking granularity, bulk put / get is no faster than single put / get as long as FIFO guarantee is required. + * A thread-safe single-reader single-writer queue. + * + * The implementation uses multiple queues and reads/writes in rotary fashion + * in an attempt to limit contention. + * + * All Queue instances must be instantiated by Bro's main thread. + * + * TODO: Unclear how critical performance is for this qeueue. We could like;y + * optimize it further if helpful. */ - template -class Queue_ +class Queue { public: - Queue_(); - ~Queue_(); + /** + * Constructor. + */ + Queue(); + /** + * Destructor. + */ + ~Queue(); + + /** + * Retrieves one elment. + */ T Get(); + + /** + * Queues one element. + */ void Put(T data); + + /** + * Returns true if the next Get() operation will succeed. + */ bool Ready(); + + /** + * Returns the number of queued items not yet retrieved. + */ uint64_t Size(); private: @@ -37,7 +67,7 @@ private: int read_ptr; // Where the next operation will read from int write_ptr; // Where the next operation will write to - uint64_t size; + uint64_t size; // Current queue size. }; inline static void safe_lock(pthread_mutex_t* mutex) @@ -53,7 +83,7 @@ inline static void safe_unlock(pthread_mutex_t* mutex) } template -inline Queue_::Queue_() +inline Queue::Queue() { read_ptr = 0; write_ptr = 0; @@ -69,7 +99,7 @@ inline Queue_::Queue_() } template -inline Queue_::~Queue_() +inline Queue::~Queue() { for( int i = 0; i < NUM_QUEUES; ++i ) { @@ -79,7 +109,7 @@ inline Queue_::~Queue_() } template -inline T Queue_::Get() +inline T Queue::Get() { safe_lock(&mutex[read_ptr]); @@ -100,7 +130,7 @@ inline T Queue_::Get() } template -inline void Queue_::Put(T data) +inline void Queue::Put(T data) { safe_lock(&mutex[write_ptr]); @@ -121,7 +151,7 @@ inline void Queue_::Put(T data) template -inline bool Queue_::Ready() +inline bool Queue::Ready() { safe_lock(&mutex[read_ptr]); @@ -133,7 +163,7 @@ inline bool Queue_::Ready() } template -inline uint64_t Queue_::Size() +inline uint64_t Queue::Size() { safe_lock(&mutex[read_ptr]); From 4f0fc571eff7e6fc3db5c06f8097159c765ee69d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 1 Feb 2012 00:34:18 -0800 Subject: [PATCH 07/42] Doing bulkd writes instead of individual writes now. Also slight change to Writer API, going back to how the rotate methods were before. --- src/logging/WriterBackend.cc | 86 +++++++++++++++++++++++------------ src/logging/WriterBackend.h | 31 +++++++------ src/logging/WriterFrontend.cc | 60 ++++++++++++++++++++---- src/logging/WriterFrontend.h | 11 +++-- src/logging/writers/Ascii.cc | 5 +- src/logging/writers/Ascii.h | 4 +- src/logging/writers/None.cc | 5 +- src/logging/writers/None.h | 4 +- src/threading/MsgThread.h | 16 ++++++- 9 files changed, 155 insertions(+), 67 deletions(-) diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 095490edc4..7c9c1d10ca 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -31,13 +31,13 @@ private: bool terminating; }; -class DisableMessage : public threading::OutputMessage +class FlushWriteBufferMessage : public threading::OutputMessage { public: - DisableMessage(WriterFrontend* writer) - : threading::OutputMessage("Disable", writer) {} + FlushWriteBufferMessage(WriterFrontend* writer) + : threading::OutputMessage("FlushWriteBuffer", writer) {} - virtual bool Process() { Object()->SetDisable(); return true; } + virtual bool Process() { Object()->FlushWriteBuffer(); return true; } }; } @@ -65,25 +65,31 @@ WriterBackend::~WriterBackend() } } -void WriterBackend::DeleteVals(Value** vals) +void WriterBackend::DeleteVals(int num_writes, Value*** vals) { - // Note this code is duplicated in Manager::DeleteVals(). - for ( int i = 0; i < num_fields; i++ ) - delete vals[i]; + for ( int j = 0; j < num_writes; ++j ) + { + // Note this code is duplicated in Manager::DeleteVals(). + for ( int i = 0; i < num_fields; i++ ) + delete vals[j][i]; + + delete [] vals[j]; + } delete [] vals; } -bool WriterBackend::FinishedRotation(WriterFrontend* writer, string new_name, string old_name, +bool WriterBackend::FinishedRotation(string new_name, string old_name, double open, double close, bool terminating) { - SendOut(new RotationFinishedMessage(writer, new_name, old_name, open, close, terminating)); + SendOut(new RotationFinishedMessage(frontend, new_name, old_name, open, close, terminating)); return true; } -bool WriterBackend::Init(string arg_path, int arg_num_fields, - const Field* const * arg_fields) +bool WriterBackend::Init(WriterFrontend* arg_frontend, string arg_path, int arg_num_fields, + const Field* const * arg_fields) { + frontend = arg_frontend; path = arg_path; num_fields = arg_num_fields; fields = arg_fields; @@ -94,7 +100,7 @@ bool WriterBackend::Init(string arg_path, int arg_num_fields, return true; } -bool WriterBackend::Write(int arg_num_fields, Value** vals) +bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals) { // Double-check that the arguments match. If we get this from remote, // something might be mixed up. @@ -107,30 +113,42 @@ bool WriterBackend::Write(int arg_num_fields, Value** vals) Debug(DBG_LOGGING, msg); #endif - DeleteVals(vals); + DeleteVals(num_writes, vals); return false; } - for ( int i = 0; i < num_fields; ++i ) - { - if ( vals[i]->type != fields[i]->type ) - { #ifdef DEBUG - const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)", - vals[i]->type, fields[i]->type); - Debug(DBG_LOGGING, msg); -#endif + // Double-check all the types match. + for ( int j = 0; j < num_writes; j++ ) + { + for ( int i = 0; i < num_fields; ++i ) + { + if ( vals[j][i]->type != fields[i]->type ) + { + const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)", + vals[j][i]->type, fields[i]->type); + Debug(DBG_LOGGING, msg); - DeleteVals(vals); - return false; + DeleteVals(num_writes, vals); + return false; + } } } +#endif - bool result = DoWrite(num_fields, fields, vals); + bool success = true; - DeleteVals(vals); + for ( int j = 0; j < num_writes; j++ ) + { + success = DoWrite(num_fields, fields, vals[j]); - return result; + if ( ! success ) + break; + } + + DeleteVals(num_writes, vals); + + return success; } bool WriterBackend::SetBuf(bool enabled) @@ -144,10 +162,10 @@ bool WriterBackend::SetBuf(bool enabled) return DoSetBuf(enabled); } -bool WriterBackend::Rotate(WriterFrontend* writer, string rotated_path, - double open, double close, bool terminating) +bool WriterBackend::Rotate(string rotated_path, double open, + double close, bool terminating) { - return DoRotate(writer, rotated_path, open, close, terminating); + return DoRotate(rotated_path, open, close, terminating); } bool WriterBackend::Flush() @@ -159,3 +177,11 @@ bool WriterBackend::Finish() { return DoFinish(); } + +bool WriterBackend::DoHeartbeat(double network_time, double current_time) + { + SendOut(new FlushWriteBufferMessage(frontend)); + return true; + } + + diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index d1e4634e6d..27f4fe45a5 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -19,6 +19,12 @@ public: virtual ~WriterBackend(); // One-time initialization of the writer to define the logged fields. + // + // "frontend" is the frontend writer that created this backend. The + // *only* purpose of this value is to be passed back via messages as + // a argument to callbacks. One must not otherwise access the + // frontend, it's running in a different thread. + // // Interpretation of "path" is left to the writer, and will be // corresponding the value configured on the script-level. // @@ -27,7 +33,7 @@ public: // // The new instance takes ownership of "fields", and will delete them // when done. - bool Init(string path, int num_fields, const Field* const * fields); + bool Init(WriterFrontend* frontend, string path, int num_fields, const Field* const * fields); // Writes one log entry. The method takes ownership of "vals" and // will return immediately after queueing the write request, which is @@ -38,7 +44,7 @@ public: // // Returns false if an error occured, in which case the writer must // not be used any further. - bool Write(int num_fields, Value** vals); + bool Write(int num_fields, int num_writes, Value*** vals); // Sets the buffering status for the writer, if the writer supports // that. (If not, it will be ignored). @@ -50,7 +56,7 @@ public: // Triggers rotation, if the writer supports that. (If not, it will // be ignored). - bool Rotate(WriterFrontend* writer, string rotated_path, double open, double close, bool terminating); + bool Rotate(string rotated_path, double open, double close, bool terminating); // Finishes writing to this logger regularly. Must not be called if // an error has been indicated earlier. After calling this, no @@ -81,9 +87,10 @@ public: // // terminating: True if rotation request occured due to the main Bro // process shutting down. - bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, + bool FinishedRotation(string new_name, string old_name, double open, double close, bool terminating); + protected: // Methods for writers to override. If any of these returs false, it // will be assumed that a fatal error has occured that prevents the @@ -128,11 +135,6 @@ protected: // RotationDone() to signal the log manager that potential // postprocessors can now run. // - // "writer" is the frontend writer that triggered the rotation. The - // *only* purpose of this value is to be passed into - // FinishedRotation() once done. You must not otherwise access the - // frontend, it's running in a different thread. - // // "rotate_path" reflects the path to where the rotated output is to // be moved, with specifics depending on the writer. It should // generally be interpreted in a way consistent with that of "path" @@ -149,8 +151,8 @@ protected: // // A writer may ignore rotation requests if it doesn't fit with its // semantics (but must still return true in that case). - virtual bool DoRotate(WriterFrontend* writer, string rotated_path, - double open, double close, bool terminating) = 0; + virtual bool DoRotate(string rotated_path, double open, double close, + bool terminating) = 0; // Called once on termination. Not called when any of the other // methods has previously signaled an error, i.e., executing this @@ -158,7 +160,9 @@ protected: virtual bool DoFinish() = 0; // Triggered by regular heartbeat messages from the main process. - virtual bool DoHeartbeat(double network_time, double current_time) { return true; }; + // + // Note when overriding, you must call WriterBackend::DoHeartbeat(). + virtual bool DoHeartbeat(double network_time, double current_time); private: friend class Manager; @@ -169,8 +173,9 @@ private: bool Disabled() { return disabled; } // Deletes the values as passed into Write(). - void DeleteVals(Value** vals); + void DeleteVals(int num_writes, Value*** vals); + WriterFrontend* frontend; string path; int num_fields; const Field* const * fields; diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 92c93c1c56..2f7c1d6e7e 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -9,13 +9,14 @@ namespace logging { class InitMessage : public threading::InputMessage { public: - InitMessage(WriterBackend* backend, const string path, const int num_fields, const Field* const *fields) + InitMessage(WriterBackend* backend, WriterFrontend* frontend, const string path, const int num_fields, const Field* const *fields) : threading::InputMessage("Init", backend), path(path), num_fields(num_fields), fields(fields) { } - virtual bool Process() { return Object()->Init(path, num_fields, fields); } + virtual bool Process() { return Object()->Init(frontend, path, num_fields, fields); } private: + WriterFrontend* frontend; const string path; const int num_fields; const Field * const* fields; @@ -31,7 +32,7 @@ public: rotated_path(rotated_path), open(open), close(close), terminating(terminating) { } - virtual bool Process() { return Object()->Rotate(frontend, rotated_path, open, close, terminating); } + virtual bool Process() { return Object()->Rotate(rotated_path, open, close, terminating); } private: WriterFrontend* frontend; @@ -44,16 +45,16 @@ private: class WriteMessage : public threading::InputMessage { public: - WriteMessage(WriterBackend* backend, const int num_fields, Value **vals) + WriteMessage(WriterBackend* backend, int num_fields, int num_writes, Value*** vals) : threading::InputMessage("Write", backend), - num_fields(num_fields), fields(fields), vals(vals) {} + num_fields(num_fields), vals(vals) {} - virtual bool Process() { return Object()->Write(num_fields, vals); } + virtual bool Process() { return Object()->Write(num_fields, num_writes, vals); } private: int num_fields; - Field* const* fields; - Value **vals; + int num_writes; + Value ***vals; }; class SetBufMessage : public threading::InputMessage @@ -96,6 +97,8 @@ using namespace logging; WriterFrontend::WriterFrontend(bro_int_t type) { disabled = initialized = false; + buf = true; + write_buffer_pos = 0; backend = log_mgr->CreateBackend(type); assert(backend); @@ -108,6 +111,7 @@ WriterFrontend::~WriterFrontend() void WriterFrontend::Stop() { + FlushWriteBuffer(); SetDisable(); backend->Stop(); } @@ -125,7 +129,7 @@ void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* cons fields = arg_fields; initialized = true; - backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + backend->SendIn(new InitMessage(backend, this, arg_path, arg_num_fields, arg_fields)); } void WriterFrontend::Write(int num_fields, Value** vals) @@ -133,7 +137,34 @@ void WriterFrontend::Write(int num_fields, Value** vals) if ( disabled ) return; - backend->SendIn(new WriteMessage(backend, num_fields, vals)); + if ( ! write_buffer ) + { + // Need new buffer. + write_buffer = new Value**[WRITER_BUFFER_SIZE]; + write_buffer_pos = 0; + } + + if ( write_buffer_pos >= WRITER_BUFFER_SIZE ) + // Buffer full. + FlushWriteBuffer(); + + write_buffer[write_buffer_pos++] = vals; + + if ( ! buf ) + // Send out immediately if we don't want buffering. + FlushWriteBuffer(); + } + +void WriterFrontend::FlushWriteBuffer() + { + if ( ! write_buffer_pos ) + // Nothing to do. + return; + + backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer)); + + // Clear buffer (no delete, we pass ownership to child thread.) + write_buffer = 0; } void WriterFrontend::SetBuf(bool enabled) @@ -141,7 +172,13 @@ void WriterFrontend::SetBuf(bool enabled) if ( disabled ) return; + buf = enabled; + backend->SendIn(new SetBufMessage(backend, enabled)); + + if ( ! buf ) + // Make sure no longer buffer any still queued data. + FlushWriteBuffer(); } void WriterFrontend::Flush() @@ -149,6 +186,7 @@ void WriterFrontend::Flush() if ( disabled ) return; + FlushWriteBuffer(); backend->SendIn(new FlushMessage(backend)); } @@ -157,6 +195,7 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool if ( disabled ) return; + FlushWriteBuffer(); backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); } @@ -165,6 +204,7 @@ void WriterFrontend::Finish() if ( disabled ) return; + FlushWriteBuffer(); backend->SendIn(new FinishMessage(backend)); } diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index 1998429d38..ed1a674842 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -34,6 +34,7 @@ public: void Write(int num_fields, Value** vals); void SetBuf(bool enabled); void Flush(); + void FlushWriteBuffer(); void Rotate(string rotated_path, double open, double close, bool terminating); void Finish(); @@ -49,18 +50,22 @@ public: protected: friend class Manager; - WriterBackend* backend; bool disabled; bool initialized; + bool buf; string path; int num_fields; const Field* const * fields; + + // Buffer for bulk writes. + static const int WRITER_BUFFER_SIZE = 50; + + int write_buffer_pos; + Value*** write_buffer; }; } - - #endif diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 70f513be3b..a1ceb6e217 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -317,8 +317,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, return true; } -bool Ascii::DoRotate(WriterFrontend* writer, string rotated_path, double open, - double close, bool terminating) +bool Ascii::DoRotate(string rotated_path, double open, double close, bool terminating) { // Don't rotate special files or if there's not one currently open. if ( ! file || IsSpecial(Path()) ) @@ -330,7 +329,7 @@ bool Ascii::DoRotate(WriterFrontend* writer, string rotated_path, double open, string nname = rotated_path + "." + LogExt(); rename(fname.c_str(), nname.c_str()); - if ( ! FinishedRotation(writer, nname, fname, open, close, terminating) ) + if ( ! FinishedRotation(nname, fname, open, close, terminating) ) { Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str())); return false; diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 37fcfef267..0c627c68e9 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -23,8 +23,8 @@ protected: virtual bool DoWrite(int num_fields, const Field* const * fields, Value** vals); virtual bool DoSetBuf(bool enabled); - virtual bool DoRotate(WriterFrontend* writer, string rotated_path, - double open, double close, bool terminating); + virtual bool DoRotate(string rotated_path, double open, + double close, bool terminating); virtual bool DoFlush(); virtual bool DoFinish(); diff --git a/src/logging/writers/None.cc b/src/logging/writers/None.cc index e419d88a6b..a9a7872f85 100644 --- a/src/logging/writers/None.cc +++ b/src/logging/writers/None.cc @@ -4,10 +4,9 @@ using namespace logging; using namespace writer; -bool None::DoRotate(WriterFrontend* writer, string rotated_path, - double open, double close, bool terminating) +bool None::DoRotate(string rotated_path, double open, double close, bool terminating) { - if ( ! FinishedRotation(writer, string("/dev/null"), Path(), open, close, terminating)) + if ( ! FinishedRotation(string("/dev/null"), Path(), open, close, terminating)) { Error(Fmt("error rotating %s", Path().c_str())); return false; diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h index 9b2ab6c698..9360ef44f6 100644 --- a/src/logging/writers/None.h +++ b/src/logging/writers/None.h @@ -23,8 +23,8 @@ protected: virtual bool DoWrite(int num_fields, const Field* const * fields, Value** vals) { return true; } virtual bool DoSetBuf(bool enabled) { return true; } - virtual bool DoRotate(WriterFrontend* writer, string rotated_path, - double open, double close, bool terminating); + virtual bool DoRotate(string rotated_path, double open, + double close, bool terminating); virtual bool DoFlush() { return true; } virtual bool DoFinish() { return true; } }; diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 8f37041bb6..ec249e90ad 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -184,8 +184,11 @@ protected: * Triggers a heartbeat message being sent to the client thread. * * This is method is called regularly by the threading::Manager. + * + * Can be overriden in derived classed to hook into the heart beat, + * but must call the parent implementation. */ - void Heartbeat(); + virtual void Heartbeat(); /** * Overriden from BasicThread. @@ -194,6 +197,17 @@ protected: virtual void Run(); virtual void OnStop(); + /** + * Regulatly triggered for execution in the child thread. + * + * When overriding, one must call the parent class' implementation. + * + * network_time: The network_time when the heartbeat was trigger by + * the main thread. + * + * current_time: Wall clock when the heartbeat was trigger by the + * main thread. + */ virtual bool DoHeartbeat(double network_time, double current_time) { return true; } private: From 1058e11ffbe9c6e659f963a87a9c696965207e58 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 1 Feb 2012 04:40:01 -0800 Subject: [PATCH 08/42] Adding thread statistics to prof.log --- src/Stats.cc | 15 ++++++++++++++- src/threading/Manager.cc | 17 +++++++++++++++++ src/threading/Manager.h | 21 +++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/Stats.cc b/src/Stats.cc index 55835613e9..05ce33daed 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -9,7 +9,7 @@ #include "ConnCompressor.h" #include "DNS_Mgr.h" #include "Trigger.h" - +#include "threading/Manager.h" int killed_by_inactivity = 0; @@ -217,6 +217,19 @@ void ProfileLogger::Log() current_timers[i])); } + file->Write(fmt("%0.6f Threads: current=%d\n", network_time, thread_mgr->NumThreads())); + + const threading::Manager::msg_stats_list& thread_stats = thread_mgr->GetMsgThreadStats(); + for ( threading::Manager::msg_stats_list::const_iterator i = thread_stats.begin(); + i != thread_stats.end(); ++i ) + { + threading::MsgThread::Stats s = i->second; + file->Write(fmt(" %20s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", + i->first.c_str(), + s.sent_in, s.sent_out, + s.pending_in, s.pending_out)); + } + // Script-level state. unsigned int size, mem = 0; PDict(ID)* globals = global_scope()->Vars(); diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d963876755..2e8f6eb1fc 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -101,4 +101,21 @@ void Manager::Process() next_beat = network_time + HEART_BEAT_INTERVAL; } +const threading::Manager::msg_stats_list& threading::Manager::GetMsgThreadStats() + { + stats.clear(); + + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + { + MsgThread* t = *i; + + MsgThread::Stats s; + t->GetStats(&s); + + stats.push_back(std::make_pair(t->Name(),s)); + } + + return stats; + } + diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 2c4f88fa1e..d2f97209c9 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -43,6 +43,25 @@ public: */ void Terminate(); + typedef std::list > msg_stats_list; + + /** + * Returns statistics from all current MsgThread instances. + * + * @return A list of statistics, with one entry for each MsgThread. + * Each entry is a tuple of thread name and statistics. The list + * reference remains valid until the next call to this method (or + * termination of the manager). + */ + const msg_stats_list& GetMsgThreadStats(); + + /** + * Returns the number of currently active threads. This counts all + * threads that are not yet joined, includingt any potentially in + * Terminating() state. + */ + int NumThreads() const { return all_threads.size(); } + protected: friend class BasicThread; friend class MsgThread; @@ -96,6 +115,8 @@ private: bool did_process; // True if the last Process() found some work to do. double next_beat; // Timestamp when the next heartbeat will be sent. + + msg_stats_list stats; }; } From 29fc56105ddabcc3f29c9b3457c4b9d53f0bca5e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 1 Feb 2012 07:16:24 -0800 Subject: [PATCH 09/42] Documenting logging API. --- src/logging/Manager.cc | 2 +- src/logging/Manager.h | 204 ++++++++++++++++--- src/logging/WriterBackend.cc | 56 +++++- src/logging/WriterBackend.h | 357 ++++++++++++++++++++++------------ src/logging/WriterFrontend.cc | 2 +- src/logging/WriterFrontend.h | 178 ++++++++++++++--- src/logging/writers/Ascii.h | 4 +- src/logging/writers/None.h | 2 +- 8 files changed, 625 insertions(+), 180 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 09c5030fdc..0876b10eb2 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -1279,7 +1279,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, } WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, Field** fields) + int num_fields, const Field* const* fields) { Stream* stream = FindStream(id); diff --git a/src/logging/Manager.h b/src/logging/Manager.h index 7fa2c271db..1267a19ca7 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -15,32 +15,63 @@ class RotationTimer; namespace logging { -// Description of a log field. +/** + * Definition of a log file, i.e., one column of a log stream. + */ struct Field { - string name; - TypeTag type; - // inner type of sets - TypeTag subtype; + string name; //! Name of the field. + TypeTag type; //! Type of the field. + TypeTag subtype; //! Inner type for sets. + /** + * Constructor. + */ Field() { subtype = TYPE_VOID; } + + /** + * Copy constructor. + */ Field(const Field& other) : name(other.name), type(other.type), subtype(other.subtype) { } - // (Un-)serialize. + /** + * Unserializes a field. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ bool Read(SerializationFormat* fmt); + + /** + * Serializes a field. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ bool Write(SerializationFormat* fmt) const; }; -// Values as logged by a writer. +/** + * Definition of a log value, i.e., a entry logged by a stream. + * + * This struct essentialy represents a serialization of a Val instance (for + * those Vals supported). + */ struct Value { - TypeTag type; - bool present; // False for unset fields. + TypeTag type; //! The type of the value. + bool present; //! False for optional record fields that are not set. - // The following union is a subset of BroValUnion, including only the - // types we can log directly. struct set_t { bro_int_t size; Value** vals; }; typedef set_t vec_t; + /** + * This union is a subset of BroValUnion, including only the types we + * can log directly. See IsCompatibleType(). + */ union _val { bro_int_t int_val; bro_uint_t uint_val; @@ -52,42 +83,173 @@ struct Value { vec_t vector_val; } val; + /** + * Constructor. + * + * arg_type: The type of the value. + * + * arg_present: False if the value represents an optional record field + * that is not set. + */ Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) : type(arg_type), present(arg_present) {} + + /** + * Destructor. + */ ~Value(); - // (Un-)serialize. + /** + * Unserializes a value. + * + * @param fmt The serialization format to use. The format handles low-level I/O. + * + * @return False if an error occured. + */ bool Read(SerializationFormat* fmt); + + /** + * Serializes a value. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ bool Write(SerializationFormat* fmt) const; - // Returns true if the type can be logged the framework. If - // `atomic_only` is true, will not permit composite types. + /** + * Returns true if the type can be represented by a Value. If + * `atomic_only` is true, will not permit composite types. + */ static bool IsCompatibleType(BroType* t, bool atomic_only=false); private: - Value(const Value& other) { } + Value(const Value& other) { } // Disabled. }; class WriterBackend; class WriterFrontend; class RotationFinishedMessage; +/** + * Singleton class for managing log streams. + */ class Manager { public: + /** + * Constructor. + */ Manager(); + + /** + * Destructor. + */ ~Manager(); - // These correspond to the BiFs visible on the scripting layer. The - // actual BiFs just forward here. + /** + * Creates a new log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param stream A record of script type \c Log::Stream. + * + * This method corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool CreateStream(EnumVal* id, RecordVal* stream); + + /** + * Enables a log log stream. + * + * @param id The enum value corresponding the log stream. + * + * This method corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool EnableStream(EnumVal* id); + + /** + * Disables a log stream. + * + * @param id The enum value corresponding the log stream. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool DisableStream(EnumVal* id); + + /** + * Adds a filter to a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param filter A record of script type \c Log::Filter. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool AddFilter(EnumVal* id, RecordVal* filter); + + /** + * Removes a filter from a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param name The name of the filter to remove. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool RemoveFilter(EnumVal* id, StringVal* name); + + /** + * Removes a filter from a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param name The name of the filter to remove. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool RemoveFilter(EnumVal* id, string name); + + /** + * Write a record to a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param colums A record of the type defined for the stream's + * columns. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool Write(EnumVal* id, RecordVal* columns); - bool SetBuf(EnumVal* id, bool enabled); // Adjusts all writers. - bool Flush(EnumVal* id); // Flushes all writers.. + + /** + * Sets log streams buffering state. This adjusts all associated + * writers to the new state. + * + * @param id The enum value corresponding the log stream. + * + * @param enabled False to disable buffering (default is enabled). + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool SetBuf(EnumVal* id, bool enabled); + + /** + * Flushes a log stream. This flushed all associated writers. + * + * @param id The enum value corresponding the log stream. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool Flush(EnumVal* id); protected: friend class WriterFrontend; @@ -103,7 +265,7 @@ protected: // Takes ownership of fields. WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, Field** fields); + int num_fields, const Field* const* fields); // Takes ownership of values.. bool Write(EnumVal* id, EnumVal* writer, string path, @@ -112,8 +274,6 @@ protected: // Announces all instantiated writers to peer. void SendAllWritersTo(RemoteSerializer::PeerID peer); - //// Functions safe to use by writers. - // Signals that a file has been rotated. bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, double open, double close, bool terminating); diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 7c9c1d10ca..e361ca69d3 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -40,6 +40,15 @@ public: virtual bool Process() { Object()->FlushWriteBuffer(); return true; } }; +class DisableMessage : public threading::OutputMessage +{ +public: + DisableMessage(WriterFrontend* writer) + : threading::OutputMessage("Disable", writer) {} + + virtual bool Process() { Object()->SetDisable(); return true; } +}; + } // Backend methods. @@ -86,8 +95,13 @@ bool WriterBackend::FinishedRotation(string new_name, string old_name, return true; } +void WriterBackend::DisableFrontend() + { + SendOut(new DisableMessage(frontend)); + } + bool WriterBackend::Init(WriterFrontend* arg_frontend, string arg_path, int arg_num_fields, - const Field* const * arg_fields) + const Field* const* arg_fields) { frontend = arg_frontend; path = arg_path; @@ -95,7 +109,10 @@ bool WriterBackend::Init(WriterFrontend* arg_frontend, string arg_path, int arg_ fields = arg_fields; if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) + { + DisableFrontend(); return false; + } return true; } @@ -114,6 +131,7 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals) #endif DeleteVals(num_writes, vals); + DisableFrontend(); return false; } @@ -129,6 +147,7 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals) vals[j][i]->type, fields[i]->type); Debug(DBG_LOGGING, msg); + DisableFrontend(); DeleteVals(num_writes, vals); return false; } @@ -148,6 +167,9 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals) DeleteVals(num_writes, vals); + if ( ! success ) + DisableFrontend(); + return success; } @@ -159,23 +181,47 @@ bool WriterBackend::SetBuf(bool enabled) buffering = enabled; - return DoSetBuf(enabled); + if ( ! DoSetBuf(enabled) ) + { + DisableFrontend(); + return false; + } + + return true; } bool WriterBackend::Rotate(string rotated_path, double open, double close, bool terminating) { - return DoRotate(rotated_path, open, close, terminating); + if ( ! DoRotate(rotated_path, open, close, terminating) ) + { + DisableFrontend(); + return false; + } + + return true; } bool WriterBackend::Flush() { - return DoFlush(); + if ( ! DoFlush() ) + { + DisableFrontend(); + return false; + } + + return true; } bool WriterBackend::Finish() { - return DoFinish(); + if ( ! DoFlush() ) + { + DisableFrontend(); + return false; + } + + return true; } bool WriterBackend::DoHeartbeat(double network_time, double current_time) diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 27f4fe45a5..b5d313a480 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -11,180 +11,291 @@ namespace logging { -// The backend runs in its own thread, separate from the main process. +/** + * Base class for writer implementation. When the logging::Manager creates a + * new logging filter, it instantiates a WriterFrontend. That then in turn + * creates a WriterBackend of the right type. The frontend then forwards + * message over the backend as its methods are called. + * + * All of this methods must be called only from the corresponding child + * thread (the constructor is the one exception.) + */ class WriterBackend : public threading::MsgThread { public: + /** + * Constructor. + * + * @param name A descriptive name for writer's type (e.g., \c Ascii). + */ WriterBackend(const string& name); + + /** + * Destructor. + */ virtual ~WriterBackend(); - // One-time initialization of the writer to define the logged fields. - // - // "frontend" is the frontend writer that created this backend. The - // *only* purpose of this value is to be passed back via messages as - // a argument to callbacks. One must not otherwise access the - // frontend, it's running in a different thread. - // - // Interpretation of "path" is left to the writer, and will be - // corresponding the value configured on the script-level. - // - // Returns false if an error occured, in which case the writer must - // not be used further. - // - // The new instance takes ownership of "fields", and will delete them - // when done. - bool Init(WriterFrontend* frontend, string path, int num_fields, const Field* const * fields); + /** + * One-time initialization of the writer to define the logged fields. + * + * @param frontend The frontend writer that created this backend. The + * *only* purpose of this value is to be passed back via messages as + * a argument to callbacks. One must not otherwise access the + * frontend, it's running in a different thread. + * + * @param path A string left to the interpretation of the writer + * implementation; it corresponds to the value configured on the + * script-level for the logging filter. + * + * @param num_fields The number of log fields for the stream. + * + * @param fields An array of size \a num_fields with the log fields. + * The methods takes ownership of the array. + * + * @return False if an error occured. + */ + bool Init(WriterFrontend* frontend, string path, int num_fields, const Field* const* fields); - // Writes one log entry. The method takes ownership of "vals" and - // will return immediately after queueing the write request, which is - // potentially before output has actually been written out. - // - // num_fields and the types of the Values must match what was passed - // to Init(). - // - // Returns false if an error occured, in which case the writer must - // not be used any further. + /** + * Writes one log entry. + * + * @param num_fields: The number of log fields for this stream. The + * value must match what was passed to Init(). + * + * @param An array of size \a num_fields with the log values. Their + * types musst match with the field passed to Init(). The method + * takes ownership of \a vals.. + * + * Returns false if an error occured, in which case the writer must + * not be used any further. + * + * @return False if an error occured. + */ bool Write(int num_fields, int num_writes, Value*** vals); - // Sets the buffering status for the writer, if the writer supports - // that. (If not, it will be ignored). + /** + * Sets the buffering status for the writer, assuming the writer + * supports that. (If not, it will be ignored). + * + * @param enabled False if buffering is to be disabled (by default + * it's on). + * + * @return False if an error occured. + */ bool SetBuf(bool enabled); - // Flushes any currently buffered output, if the writer supports - // that. (If not, it will be ignored). + /** + * Flushes any currently buffered output, assuming the writer + * supports that. (If not, it will be ignored). + * + * @return False if an error occured. + */ bool Flush(); - // Triggers rotation, if the writer supports that. (If not, it will - // be ignored). + /** + * Triggers rotation, if the writer supports that. (If not, it will + * be ignored). + * + * @return False if an error occured. + */ bool Rotate(string rotated_path, double open, double close, bool terminating); - // Finishes writing to this logger regularly. Must not be called if - // an error has been indicated earlier. After calling this, no - // further writing must be performed. + /** + * Finishes writing to this logger in a regularl fashion. Must not be + * called if an error has been indicated earlier. After calling this, + * no further writing must be performed. + * + * @return False if an error occured. + */ bool Finish(); - //// Thread-safe methods that may be called from the writer - //// implementation. + /** + * Disables the frontend that has instantiated this backend. Once + * disabled,the frontend will not send any further message over. + */ + void DisableFrontend(); - // The following methods return the information as passed to Init(). + /** + * Returns the log path as passed into the constructor. + */ const string Path() const { return path; } + + /** + * Returns the number of log fields as passed into the constructor. + */ int NumFields() const { return num_fields; } + + /** + * Returns the log fields as passed into the constructor. + */ const Field* const * Fields() const { return fields; } - // Returns the current buffering state. + /** + * Returns the current buffering state. + * + * @return True if buffering is enabled. + */ bool IsBuf() { return buffering; } - // Signals to the log manager that a file has been rotated. - // - // writer: The frontend writer that triggered the rotation. This must - // be the value passed into DoRotate(). - // - // new_name: The filename of the rotated file. old_name: The filename - // of the origina file. - // - // open/close: The timestamps when the original file was opened and - // closed, respectively. - // - // terminating: True if rotation request occured due to the main Bro - // process shutting down. + /** + * Signals that a file has been rotated. This must be called by a + * writer's implementation of DoRotate() once rotation has finished. + * + * Most of the parameters should be passed through from DoRotate(). + * + * @param new_name The filename of the rotated file. + * + * @param old_name The filename of the original file. + * + * @param open: The timestamp when the original file was opened. + * + * @param close: The timestamp when the origina file was closed. + * + * @param terminating: True if the original rotation request occured + * due to the main Bro process shutting down. + */ bool FinishedRotation(string new_name, string old_name, double open, double close, bool terminating); protected: - // Methods for writers to override. If any of these returs false, it - // will be assumed that a fatal error has occured that prevents the - // writer from further operation. It will then be disabled and - // deleted. When returning false, the writer should also report the - // error via Error(). Note that even if a writer does not support the - // functionality for one these methods (like rotation), it must still - // return true if that is not to be considered a fatal error. - // - // Called once for initialization of the writer. + /** + * Writer-specific intialization method. + * + * A writer implementation must override this method. If it returns + * false, it will be assumed that a fatal error has occured that + * prevents the writer from further operation; it will then be + * disabled and eventually deleted. When returning false, an + * implementation should also call Error() to indicate what happened. + */ virtual bool DoInit(string path, int num_fields, - const Field* const * fields) = 0; + const Field* const* fields) = 0; - // Called once per log entry to record. - virtual bool DoWrite(int num_fields, const Field* const * fields, + /** + * Writer-specific output method implementing recording of fone log + * entry. + * + * A writer implementation must override this method. If it returns + * false, it will be assumed that a fatal error has occured that + * prevents the writer from further operation; it will then be + * disabled and eventually deleted. When returning false, an + * implementation should also call Error() to indicate what happened. + */ + virtual bool DoWrite(int num_fields, const Field* const* fields, Value** vals) = 0; - // Called when the buffering status for this writer is changed. If - // buffering is disabled, the writer should attempt to write out - // information as quickly as possible even if doing so may have a - // performance impact. If enabled (which is the default), it may - // buffer data as helpful and write it out later in a way optimized - // for performance. The current buffering state can be queried via - // IsBuf(). - // - // A writer may ignore buffering changes if it doesn't fit with its - // semantics (but must still return true in that case). + /** + * Writer-specific method implementing a change of fthe buffering + * state. If buffering is disabled, the writer should attempt to + * write out information as quickly as possible even if doing so may + * have a performance impact. If enabled (which is the default), it + * may buffer data as helpful and write it out later in a way + * optimized for performance. The current buffering state can be + * queried via IsBuf(). + * + * A writer implementation must override this method but it can just + * ignore calls if buffering doesn't align with its semantics. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + */ virtual bool DoSetBuf(bool enabled) = 0; - // Called to flush any currently buffered output. - // - // A writer may ignore flush requests if it doesn't fit with its - // semantics (but must still return true in that case). + /** + * Writer-specific method implementing flushing of its output. + * + * A writer implementation must override this method but it can just + * ignore calls if flushing doesn't align with its semantics. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + */ virtual bool DoFlush() = 0; - // Called when a log output is to be rotated. Most directly this only - // applies to writers writing into files, which should then close the - // current file and open a new one. However, a writer may also - // trigger other apppropiate actions if semantics are similar. - // - // Once rotation has finished, the implementation should call - // RotationDone() to signal the log manager that potential - // postprocessors can now run. - // - // "rotate_path" reflects the path to where the rotated output is to - // be moved, with specifics depending on the writer. It should - // generally be interpreted in a way consistent with that of "path" - // as passed into DoInit(). As an example, for file-based output, - // "rotate_path" could be the original filename extended with a - // timestamp indicating the time of the rotation. - // - // "open" and "close" are the network time's when the *current* file - // was opened and closed, respectively. - // - // "terminating" indicated whether the rotation request occurs due - // the main Bro prcoess terminating (and not because we've reach a - // regularly scheduled time for rotation). - // - // A writer may ignore rotation requests if it doesn't fit with its - // semantics (but must still return true in that case). + /** + * Writer-specific method implementing log rotation. Most directly + * this only applies to writers writing into files, which should then + * close the current file and open a new one. However, a writer may + * also trigger other apppropiate actions if semantics are similar. * + * Once rotation has finished, the implementation must call + * FinishedRotation() to signal the log manager that potential + * postprocessors can now run. + * + * A writer implementation must override this method but it can just + * ignore calls if flushing doesn't align with its semantics. It + * still needs to call FinishedRotation() though. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + * + * @param rotate_path Reflects the path to where the rotated output + * is to be moved, with specifics depending on the writer. It should + * generally be interpreted in a way consistent with that of \c path + * as passed into DoInit(). As an example, for file-based output, \c + * rotate_path could be the original filename extended with a + * timestamp indicating the time of the rotation. + * + * @param open The network time when the *current* file was opened. + * + * @param close The network time when the *current* file was closed. + * + * @param terminating Indicates whether the rotation request occurs + * due the main Bro prcoess terminating (and not because we've + * reached a regularly scheduled time for rotation). + */ virtual bool DoRotate(string rotated_path, double open, double close, bool terminating) = 0; - // Called once on termination. Not called when any of the other - // methods has previously signaled an error, i.e., executing this - // method signals a regular shutdown of the writer. + /** + * Writer-specific method implementing log output finalization at + * termination. Not called when any of the other methods has + * previously signaled an error, i.e., executing this method signals + * a regular shutdown of the writer. + * + * A writer implementation must override this method but it can just + * ignore calls if flushing doesn't align with its semantics. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + */ virtual bool DoFinish() = 0; - // Triggered by regular heartbeat messages from the main process. - // - // Note when overriding, you must call WriterBackend::DoHeartbeat(). + /** + * Triggered by regular heartbeat messages from the main thread. + * + * This method can be overridden but once must call + * WriterBackend::DoHeartbeat(). + */ virtual bool DoHeartbeat(double network_time, double current_time); private: friend class Manager; - // When an error occurs, we call this method to set a flag marking - // the writer as disabled. The Manager will check the flag later and - // remove the writer. - bool Disabled() { return disabled; } - - // Deletes the values as passed into Write(). + /** + * Deletes the values as passed into Write(). + */ void DeleteVals(int num_writes, Value*** vals); + // Frontend that instantiated us. This object must not be access from + // this class, it's running in a different thread! WriterFrontend* frontend; - string path; - int num_fields; - const Field* const * fields; - bool buffering; - bool disabled; - // For implementing Fmt(). - char* buf; - unsigned int buf_len; + string path; // Log path. + int num_fields; // Number of log fields. + const Field* const* fields; // Log fields. + bool buffering; // True if buffering is enabled. }; diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 2f7c1d6e7e..137cdf90ec 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -9,7 +9,7 @@ namespace logging { class InitMessage : public threading::InputMessage { public: - InitMessage(WriterBackend* backend, WriterFrontend* frontend, const string path, const int num_fields, const Field* const *fields) + InitMessage(WriterBackend* backend, WriterFrontend* frontend, const string path, const int num_fields, const Field* const* fields) : threading::InputMessage("Init", backend), path(path), num_fields(num_fields), fields(fields) { } diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index ed1a674842..6f1bb4ea1b 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -1,6 +1,4 @@ // See the file "COPYING" in the main distribution directory for copyright. -// -// Bridge class between main process and writer threads. #ifndef LOGGING_WRITERFRONTEND_H #define LOGGING_WRITERFRONTEND_H @@ -13,57 +11,187 @@ namespace logging { class WriterBackend; +/** + * Bridge class between the logging::Manager and backend writer threads. The + * Manager instantiates one \a WriterFrontend for each open logging filter. + * Each frontend in turns instantiates a WriterBackend-derived class + * internally that's specific to the particular output format. That backend + * spawns a new thread, and it receives messages from the frontend that + * correspond to method called by the manager. + * + */ class WriterFrontend { public: + /** + * Constructor. + * + * type: The backend writer type, with the value corresponding to the + * script-level \c Log::Writer enum (e.g., \a WRITER_ASCII). The + * frontend will internally instantiate a WriterBackend of the + * corresponding type. + * + * Frontends must only be instantiated by the main thread. + */ WriterFrontend(bro_int_t type); + + /** + * Destructor. + * + * Frontends must only be destroyed by the main thread. + */ virtual ~WriterFrontend(); - // Disables the writers and stop the backend thread. + /** + * Stops all output to this writer. Calling this methods disables all + * message forwarding to the backend and stops the backend thread. + * + * This method must only be called from the main thread. + */ void Stop(); - // Interface methods to interact with the writer from the main thread - // (and only from the main thread), typicalli from the log manager. - // All these methods forward (via inter-thread messaging) to the - // corresponding methods of an internally created WriterBackend. See - // there for documentation. - // - // If any of these operations fails, the writer will be automatically - // (but asynchronoulsy) disabled. + /** + * Initializes the writer. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See WriterBackend::Init() for arguments. The method takes + * ownership of \a fields. + * + * This method must only be called from the main thread. + */ + void Init(string path, int num_fields, const Field* const* fields); - void Init(string path, int num_fields, const Field* const * fields); + /** + * Write out a record. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * As an optimization, if buffering is enabled (which is the default) + * this method may buffer several writes and send them over to the + * backend in bulk with a single message. An explicit bulk write of + * all currently buffered data can be triggered with + * FlushWriteBuffer(). The backend writer triggers this with a + * message at every heartbeat. + * + * See WriterBackend::Writer() for arguments (except that this method + * takes only a single record, not an array). The method takes + * ownership of \a vals. + * + * This method must only be called from the main thread. + */ void Write(int num_fields, Value** vals); + + /** + * Sets the buffering state. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See WriterBackend::SetBuf() for arguments. + * + * This method must only be called from the main thread. + */ void SetBuf(bool enabled); + + /** + * Flushes the output. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. In addition, it also triggers + * FlushWriteBuffer(). If the backend method fails, it sends a + * message back that will asynchronously call Disable(). + * + * This method must only be called from the main thread. + */ void Flush(); - void FlushWriteBuffer(); + + /** + * Triggers log rotation. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See WriterBackend::Rotate() for arguments. + * + * This method must only be called from the main thread. + */ void Rotate(string rotated_path, double open, double close, bool terminating); + + /** + * Finalizes writing to this tream. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * This method must only be called from the main thread. + */ void Finish(); - // Calling this disable the writer. All methods calls will be no-ops - // from now on. The Manager will eventually remove disabled writers. + /** + * Explicitly triggers a transfer of all potentially buffered Write() + * operations over to the backend. + * + * This method must only be called from the main thread. + */ + void FlushWriteBuffer(); + + /** + * Disables the writer frontend. From now on, all method calls that + * would normally send message over to the backend, turn into no-ops. + * Note though that it does not stop the backend itself, use Stop() + * to do thast as well (this method is primarily for use as callback + * when the backend wants to disable the frontend). + * + * Disabled frontend will eventually be discarded by the + * logging::Manager. + * + * This method must only be called from the main thread. + */ void SetDisable() { disabled = true; } + + /** + * Returns true if the writer frontend has been disabled with SetDisable(). + */ bool Disabled() { return disabled; } + /** + * Returns the log path as passed into the constructor. + */ const string Path() const { return path; } + + /** + * Returns the number of log fields as passed into the constructor. + */ int NumFields() const { return num_fields; } + + /** + * Returns the log fields as passed into the constructor. + */ const Field* const * Fields() const { return fields; } protected: friend class Manager; - WriterBackend* backend; - bool disabled; - bool initialized; - bool buf; + WriterBackend* backend; // The backend we have instanatiated. + bool disabled; // True if disabled. + bool initialized; // True if initialized. + bool buf; // True if buffering is enabled (default). - string path; - int num_fields; - const Field* const * fields; + string path; // The log path. + int num_fields; // The number of log fields. + const Field* const* fields; // The log fields. // Buffer for bulk writes. static const int WRITER_BUFFER_SIZE = 50; - - int write_buffer_pos; - Value*** write_buffer; + int write_buffer_pos; // Position of next write in buffer. + Value*** write_buffer; // Buffer of size WRITER_BUFFER_SIZE. }; } diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 0c627c68e9..4a9dea4950 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -19,8 +19,8 @@ public: protected: virtual bool DoInit(string path, int num_fields, - const Field* const * fields); - virtual bool DoWrite(int num_fields, const Field* const * fields, + const Field* const* fields); + virtual bool DoWrite(int num_fields, const Field* const* fields, Value** vals); virtual bool DoSetBuf(bool enabled); virtual bool DoRotate(string rotated_path, double open, diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h index 9360ef44f6..b25bb09348 100644 --- a/src/logging/writers/None.h +++ b/src/logging/writers/None.h @@ -20,7 +20,7 @@ protected: virtual bool DoInit(string path, int num_fields, const Field* const * fields) { return true; } - virtual bool DoWrite(int num_fields, const Field* const * fields, + virtual bool DoWrite(int num_fields, const Field* const* fields, Value** vals) { return true; } virtual bool DoSetBuf(bool enabled) { return true; } virtual bool DoRotate(string rotated_path, double open, From ffb4094d365c61a9df8fea0d50ae375f06cc56b4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 3 Feb 2012 02:41:10 -0800 Subject: [PATCH 10/42] Bugfixes --- src/logging/Manager.cc | 8 ++++---- src/logging/Manager.h | 2 +- src/logging/WriterBackend.cc | 8 ++++---- src/logging/WriterBackend.h | 15 ++++++++------- src/logging/WriterFrontend.cc | 22 ++++++++++------------ src/logging/writers/Ascii.cc | 2 +- src/logging/writers/Ascii.h | 5 +++-- src/logging/writers/None.h | 5 +++-- src/threading/MsgThread.cc | 8 +------- 9 files changed, 35 insertions(+), 40 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 0876b10eb2..63d4c60a5c 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -21,7 +21,7 @@ struct WriterDefinition { bro_int_t type; // The type. const char *name; // Descriptive name for error messages. bool (*init)(); // An optional one-time initialization function. - WriterBackend* (*factory)(); // A factory function creating instances. + WriterBackend* (*factory)(WriterFrontend* frontend); // A factory function creating instances. }; // Static table defining all availabel log writers. @@ -30,7 +30,7 @@ WriterDefinition log_writers[] = { { BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate }, // End marker, don't touch. - { BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)())0 } + { BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)(WriterFrontend* frontend))0 } }; struct Manager::Filter { @@ -436,7 +436,7 @@ Manager::~Manager() delete *s; } -WriterBackend* Manager::CreateBackend(bro_int_t type) +WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) { WriterDefinition* ld = log_writers; @@ -478,7 +478,7 @@ WriterBackend* Manager::CreateBackend(bro_int_t type) assert(ld->factory); - WriterBackend* backend = (*ld->factory)(); + WriterBackend* backend = (*ld->factory)(frontend); assert(backend); return backend; } diff --git a/src/logging/Manager.h b/src/logging/Manager.h index 1267a19ca7..f6829b3554 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -259,7 +259,7 @@ protected: // Instantiates a new WriterBackend of the given type (note that // doing so creates a new thread!). - WriterBackend* CreateBackend(bro_int_t type); + WriterBackend* CreateBackend(WriterFrontend* frontend, bro_int_t type); //// Function also used by the RemoteSerializer. diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index e361ca69d3..fe3a6ef560 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -55,12 +55,13 @@ public: using namespace logging; -WriterBackend::WriterBackend(const string& name) : MsgThread(name) +WriterBackend::WriterBackend(WriterFrontend* arg_frontend, const string& name) : MsgThread(name) { path = ""; num_fields = 0; fields = 0; buffering = true; + frontend = arg_frontend; } WriterBackend::~WriterBackend() @@ -100,10 +101,8 @@ void WriterBackend::DisableFrontend() SendOut(new DisableMessage(frontend)); } -bool WriterBackend::Init(WriterFrontend* arg_frontend, string arg_path, int arg_num_fields, - const Field* const* arg_fields) +bool WriterBackend::Init(string arg_path, int arg_num_fields, const Field* const* arg_fields) { - frontend = arg_frontend; path = arg_path; num_fields = arg_num_fields; fields = arg_fields; @@ -227,6 +226,7 @@ bool WriterBackend::Finish() bool WriterBackend::DoHeartbeat(double network_time, double current_time) { SendOut(new FlushWriteBufferMessage(frontend)); + return true; } diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index b5d313a480..21dcd41ff7 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -26,9 +26,15 @@ public: /** * Constructor. * + * @param frontend The frontend writer that created this backend. The + * *only* purpose of this value is to be passed back via messages as + * a argument to callbacks. One must not otherwise access the + * frontend, it's running in a different thread. + * * @param name A descriptive name for writer's type (e.g., \c Ascii). + * */ - WriterBackend(const string& name); + WriterBackend(WriterFrontend* frontend, const string& name); /** * Destructor. @@ -38,11 +44,6 @@ public: /** * One-time initialization of the writer to define the logged fields. * - * @param frontend The frontend writer that created this backend. The - * *only* purpose of this value is to be passed back via messages as - * a argument to callbacks. One must not otherwise access the - * frontend, it's running in a different thread. - * * @param path A string left to the interpretation of the writer * implementation; it corresponds to the value configured on the * script-level for the logging filter. @@ -54,7 +55,7 @@ public: * * @return False if an error occured. */ - bool Init(WriterFrontend* frontend, string path, int num_fields, const Field* const* fields); + bool Init(string path, int num_fields, const Field* const* fields); /** * Writes one log entry. diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 137cdf90ec..79c180b749 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -9,14 +9,13 @@ namespace logging { class InitMessage : public threading::InputMessage { public: - InitMessage(WriterBackend* backend, WriterFrontend* frontend, const string path, const int num_fields, const Field* const* fields) + InitMessage(WriterBackend* backend, const string path, const int num_fields, const Field* const* fields) : threading::InputMessage("Init", backend), path(path), num_fields(num_fields), fields(fields) { } - virtual bool Process() { return Object()->Init(frontend, path, num_fields, fields); } + virtual bool Process() { return Object()->Init(path, num_fields, fields); } private: - WriterFrontend* frontend; const string path; const int num_fields; const Field * const* fields; @@ -47,7 +46,7 @@ class WriteMessage : public threading::InputMessage public: WriteMessage(WriterBackend* backend, int num_fields, int num_writes, Value*** vals) : threading::InputMessage("Write", backend), - num_fields(num_fields), vals(vals) {} + num_fields(num_fields), num_writes(num_writes), vals(vals) {} virtual bool Process() { return Object()->Write(num_fields, num_writes, vals); } @@ -98,8 +97,9 @@ WriterFrontend::WriterFrontend(bro_int_t type) { disabled = initialized = false; buf = true; + write_buffer = 0; write_buffer_pos = 0; - backend = log_mgr->CreateBackend(type); + backend = log_mgr->CreateBackend(this, type); assert(backend); backend->Start(); @@ -129,7 +129,7 @@ void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* cons fields = arg_fields; initialized = true; - backend->SendIn(new InitMessage(backend, this, arg_path, arg_num_fields, arg_fields)); + backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); } void WriterFrontend::Write(int num_fields, Value** vals) @@ -144,15 +144,12 @@ void WriterFrontend::Write(int num_fields, Value** vals) write_buffer_pos = 0; } - if ( write_buffer_pos >= WRITER_BUFFER_SIZE ) - // Buffer full. - FlushWriteBuffer(); - write_buffer[write_buffer_pos++] = vals; - if ( ! buf ) - // Send out immediately if we don't want buffering. + if ( write_buffer_pos >= WRITER_BUFFER_SIZE || ! buf ) + // Buffer full (or no bufferin desired). FlushWriteBuffer(); + } void WriterFrontend::FlushWriteBuffer() @@ -165,6 +162,7 @@ void WriterFrontend::FlushWriteBuffer() // Clear buffer (no delete, we pass ownership to child thread.) write_buffer = 0; + write_buffer_pos = 0; } void WriterFrontend::SetBuf(bool enabled) diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index a1ceb6e217..0e522dde1d 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -10,7 +10,7 @@ using namespace logging; using namespace writer; -Ascii::Ascii() : WriterBackend("Ascii") +Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend, "Ascii") { file = 0; diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 4a9dea4950..4a24aad9b7 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -11,10 +11,11 @@ namespace logging { namespace writer { class Ascii : public WriterBackend { public: - Ascii(); + Ascii(WriterFrontend* frontend); ~Ascii(); - static WriterBackend* Instantiate() { return new Ascii; } + static WriterBackend* Instantiate(WriterFrontend* frontend) + { return new Ascii(frontend); } static string LogExt(); protected: diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h index b25bb09348..19dc685ecb 100644 --- a/src/logging/writers/None.h +++ b/src/logging/writers/None.h @@ -11,10 +11,11 @@ namespace logging { namespace writer { class None : public WriterBackend { public: - None() : WriterBackend("None") {} + None(WriterFrontend* frontend) : WriterBackend(frontend, "None") {} ~None() {}; - static WriterBackend* Instantiate() { return new None; } + static WriterBackend* Instantiate(WriterFrontend* frontend) + { return new None(frontend); } protected: virtual bool DoInit(string path, int num_fields, diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 455c177df6..52da7c7400 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -214,13 +214,7 @@ BasicOutputMessage* MsgThread::RetrieveOut() BasicOutputMessage* msg = queue_out.Get(); assert(msg); -#ifdef DEBUG - if ( msg->Name() != "DebugMessage" ) // Avoid recursion. - { - string s = Fmt("Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str()); - Debug(DBG_THREADING, s.c_str()); - } -#endif + DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str()); return msg; } From 4879cb7b0ddcba2111deebb688d18a5c9cd9c8af Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 3 Feb 2012 03:03:38 -0800 Subject: [PATCH 11/42] Improved signal handling. Sending SIGTERM triggers a normal shutdown of all threads that waits until they have processed their remaining data. However, sending a 2nd SIGTERM while waiting for them to finish will immediately kill them all. --- src/main.cc | 7 +++++++ src/threading/BasicThread.cc | 11 +++++++++++ src/threading/BasicThread.h | 9 +++++++++ src/threading/Manager.cc | 13 +++++++++++++ src/threading/Manager.h | 16 ++++++++++++++++ 5 files changed, 56 insertions(+) diff --git a/src/main.cc b/src/main.cc index 58a23e6c80..e224910db4 100644 --- a/src/main.cc +++ b/src/main.cc @@ -333,6 +333,13 @@ RETSIGTYPE sig_handler(int signo) { set_processing_status("TERMINATING", "sig_handler"); signal_val = signo; + + if ( thread_mgr->Terminating() && (signal_val == SIGTERM || signal_val == SIGINT) ) + // If the thread manager is already terminating (i.e., + // waiting for child threads to exit), another term signal + // will send the threads a kill. + thread_mgr->KillThreads(); + return RETSIGVAL; } diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index f7bd2afbcd..73dc562b31 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -113,6 +113,17 @@ void BasicThread::Join() pthread = 0; } +void BasicThread::Kill() + { + if ( ! (started && pthread) ) + return; + + // I believe this is safe to call from a signal handler ... Not error + // checking so that killing doesn't bail out if we have already + // terminated. + pthread_kill(pthread, SIGKILL); + } + void* BasicThread::launcher(void *arg) { BasicThread* thread = (BasicThread *)arg; diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index df5665c464..aeafc61c52 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -121,6 +121,15 @@ protected: */ void Join(); + /** + * Kills the thread immediately. One still needs to call Join() + * afterwards. + * + * This is called from the threading::Manager and safe to execute + * during a signal handler. + */ + void Kill(); + private: // pthread entry function. static void* launcher(void *arg); diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 2e8f6eb1fc..d07311bbe8 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -9,6 +9,7 @@ Manager::Manager() did_process = false; next_beat = 0; + terminating = false; } Manager::~Manager() @@ -21,6 +22,8 @@ void Manager::Terminate() { DBG_LOG(DBG_THREADING, "Terminating thread manager ..."); + terminating = true; + // First process remaining thread output for the message threads. do Process(); while ( did_process ); @@ -37,6 +40,16 @@ void Manager::Terminate() all_threads.clear(); msg_threads.clear(); + + terminating = false; + } + +void Manager::KillThreads() + { + DBG_LOG(DBG_THREADING, "Killing threads ..."); + + for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ ) + (*i)->Kill(); } void Manager::AddThread(BasicThread* thread) diff --git a/src/threading/Manager.h b/src/threading/Manager.h index d2f97209c9..7d9ba766d4 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -43,6 +43,21 @@ public: */ void Terminate(); + /** + * Returns True if we are currently in Terminate() waiting for + * threads to exit. + */ + bool Terminating() const { return terminating; } + + /** + * Immediately kills all child threads. It does however not yet join + * them, one still needs to call Terminate() for that. + * + * This method is safe to call from a signal handler, and can in fact + * be called while Terminate() is already in progress. + */ + void KillThreads(); + typedef std::list > msg_stats_list; /** @@ -115,6 +130,7 @@ private: bool did_process; // True if the last Process() found some work to do. double next_beat; // Timestamp when the next heartbeat will be sent. + bool terminating; // True if we are in Terminate(). msg_stats_list stats; }; From cf6a346b865fc95275eee83d25656fe95f7bafe3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 3 Feb 2012 03:27:25 -0800 Subject: [PATCH 12/42] Fixing prof.log output. The queue Size() method was not yet atomic. --- src/Stats.cc | 3 ++- src/threading/MsgThread.cc | 1 + src/threading/Queue.h | 17 ++++++++++------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Stats.cc b/src/Stats.cc index 05ce33daed..27b433c9ee 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -224,7 +224,8 @@ void ProfileLogger::Log() i != thread_stats.end(); ++i ) { threading::MsgThread::Stats s = i->second; - file->Write(fmt(" %20s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", + file->Write(fmt("%0.6f %-15s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", + network_time, i->first.c_str(), s.sent_in, s.sent_out, s.pending_in, s.pending_out)); diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 52da7c7400..1bda8943da 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -206,6 +206,7 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force) return; queue_out.Put(msg); + ++cnt_sent_out; } diff --git a/src/threading/Queue.h b/src/threading/Queue.h index add7019f9c..a25f897d23 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -67,7 +67,6 @@ private: int read_ptr; // Where the next operation will read from int write_ptr; // Where the next operation will write to - uint64_t size; // Current queue size. }; inline static void safe_lock(pthread_mutex_t* mutex) @@ -120,7 +119,6 @@ inline T Queue::Get() T data = messages[read_ptr].front(); messages[read_ptr].pop(); - --size; read_ptr = (read_ptr + 1) % NUM_QUEUES; @@ -139,7 +137,6 @@ inline void Queue::Put(T data) bool need_signal = messages[write_ptr].empty(); messages[write_ptr].push(data); - ++size; if ( need_signal ) pthread_cond_signal(&has_data[write_ptr]); @@ -165,13 +162,19 @@ inline bool Queue::Ready() template inline uint64_t Queue::Size() { - safe_lock(&mutex[read_ptr]); + // Need to lock all queues. + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_lock(&mutex[i]); - uint64_t s = size; + uint64_t size = 0; - safe_unlock(&mutex[read_ptr]); + for ( int i = 0; i < NUM_QUEUES; i++ ) + size += messages[i].size(); - return s; + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_unlock(&mutex[i]); + + return size; } } From 70fe7876a176e15bd0c8046dab939ad0f7fbde90 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 3 Feb 2012 04:01:24 -0800 Subject: [PATCH 13/42] Updating thread naming. Also includes experimental code to adapt the thread name as shown by top, but it's untested. --- src/Stats.cc | 2 +- src/logging/Manager.cc | 2 ++ src/logging/WriterBackend.cc | 6 +++++- src/logging/WriterBackend.h | 2 +- src/logging/WriterFrontend.cc | 9 +++++++++ src/logging/WriterFrontend.h | 12 ++++++++++++ src/logging/writers/Ascii.cc | 2 +- src/logging/writers/None.h | 2 +- src/threading/BasicThread.cc | 25 +++++++++++++++++++++++-- src/threading/BasicThread.h | 21 ++++++++++++++++++--- src/threading/MsgThread.cc | 8 +++++++- src/threading/MsgThread.h | 4 +--- 12 files changed, 81 insertions(+), 14 deletions(-) diff --git a/src/Stats.cc b/src/Stats.cc index 27b433c9ee..f418e9cccc 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -224,7 +224,7 @@ void ProfileLogger::Log() i != thread_stats.end(); ++i ) { threading::MsgThread::Stats s = i->second; - file->Write(fmt("%0.6f %-15s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", + file->Write(fmt("%0.6f %-25s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", network_time, i->first.c_str(), s.sent_in, s.sent_out, diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 63d4c60a5c..6d53ea363f 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -480,6 +480,8 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) WriterBackend* backend = (*ld->factory)(frontend); assert(backend); + + frontend->ty_name = ld->name; return backend; } diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index fe3a6ef560..4d2e497b14 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -55,13 +55,15 @@ public: using namespace logging; -WriterBackend::WriterBackend(WriterFrontend* arg_frontend, const string& name) : MsgThread(name) +WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread() { path = ""; num_fields = 0; fields = 0; buffering = true; frontend = arg_frontend; + + SetName(frontend->Name()); } WriterBackend::~WriterBackend() @@ -107,6 +109,8 @@ bool WriterBackend::Init(string arg_path, int arg_num_fields, const Field* const num_fields = arg_num_fields; fields = arg_fields; + SetName(frontend->Name()); + if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) { DisableFrontend(); diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 21dcd41ff7..33271e43f9 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -34,7 +34,7 @@ public: * @param name A descriptive name for writer's type (e.g., \c Ascii). * */ - WriterBackend(WriterFrontend* frontend, const string& name); + WriterBackend(WriterFrontend* frontend); /** * Destructor. diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 79c180b749..79278870f9 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -99,6 +99,7 @@ WriterFrontend::WriterFrontend(bro_int_t type) buf = true; write_buffer = 0; write_buffer_pos = 0; + ty_name = ""; backend = log_mgr->CreateBackend(this, type); assert(backend); @@ -109,6 +110,14 @@ WriterFrontend::~WriterFrontend() { } +string WriterFrontend::Name() const + { + if ( path.size() ) + return ty_name; + + return ty_name + "/" + path; + } + void WriterFrontend::Stop() { FlushWriteBuffer(); diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index 6f1bb4ea1b..e0bc590dfc 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -30,6 +30,9 @@ public: * frontend will internally instantiate a WriterBackend of the * corresponding type. * + * name: A descriptive name for the backend wroter type (e.g., \c + * Ascii). + * * Frontends must only be instantiated by the main thread. */ WriterFrontend(bro_int_t type); @@ -171,6 +174,14 @@ public: */ int NumFields() const { return num_fields; } + /** + * Returns a descriptive name for the writer, including the type of + * the backend and the path used. + * + * This method is safe to call from any thread. + */ + string Name() const; + /** * Returns the log fields as passed into the constructor. */ @@ -184,6 +195,7 @@ protected: bool initialized; // True if initialized. bool buf; // True if buffering is enabled (default). + string ty_name; // Name of the backend type. Set by the manager. string path; // The log path. int num_fields; // The number of log fields. const Field* const* fields; // The log fields. diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 0e522dde1d..7cc8459e68 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -10,7 +10,7 @@ using namespace logging; using namespace writer; -Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend, "Ascii") +Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) { file = 0; diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h index 19dc685ecb..6a62161f49 100644 --- a/src/logging/writers/None.h +++ b/src/logging/writers/None.h @@ -11,7 +11,7 @@ namespace logging { namespace writer { class None : public WriterBackend { public: - None(WriterFrontend* frontend) : WriterBackend(frontend, "None") {} + None(WriterFrontend* frontend) : WriterBackend(frontend) {} ~None() {}; static WriterBackend* Instantiate(WriterFrontend* frontend) diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 73dc562b31..4d51c3c4e4 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -9,7 +9,7 @@ using namespace threading; uint64_t BasicThread::thread_counter = 0; -BasicThread::BasicThread(const string& arg_name) +BasicThread::BasicThread() { started = false; terminating = false; @@ -18,7 +18,7 @@ BasicThread::BasicThread(const string& arg_name) buf = 0; buf_len = 1024; - name = Fmt("%s@%d", arg_name.c_str(), ++thread_counter); + name = Fmt("thread-%d", ++thread_counter); thread_mgr->AddThread(this); } @@ -27,6 +27,27 @@ BasicThread::~BasicThread() { } +void BasicThread::SetName(const string& arg_name) + { + // Slight race condition here with reader threads, but shouldn't matter. + name = arg_name; + } + +void BasicThread::SetOSName(const string& name) + { +#ifdef LINUX + pthread_setname_np(pthread_self(), name.c_str()); +#endif + +#ifdef __APPLE__ + pthread_setname_np(name.c_str()); +#endif + +#ifdef FREEBSD + pthread_set_name_np(pthread_self(), name, name.c_str()); +#endif + } + const char* BasicThread::Fmt(const char* format, ...) { if ( ! buf ) diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index aeafc61c52..6d2f739620 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -34,16 +34,31 @@ public: * @param name A descriptive name for thread the thread. This may * show up in messages to the user. */ - BasicThread(const string& name); // Managed by manager, must not delete otherwise. + BasicThread(); /** - * Returns a descriptive name for the thread. This is the name passed - * into the constructor. + * Returns a descriptive name for the thread. If not set via + * SetName(). If not set, a default name is choosen automatically. * * This method is safe to call from any thread. */ const string& Name() const { return name; } + /** + * Sets a descriptive name for the thread. This should be a string + * that's useful in output presented to the user and uniquely + * identifies the thread. + * + * This method must be called only from the thread itself. + */ + void SetName(const string& name); + + /** + * Set the name shown by the OS as the thread's description. Not + * supported on all OSs. + */ + void SetOSName(const string& name); + /** * Starts the thread. Calling this methods will spawn a new OS thread * executing Run(). Note that one can't restart a thread after a diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 1bda8943da..f41b20ddf9 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -127,7 +127,7 @@ bool ReporterMessage::Process() return true; } -MsgThread::MsgThread(const string& name) : BasicThread(name) +MsgThread::MsgThread() : BasicThread() { cnt_sent_in = cnt_sent_out = 0; thread_mgr->AddMsgThread(this); @@ -142,6 +142,12 @@ void MsgThread::OnStop() void MsgThread::Heartbeat() { SendIn(new HeartbeatMessage(this, network_time, current_time())); + + string name = Fmt("%s (%d/%d)", name.c_str(), + cnt_sent_in - queue_in.Size(), + cnt_sent_out - queue_out.Size()); + + SetOSName(name.c_str()); } void MsgThread::Info(const char* msg) diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index ec249e90ad..459ac6c603 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -34,10 +34,8 @@ public: * threading::Manager. * * Only Bro's main thread may instantiate a new thread. - * - * @param name A descriptive name. This is passed on to BasicThread(). */ - MsgThread(const string& name); + MsgThread(); /** * Sends a message to the child thread. The message will be proceesed From a0487ecb30e365956525aa6eb7cd1703d680b138 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 3 Feb 2012 14:12:29 -0800 Subject: [PATCH 14/42] move Value and Field from the logging namespace to the threading namespace, because other modules using threading will need them. --- src/Attr.cc | 4 +- src/CMakeLists.txt | 1 + src/RemoteSerializer.cc | 21 +- src/RemoteSerializer.h | 10 +- src/logging/Manager.cc | 328 +--------------------------- src/logging/Manager.h | 124 +---------- src/logging/WriterBackend.cc | 4 + src/logging/WriterBackend.h | 16 +- src/logging/WriterFrontend.cc | 4 + src/logging/WriterFrontend.h | 10 +- src/logging/writers/Ascii.cc | 3 + src/logging/writers/Ascii.h | 8 +- src/logging/writers/None.h | 6 +- src/threading/SerializationTypes.cc | 319 +++++++++++++++++++++++++++ src/threading/SerializationTypes.h | 126 +++++++++++ 15 files changed, 512 insertions(+), 472 deletions(-) create mode 100644 src/threading/SerializationTypes.cc create mode 100644 src/threading/SerializationTypes.h diff --git a/src/Attr.cc b/src/Attr.cc index b877250f52..40c6c1a75c 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -5,7 +5,7 @@ #include "Attr.h" #include "Expr.h" #include "Serializer.h" -#include "logging/Manager.h" +#include "threading/SerializationTypes.h" const char* attr_name(attr_tag t) { @@ -416,7 +416,7 @@ void Attributes::CheckAttr(Attr* a) break; case ATTR_LOG: - if ( ! logging::Value::IsCompatibleType(type) ) + if ( ! threading::Value::IsCompatibleType(type) ) Error("&log applied to a type that cannot be logged"); break; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 61a4847b70..7a3cc4babf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -411,6 +411,7 @@ set(bro_SRCS threading/BasicThread.cc threading/Manager.cc threading/MsgThread.cc + threading/SerializationTypes.cc logging/Manager.cc logging/WriterBackend.cc diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index a75812b42b..ba2598c018 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -184,6 +184,7 @@ #include "File.h" #include "Conn.h" #include "Reporter.h" +#include "threading/SerializationTypes.h" #include "logging/Manager.h" extern "C" { @@ -2476,7 +2477,7 @@ bool RemoteSerializer::ProcessRemotePrint() return true; } -bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields) +bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields) { loop_over_list(peers, i) { @@ -2486,7 +2487,7 @@ bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string return true; } -bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields) +bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields) { SetErrorDescr("logging"); @@ -2540,7 +2541,7 @@ error: return false; } -bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals) +bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals) { loop_over_list(peers, i) { @@ -2550,7 +2551,7 @@ bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, i return true; } -bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals) +bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals) { if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) return false; @@ -2641,7 +2642,7 @@ bool RemoteSerializer::ProcessLogCreateWriter() EnumVal* id_val = 0; EnumVal* writer_val = 0; - logging::Field** fields = 0; + threading::Field** fields = 0; BinarySerializationFormat fmt; fmt.StartRead(current_args->data, current_args->len); @@ -2658,11 +2659,11 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( ! success ) goto error; - fields = new logging::Field* [num_fields]; + fields = new threading::Field* [num_fields]; for ( int i = 0; i < num_fields; i++ ) { - fields[i] = new logging::Field; + fields[i] = new threading::Field; if ( ! fields[i]->Read(&fmt) ) goto error; } @@ -2703,7 +2704,7 @@ bool RemoteSerializer::ProcessLogWrite() // Unserialize one entry. EnumVal* id_val = 0; EnumVal* writer_val = 0; - logging::Value** vals = 0; + threading::Value** vals = 0; int id, writer; string path; @@ -2717,11 +2718,11 @@ bool RemoteSerializer::ProcessLogWrite() if ( ! success ) goto error; - vals = new logging::Value* [num_fields]; + vals = new threading::Value* [num_fields]; for ( int i = 0; i < num_fields; i++ ) { - vals[i] = new logging::Value; + vals[i] = new threading::Value; if ( ! vals[i]->Read(&fmt) ) goto error; } diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index ba0bde7d41..571fa72d39 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -15,7 +15,7 @@ class IncrementalSendTimer; -namespace logging { +namespace threading { class Field; class Value; } @@ -102,13 +102,13 @@ public: bool SendPrintHookEvent(BroFile* f, const char* txt, size_t len); // Send a request to create a writer on a remote side. - bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields); + bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields); // Broadcasts a request to create a writer. - bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields); + bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields); // Broadcast a log entry to everybody interested. - bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals); + bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals); // Synchronzizes time with all connected peers. Returns number of // current sync-point, or -1 on error. @@ -303,7 +303,7 @@ protected: bool SendID(SerialInfo* info, Peer* peer, const ID& id); bool SendCapabilities(Peer* peer); bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p); - bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals); + bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals); void UnregisterHandlers(Peer* peer); void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0); diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 6d53ea363f..2333d6c612 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -14,7 +14,11 @@ #include "writers/Ascii.h" #include "writers/None.h" +#include "threading/SerializationTypes.h" + using namespace logging; +using threading::Value; +using threading::Field; // Structure describing a log writer type. struct WriterDefinition { @@ -83,316 +87,6 @@ struct Manager::Stream { ~Stream(); }; -bool Field::Read(SerializationFormat* fmt) - { - int t; - int st; - - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); - type = (TypeTag) t; - subtype = (TypeTag) st; - - return success; - } - -bool Field::Write(SerializationFormat* fmt) const - { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); - } - -Value::~Value() - { - if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC) - && present ) - delete val.string_val; - - if ( type == TYPE_TABLE && present ) - { - for ( int i = 0; i < val.set_val.size; i++ ) - delete val.set_val.vals[i]; - - delete [] val.set_val.vals; - } - - if ( type == TYPE_VECTOR && present ) - { - for ( int i = 0; i < val.vector_val.size; i++ ) - delete val.vector_val.vals[i]; - - delete [] val.vector_val.vals; - } - } - -bool Value::IsCompatibleType(BroType* t, bool atomic_only) - { - if ( ! t ) - return false; - - switch ( t->Tag() ) { - case TYPE_BOOL: - case TYPE_INT: - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - case TYPE_SUBNET: - case TYPE_ADDR: - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - return true; - - case TYPE_RECORD: - return ! atomic_only; - - case TYPE_TABLE: - { - if ( atomic_only ) - return false; - - if ( ! t->IsSet() ) - return false; - - return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); - } - - case TYPE_VECTOR: - { - if ( atomic_only ) - return false; - - return IsCompatibleType(t->AsVectorType()->YieldType(), true); - } - - default: - return false; - } - - return false; - } - -bool Value::Read(SerializationFormat* fmt) - { - int ty; - - if ( ! (fmt->Read(&ty, "type") && fmt->Read(&present, "present")) ) - return false; - - type = (TypeTag)(ty); - - if ( ! present ) - return true; - - switch ( type ) { - case TYPE_BOOL: - case TYPE_INT: - return fmt->Read(&val.int_val, "int"); - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - return fmt->Read(&val.uint_val, "uint"); - - case TYPE_SUBNET: - { - uint32 net[4]; - if ( ! (fmt->Read(&net[0], "net0") && - fmt->Read(&net[1], "net1") && - fmt->Read(&net[2], "net2") && - fmt->Read(&net[3], "net3") && - fmt->Read(&val.subnet_val.width, "width")) ) - return false; - -#ifdef BROv6 - val.subnet_val.net[0] = net[0]; - val.subnet_val.net[1] = net[1]; - val.subnet_val.net[2] = net[2]; - val.subnet_val.net[3] = net[3]; -#else - val.subnet_val.net = net[0]; -#endif - return true; - } - - case TYPE_ADDR: - { - uint32 addr[4]; - if ( ! (fmt->Read(&addr[0], "addr0") && - fmt->Read(&addr[1], "addr1") && - fmt->Read(&addr[2], "addr2") && - fmt->Read(&addr[3], "addr3")) ) - return false; - - val.addr_val[0] = addr[0]; -#ifdef BROv6 - val.addr_val[1] = addr[1]; - val.addr_val[2] = addr[2]; - val.addr_val[3] = addr[3]; -#endif - return true; - } - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - return fmt->Read(&val.double_val, "double"); - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - { - val.string_val = new string; - return fmt->Read(val.string_val, "string"); - } - - case TYPE_TABLE: - { - if ( ! fmt->Read(&val.set_val.size, "set_size") ) - return false; - - val.set_val.vals = new Value* [val.set_val.size]; - - for ( int i = 0; i < val.set_val.size; ++i ) - { - val.set_val.vals[i] = new Value; - - if ( ! val.set_val.vals[i]->Read(fmt) ) - return false; - } - - return true; - } - - case TYPE_VECTOR: - { - if ( ! fmt->Read(&val.vector_val.size, "vector_size") ) - return false; - - val.vector_val.vals = new Value* [val.vector_val.size]; - - for ( int i = 0; i < val.vector_val.size; ++i ) - { - val.vector_val.vals[i] = new Value; - - if ( ! val.vector_val.vals[i]->Read(fmt) ) - return false; - } - - return true; - } - - default: - reporter->InternalError("unsupported type %s in Value::Write", type_name(type)); - } - - return false; - } - -bool Value::Write(SerializationFormat* fmt) const - { - if ( ! (fmt->Write((int)type, "type") && - fmt->Write(present, "present")) ) - return false; - - if ( ! present ) - return true; - - switch ( type ) { - case TYPE_BOOL: - case TYPE_INT: - return fmt->Write(val.int_val, "int"); - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - return fmt->Write(val.uint_val, "uint"); - - case TYPE_SUBNET: - { - uint32 net[4]; -#ifdef BROv6 - net[0] = val.subnet_val.net[0]; - net[1] = val.subnet_val.net[1]; - net[2] = val.subnet_val.net[2]; - net[3] = val.subnet_val.net[3]; -#else - net[0] = val.subnet_val.net; - net[1] = net[2] = net[3] = 0; -#endif - return fmt->Write(net[0], "net0") && - fmt->Write(net[1], "net1") && - fmt->Write(net[2], "net2") && - fmt->Write(net[3], "net3") && - fmt->Write(val.subnet_val.width, "width"); - } - - case TYPE_ADDR: - { - uint32 addr[4]; - addr[0] = val.addr_val[0]; -#ifdef BROv6 - addr[1] = val.addr_val[1]; - addr[2] = val.addr_val[2]; - addr[3] = val.addr_val[3]; -#else - addr[1] = addr[2] = addr[3] = 0; -#endif - return fmt->Write(addr[0], "addr0") && - fmt->Write(addr[1], "addr1") && - fmt->Write(addr[2], "addr2") && - fmt->Write(addr[3], "addr3"); - } - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - return fmt->Write(val.double_val, "double"); - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - return fmt->Write(*val.string_val, "string"); - - case TYPE_TABLE: - { - if ( ! fmt->Write(val.set_val.size, "set_size") ) - return false; - - for ( int i = 0; i < val.set_val.size; ++i ) - { - if ( ! val.set_val.vals[i]->Write(fmt) ) - return false; - } - - return true; - } - - case TYPE_VECTOR: - { - if ( ! fmt->Write(val.vector_val.size, "vector_size") ) - return false; - - for ( int i = 0; i < val.vector_val.size; ++i ) - { - if ( ! val.vector_val.vals[i]->Write(fmt) ) - return false; - } - - return true; - } - - default: - reporter->InternalError("unsupported type %s in Value::REad", type_name(type)); - } - - return false; - } - Manager::Filter::~Filter() { for ( int i = 0; i < num_fields; ++i ) @@ -552,7 +246,7 @@ bool Manager::CreateStream(EnumVal* id, RecordVal* sval) if ( ! (columns->FieldDecl(i)->FindAttr(ATTR_LOG)) ) continue; - if ( ! Value::IsCompatibleType(columns->FieldType(i)) ) + if ( ! threading::Value::IsCompatibleType(columns->FieldType(i)) ) { reporter->Error("type of field '%s' is not support for logging output", columns->FieldName(i)); @@ -1089,7 +783,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) if ( filter->local || filter->remote ) { - Value** vals = RecordToFilterVals(stream, filter, columns); + threading::Value** vals = RecordToFilterVals(stream, filter, columns); if ( filter->remote ) remote_serializer->SendLogWrite(stream->id, @@ -1125,15 +819,15 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) return true; } -Value* Manager::ValToLogVal(Val* val, BroType* ty) +threading::Value* Manager::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) ty = val->Type(); if ( ! val ) - return new Value(ty->Tag(), false); + return new threading::Value(ty->Tag(), false); - Value* lval = new Value(ty->Tag()); + threading::Value* lval = new threading::Value(ty->Tag()); switch ( lval->type ) { case TYPE_BOOL: @@ -1213,7 +907,7 @@ Value* Manager::ValToLogVal(Val* val, BroType* ty) set = new ListVal(TYPE_INT); lval->val.set_val.size = set->Length(); - lval->val.set_val.vals = new Value* [lval->val.set_val.size]; + lval->val.set_val.vals = new threading::Value* [lval->val.set_val.size]; for ( int i = 0; i < lval->val.set_val.size; i++ ) lval->val.set_val.vals[i] = ValToLogVal(set->Index(i)); @@ -1227,7 +921,7 @@ Value* Manager::ValToLogVal(Val* val, BroType* ty) VectorVal* vec = val->AsVectorVal(); lval->val.vector_val.size = vec->Size(); lval->val.vector_val.vals = - new Value* [lval->val.vector_val.size]; + new threading::Value* [lval->val.vector_val.size]; for ( int i = 0; i < lval->val.vector_val.size; i++ ) { diff --git a/src/logging/Manager.h b/src/logging/Manager.h index f6829b3554..c5d1a9fc2d 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -15,118 +15,6 @@ class RotationTimer; namespace logging { -/** - * Definition of a log file, i.e., one column of a log stream. - */ -struct Field { - string name; //! Name of the field. - TypeTag type; //! Type of the field. - TypeTag subtype; //! Inner type for sets. - - /** - * Constructor. - */ - Field() { subtype = TYPE_VOID; } - - /** - * Copy constructor. - */ - Field(const Field& other) - : name(other.name), type(other.type), subtype(other.subtype) { } - - /** - * Unserializes a field. - * - * @param fmt The serialization format to use. The format handles - * low-level I/O. - * - * @return False if an error occured. - */ - bool Read(SerializationFormat* fmt); - - /** - * Serializes a field. - * - * @param fmt The serialization format to use. The format handles - * low-level I/O. - * - * @return False if an error occured. - */ - bool Write(SerializationFormat* fmt) const; -}; - -/** - * Definition of a log value, i.e., a entry logged by a stream. - * - * This struct essentialy represents a serialization of a Val instance (for - * those Vals supported). - */ -struct Value { - TypeTag type; //! The type of the value. - bool present; //! False for optional record fields that are not set. - - struct set_t { bro_int_t size; Value** vals; }; - typedef set_t vec_t; - - /** - * This union is a subset of BroValUnion, including only the types we - * can log directly. See IsCompatibleType(). - */ - union _val { - bro_int_t int_val; - bro_uint_t uint_val; - uint32 addr_val[NUM_ADDR_WORDS]; - subnet_type subnet_val; - double double_val; - string* string_val; - set_t set_val; - vec_t vector_val; - } val; - - /** - * Constructor. - * - * arg_type: The type of the value. - * - * arg_present: False if the value represents an optional record field - * that is not set. - */ - Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) - : type(arg_type), present(arg_present) {} - - /** - * Destructor. - */ - ~Value(); - - /** - * Unserializes a value. - * - * @param fmt The serialization format to use. The format handles low-level I/O. - * - * @return False if an error occured. - */ - bool Read(SerializationFormat* fmt); - - /** - * Serializes a value. - * - * @param fmt The serialization format to use. The format handles - * low-level I/O. - * - * @return False if an error occured. - */ - bool Write(SerializationFormat* fmt) const; - - /** - * Returns true if the type can be represented by a Value. If - * `atomic_only` is true, will not permit composite types. - */ - static bool IsCompatibleType(BroType* t, bool atomic_only=false); - -private: - Value(const Value& other) { } // Disabled. -}; class WriterBackend; class WriterFrontend; @@ -168,7 +56,7 @@ public: * logging.bif, which just forwards here. */ bool EnableStream(EnumVal* id); - + /** * Disables a log stream. * @@ -265,11 +153,11 @@ protected: // Takes ownership of fields. WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, const Field* const* fields); + int num_fields, const threading::Field* const* fields); // Takes ownership of values.. bool Write(EnumVal* id, EnumVal* writer, string path, - int num_fields, Value** vals); + int num_fields, threading::Value** vals); // Announces all instantiated writers to peer. void SendAllWritersTo(RemoteSerializer::PeerID peer); @@ -282,7 +170,7 @@ protected: void Error(WriterFrontend* writer, const char* msg); // Deletes the values as passed into Write(). - void DeleteVals(int num_fields, Value** vals); + void DeleteVals(int num_fields, threading::Value** vals); private: struct Filter; @@ -292,10 +180,10 @@ private: bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); - Value** RecordToFilterVals(Stream* stream, Filter* filter, + threading::Value** RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns); - Value* ValToLogVal(Val* val, BroType* ty = 0); + threading::Value* ValToLogVal(Val* val, BroType* ty = 0); Stream* FindStream(EnumVal* id); void RemoveDisabledWriters(Stream* stream); void InstallRotationTimer(WriterInfo* winfo); diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 4d2e497b14..3ecc54e240 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -4,9 +4,13 @@ #include "WriterBackend.h" #include "WriterFrontend.h" +#include "../threading/SerializationTypes.h" // Messages sent from backend to frontend (i.e., "OutputMessages"). +using threading::Value; +using threading::Field; + namespace logging { class RotationFinishedMessage : public threading::OutputMessage diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 33271e43f9..9ffa26d0c8 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -55,7 +55,7 @@ public: * * @return False if an error occured. */ - bool Init(string path, int num_fields, const Field* const* fields); + bool Init(string path, int num_fields, const threading::Field* const* fields); /** * Writes one log entry. @@ -72,7 +72,7 @@ public: * * @return False if an error occured. */ - bool Write(int num_fields, int num_writes, Value*** vals); + bool Write(int num_fields, int num_writes, threading::Value*** vals); /** * Sets the buffering status for the writer, assuming the writer @@ -129,7 +129,7 @@ public: /** * Returns the log fields as passed into the constructor. */ - const Field* const * Fields() const { return fields; } + const threading::Field* const * Fields() const { return fields; } /** * Returns the current buffering state. @@ -170,7 +170,7 @@ protected: * implementation should also call Error() to indicate what happened. */ virtual bool DoInit(string path, int num_fields, - const Field* const* fields) = 0; + const threading::Field* const* fields) = 0; /** * Writer-specific output method implementing recording of fone log @@ -182,8 +182,8 @@ protected: * disabled and eventually deleted. When returning false, an * implementation should also call Error() to indicate what happened. */ - virtual bool DoWrite(int num_fields, const Field* const* fields, - Value** vals) = 0; + virtual bool DoWrite(int num_fields, const threading::Field* const* fields, + threading::Value** vals) = 0; /** * Writer-specific method implementing a change of fthe buffering @@ -287,7 +287,7 @@ private: /** * Deletes the values as passed into Write(). */ - void DeleteVals(int num_writes, Value*** vals); + void DeleteVals(int num_writes, threading::Value*** vals); // Frontend that instantiated us. This object must not be access from // this class, it's running in a different thread! @@ -295,7 +295,7 @@ private: string path; // Log path. int num_fields; // Number of log fields. - const Field* const* fields; // Log fields. + const threading::Field* const* fields; // Log fields. bool buffering; // True if buffering is enabled. }; diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 79278870f9..1f7af5a53d 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -1,6 +1,10 @@ #include "WriterFrontend.h" #include "WriterBackend.h" +#include "../threading/SerializationTypes.h" + +using threading::Value; +using threading::Field; namespace logging { diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index e0bc590dfc..56c8885cf9 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -64,7 +64,7 @@ public: * * This method must only be called from the main thread. */ - void Init(string path, int num_fields, const Field* const* fields); + void Init(string path, int num_fields, const threading::Field* const* fields); /** * Write out a record. @@ -86,7 +86,7 @@ public: * * This method must only be called from the main thread. */ - void Write(int num_fields, Value** vals); + void Write(int num_fields, threading::Value** vals); /** * Sets the buffering state. @@ -185,7 +185,7 @@ public: /** * Returns the log fields as passed into the constructor. */ - const Field* const * Fields() const { return fields; } + const threading::Field* const * Fields() const { return fields; } protected: friend class Manager; @@ -198,12 +198,12 @@ protected: string ty_name; // Name of the backend type. Set by the manager. string path; // The log path. int num_fields; // The number of log fields. - const Field* const* fields; // The log fields. + const threading::Field* const* fields; // The log fields. // Buffer for bulk writes. static const int WRITER_BUFFER_SIZE = 50; int write_buffer_pos; // Position of next write in buffer. - Value*** write_buffer; // Buffer of size WRITER_BUFFER_SIZE. + threading::Value*** write_buffer; // Buffer of size WRITER_BUFFER_SIZE. }; } diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 7cc8459e68..fc6832afea 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -6,9 +6,12 @@ #include "../../NetVar.h" #include "Ascii.h" +#include "../../threading/SerializationTypes.h" using namespace logging; using namespace writer; +using threading::Value; +using threading::Field; Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) { diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 4a24aad9b7..6f507aff01 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -20,9 +20,9 @@ public: protected: virtual bool DoInit(string path, int num_fields, - const Field* const* fields); - virtual bool DoWrite(int num_fields, const Field* const* fields, - Value** vals); + const threading::Field* const* fields); + virtual bool DoWrite(int num_fields, const threading::Field* const* fields, + threading::Value** vals); virtual bool DoSetBuf(bool enabled); virtual bool DoRotate(string rotated_path, double open, double close, bool terminating); @@ -31,7 +31,7 @@ protected: private: bool IsSpecial(string path) { return path.find("/dev/") == 0; } - bool DoWriteOne(ODesc* desc, Value* val, const Field* field); + bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field); bool WriteHeaderField(const string& key, const string& value); FILE* file; diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h index 6a62161f49..cce48953d1 100644 --- a/src/logging/writers/None.h +++ b/src/logging/writers/None.h @@ -19,10 +19,10 @@ public: protected: virtual bool DoInit(string path, int num_fields, - const Field* const * fields) { return true; } + const threading::Field* const * fields) { return true; } - virtual bool DoWrite(int num_fields, const Field* const* fields, - Value** vals) { return true; } + virtual bool DoWrite(int num_fields, const threading::Field* const* fields, + threading::Value** vals) { return true; } virtual bool DoSetBuf(bool enabled) { return true; } virtual bool DoRotate(string rotated_path, double open, double close, bool terminating); diff --git a/src/threading/SerializationTypes.cc b/src/threading/SerializationTypes.cc new file mode 100644 index 0000000000..01f0ac84ce --- /dev/null +++ b/src/threading/SerializationTypes.cc @@ -0,0 +1,319 @@ +// See the file "COPYING" in the main distribution directory for copyright. + + +#include "SerializationTypes.h" +#include "../RemoteSerializer.h" + + +using namespace threading; + +bool Field::Read(SerializationFormat* fmt) + { + int t; + int st; + + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); + type = (TypeTag) t; + subtype = (TypeTag) st; + + return success; + } + +bool Field::Write(SerializationFormat* fmt) const + { + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); + } + +Value::~Value() + { + if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC) + && present ) + delete val.string_val; + + if ( type == TYPE_TABLE && present ) + { + for ( int i = 0; i < val.set_val.size; i++ ) + delete val.set_val.vals[i]; + + delete [] val.set_val.vals; + } + + if ( type == TYPE_VECTOR && present ) + { + for ( int i = 0; i < val.vector_val.size; i++ ) + delete val.vector_val.vals[i]; + + delete [] val.vector_val.vals; + } + } + +bool Value::IsCompatibleType(BroType* t, bool atomic_only) + { + if ( ! t ) + return false; + + switch ( t->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_SUBNET: + case TYPE_ADDR: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + return true; + + case TYPE_RECORD: + return ! atomic_only; + + case TYPE_TABLE: + { + if ( atomic_only ) + return false; + + if ( ! t->IsSet() ) + return false; + + return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); + } + + case TYPE_VECTOR: + { + if ( atomic_only ) + return false; + + return IsCompatibleType(t->AsVectorType()->YieldType(), true); + } + + default: + return false; + } + + return false; + } + +bool Value::Read(SerializationFormat* fmt) + { + int ty; + + if ( ! (fmt->Read(&ty, "type") && fmt->Read(&present, "present")) ) + return false; + + type = (TypeTag)(ty); + + if ( ! present ) + return true; + + switch ( type ) { + case TYPE_BOOL: + case TYPE_INT: + return fmt->Read(&val.int_val, "int"); + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + return fmt->Read(&val.uint_val, "uint"); + + case TYPE_SUBNET: + { + uint32 net[4]; + if ( ! (fmt->Read(&net[0], "net0") && + fmt->Read(&net[1], "net1") && + fmt->Read(&net[2], "net2") && + fmt->Read(&net[3], "net3") && + fmt->Read(&val.subnet_val.width, "width")) ) + return false; + +#ifdef BROv6 + val.subnet_val.net[0] = net[0]; + val.subnet_val.net[1] = net[1]; + val.subnet_val.net[2] = net[2]; + val.subnet_val.net[3] = net[3]; +#else + val.subnet_val.net = net[0]; +#endif + return true; + } + + case TYPE_ADDR: + { + uint32 addr[4]; + if ( ! (fmt->Read(&addr[0], "addr0") && + fmt->Read(&addr[1], "addr1") && + fmt->Read(&addr[2], "addr2") && + fmt->Read(&addr[3], "addr3")) ) + return false; + + val.addr_val[0] = addr[0]; +#ifdef BROv6 + val.addr_val[1] = addr[1]; + val.addr_val[2] = addr[2]; + val.addr_val[3] = addr[3]; +#endif + return true; + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return fmt->Read(&val.double_val, "double"); + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + { + val.string_val = new string; + return fmt->Read(val.string_val, "string"); + } + + case TYPE_TABLE: + { + if ( ! fmt->Read(&val.set_val.size, "set_size") ) + return false; + + val.set_val.vals = new Value* [val.set_val.size]; + + for ( int i = 0; i < val.set_val.size; ++i ) + { + val.set_val.vals[i] = new Value; + + if ( ! val.set_val.vals[i]->Read(fmt) ) + return false; + } + + return true; + } + + case TYPE_VECTOR: + { + if ( ! fmt->Read(&val.vector_val.size, "vector_size") ) + return false; + + val.vector_val.vals = new Value* [val.vector_val.size]; + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + val.vector_val.vals[i] = new Value; + + if ( ! val.vector_val.vals[i]->Read(fmt) ) + return false; + } + + return true; + } + + default: + reporter->InternalError("unsupported type %s in Value::Write", type_name(type)); + } + + return false; + } + +bool Value::Write(SerializationFormat* fmt) const + { + if ( ! (fmt->Write((int)type, "type") && + fmt->Write(present, "present")) ) + return false; + + if ( ! present ) + return true; + + switch ( type ) { + case TYPE_BOOL: + case TYPE_INT: + return fmt->Write(val.int_val, "int"); + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + return fmt->Write(val.uint_val, "uint"); + + case TYPE_SUBNET: + { + uint32 net[4]; +#ifdef BROv6 + net[0] = val.subnet_val.net[0]; + net[1] = val.subnet_val.net[1]; + net[2] = val.subnet_val.net[2]; + net[3] = val.subnet_val.net[3]; +#else + net[0] = val.subnet_val.net; + net[1] = net[2] = net[3] = 0; +#endif + return fmt->Write(net[0], "net0") && + fmt->Write(net[1], "net1") && + fmt->Write(net[2], "net2") && + fmt->Write(net[3], "net3") && + fmt->Write(val.subnet_val.width, "width"); + } + + case TYPE_ADDR: + { + uint32 addr[4]; + addr[0] = val.addr_val[0]; +#ifdef BROv6 + addr[1] = val.addr_val[1]; + addr[2] = val.addr_val[2]; + addr[3] = val.addr_val[3]; +#else + addr[1] = addr[2] = addr[3] = 0; +#endif + return fmt->Write(addr[0], "addr0") && + fmt->Write(addr[1], "addr1") && + fmt->Write(addr[2], "addr2") && + fmt->Write(addr[3], "addr3"); + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return fmt->Write(val.double_val, "double"); + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + return fmt->Write(*val.string_val, "string"); + + case TYPE_TABLE: + { + if ( ! fmt->Write(val.set_val.size, "set_size") ) + return false; + + for ( int i = 0; i < val.set_val.size; ++i ) + { + if ( ! val.set_val.vals[i]->Write(fmt) ) + return false; + } + + return true; + } + + case TYPE_VECTOR: + { + if ( ! fmt->Write(val.vector_val.size, "vector_size") ) + return false; + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + if ( ! val.vector_val.vals[i]->Write(fmt) ) + return false; + } + + return true; + } + + default: + reporter->InternalError("unsupported type %s in Value::REad", type_name(type)); + } + + return false; + } + diff --git a/src/threading/SerializationTypes.h b/src/threading/SerializationTypes.h new file mode 100644 index 0000000000..8cae99c117 --- /dev/null +++ b/src/threading/SerializationTypes.h @@ -0,0 +1,126 @@ + +#ifndef THREADING_SERIALIZATIONTYPES_H +#define THREADING_SERIALIZATIONTZPES_H + +#include "../RemoteSerializer.h" + +using namespace std; + +namespace threading { + +/** + * Definition of a log file, i.e., one column of a log stream. + */ +struct Field { + string name; //! Name of the field. + TypeTag type; //! Type of the field. + TypeTag subtype; //! Inner type for sets. + + /** + * Constructor. + */ + Field() { subtype = TYPE_VOID; } + + /** + * Copy constructor. + */ + Field(const Field& other) + : name(other.name), type(other.type), subtype(other.subtype) { } + + /** + * Unserializes a field. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ + bool Read(SerializationFormat* fmt); + + /** + * Serializes a field. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ + bool Write(SerializationFormat* fmt) const; +}; + +/** + * Definition of a log value, i.e., a entry logged by a stream. + * + * This struct essentialy represents a serialization of a Val instance (for + * those Vals supported). + */ +struct Value { + TypeTag type; //! The type of the value. + bool present; //! False for optional record fields that are not set. + + struct set_t { bro_int_t size; Value** vals; }; + typedef set_t vec_t; + + /** + * This union is a subset of BroValUnion, including only the types we + * can log directly. See IsCompatibleType(). + */ + union _val { + bro_int_t int_val; + bro_uint_t uint_val; + uint32 addr_val[NUM_ADDR_WORDS]; + subnet_type subnet_val; + double double_val; + string* string_val; + set_t set_val; + vec_t vector_val; + } val; + + /** + * Constructor. + * + * arg_type: The type of the value. + * + * arg_present: False if the value represents an optional record field + * that is not set. + */ + Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) + : type(arg_type), present(arg_present) {} + + /** + * Destructor. + */ + ~Value(); + + /** + * Unserializes a value. + * + * @param fmt The serialization format to use. The format handles low-level I/O. + * + * @return False if an error occured. + */ + bool Read(SerializationFormat* fmt); + + /** + * Serializes a value. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ + bool Write(SerializationFormat* fmt) const; + + /** + * Returns true if the type can be represented by a Value. If + * `atomic_only` is true, will not permit composite types. + */ + static bool IsCompatibleType(BroType* t, bool atomic_only=false); + +private: + Value(const Value& other) { } // Disabled. +}; + +} + +#endif /* THREADING_SERIALIZATIONTZPES_H */ From 23b2c95644f3a48a3a154843041ae1b016f1dfc1 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 6 Feb 2012 10:57:07 -0800 Subject: [PATCH 15/42] backend does not need friend access to manager --- src/logging/WriterBackend.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 9ffa26d0c8..aa4448f5e2 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -282,8 +282,6 @@ protected: virtual bool DoHeartbeat(double network_time, double current_time); private: - friend class Manager; - /** * Deletes the values as passed into Write(). */ From f76bbf01a4e02fabd55ed38fba28e3acb7ab9036 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 6 Feb 2012 13:15:01 -0800 Subject: [PATCH 16/42] fix CreateBackend function - the way that the right backend was chosen & backends were initialized did not make sense... --- src/logging/Manager.cc | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index add10b3f10..e8d732d84a 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -142,12 +142,12 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) return 0; } - if ( ld->type == type ) - break; - - if ( ! ld->factory ) - // Oops, we can't instantiate this guy. - return 0; + if ( ld->type != type ) + { + // no, didn't find the right one... + ++ld; + continue; + } // If the writer has an init function, call it. if ( ld->init ) @@ -157,17 +157,24 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) // call it again later. ld->init = 0; else + { // Init failed, disable by deleting factory // function. ld->factory = 0; - DBG_LOG(DBG_LOGGING, "failed to init writer class %s", - ld->name); + DBG_LOG(DBG_LOGGING, "failed to init writer class %s", + ld->name); - return false; + return false; + } } - ++ld; + if ( ! ld->factory ) + // Oops, we can't instantiate this guy. + return 0; + + // all done. break. + break; } assert(ld->factory); From e22d396229343b4956d93b14ad9629cde4690dbf Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 6 Feb 2012 13:53:33 -0800 Subject: [PATCH 17/42] typo --- src/threading/SerializationTypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/threading/SerializationTypes.h b/src/threading/SerializationTypes.h index 11ceda929c..8f36402637 100644 --- a/src/threading/SerializationTypes.h +++ b/src/threading/SerializationTypes.h @@ -1,6 +1,6 @@ #ifndef THREADING_SERIALIZATIONTYPES_H -#define THREADING_SERIALIZATIONTZPES_H +#define THREADING_SERIALIZATIONTYPES_H #include "../RemoteSerializer.h" From b8ec653ebf31ffc7c2dffa02614f93b91bb45367 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 12 Feb 2012 09:41:43 -0800 Subject: [PATCH 18/42] Bugfixes. - Data queued at termination wasn't written out completely. - Fixed some race conditions. - Fixing IOSource integration. - Fixing setting thread names on Linux. - Fixing minor leaks. All tests now pass for me on Linux in debug and non-debug compiles. Remaining TODOs: - Needs leak check. - Test on MacOS and FreeBSD. - More testing: - High volume traffic. - Different platforms. --- src/logging/Manager.cc | 9 +++++++++ src/logging/Manager.h | 8 +++++++- src/logging/WriterBackend.cc | 8 ++++++-- src/logging/WriterFrontend.cc | 6 ++++-- src/logging/WriterFrontend.h | 3 --- src/logging/writers/Ascii.cc | 24 ++++++++++++++++++++++-- src/main.cc | 1 + src/threading/BasicThread.cc | 11 +++++++++-- src/threading/BasicThread.h | 2 +- src/threading/Manager.cc | 30 +++++++++++++++++++----------- src/threading/MsgThread.cc | 21 +++++++++++++++++---- src/threading/MsgThread.h | 7 +++++-- 12 files changed, 100 insertions(+), 30 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 6d53ea363f..593766e52a 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -1470,6 +1470,15 @@ bool Manager::Flush(EnumVal* id) return true; } +void Manager::Terminate() + { + for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) + { + if ( *s ) + Flush((*s)->id); + } + } + void Manager::Error(WriterFrontend* writer, const char* msg) { reporter->Error("error with writer for %s: %s", diff --git a/src/logging/Manager.h b/src/logging/Manager.h index f6829b3554..d12fc7e8fe 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -251,6 +251,12 @@ public: */ bool Flush(EnumVal* id); + /** + * Prepares the log manager to terminate. This will flush all log + * stream. + */ + void Terminate(); + protected: friend class WriterFrontend; friend class RotationFinishedMessage; @@ -258,7 +264,7 @@ protected: friend class ::RotationTimer; // Instantiates a new WriterBackend of the given type (note that - // doing so creates a new thread!). + // doing so creates a new thread!). WriterBackend* CreateBackend(WriterFrontend* frontend, bro_int_t type); //// Function also used by the RemoteSerializer. diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 4d2e497b14..fa86fce324 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -57,7 +57,7 @@ using namespace logging; WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread() { - path = ""; + path = ""; num_fields = 0; fields = 0; buffering = true; @@ -109,7 +109,9 @@ bool WriterBackend::Init(string arg_path, int arg_num_fields, const Field* const num_fields = arg_num_fields; fields = arg_fields; - SetName(frontend->Name()); + string name = Fmt("%s/%s", path.c_str(), frontend->Name().c_str()); + + SetName(name); if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) { @@ -229,6 +231,8 @@ bool WriterBackend::Finish() bool WriterBackend::DoHeartbeat(double network_time, double current_time) { + MsgThread::DoHeartbeat(network_time, current_time); + SendOut(new FlushWriteBufferMessage(frontend)); return true; diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 79278870f9..0a8ff4b09d 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -1,4 +1,6 @@ +#include "Net.h" + #include "WriterFrontend.h" #include "WriterBackend.h" @@ -155,8 +157,8 @@ void WriterFrontend::Write(int num_fields, Value** vals) write_buffer[write_buffer_pos++] = vals; - if ( write_buffer_pos >= WRITER_BUFFER_SIZE || ! buf ) - // Buffer full (or no bufferin desired). + if ( write_buffer_pos >= WRITER_BUFFER_SIZE || ! buf || terminating ) + // Buffer full (or no bufferin desired or termiating). FlushWriteBuffer(); } diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index e0bc590dfc..a1a1e2b86a 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -30,9 +30,6 @@ public: * frontend will internally instantiate a WriterBackend of the * corresponding type. * - * name: A descriptive name for the backend wroter type (e.g., \c - * Ascii). - * * Frontends must only be instantiated by the main thread. */ WriterFrontend(bro_int_t type); diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 7cc8459e68..5429bf0b97 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -171,14 +171,34 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) break; case TYPE_SUBNET: - desc->Add(dotted_addr(val->val.subnet_val.net)); + { + // FIXME: This will be replaced with string(addr) once the + // IPV6 branch is merged in. + uint32_t addr = ntohl(val->val.subnet_val.net); + char buf[32]; + snprintf(buf, sizeof(buf), "%d.%d.%d.%d", + addr >> 24, (addr >> 16) & 0xff, + (addr >> 8) & 0xff, addr & 0xff); + + desc->Add(buf); desc->Add("/"); desc->Add(val->val.subnet_val.width); break; + } case TYPE_ADDR: - desc->Add(dotted_addr(val->val.addr_val)); + { + // FIXME: This will be replaced with string(addr) once the + // IPV6 branch is merged in. + uint32_t addr = ntohl(*val->val.addr_val); + char buf[32]; + snprintf(buf, sizeof(buf), "%d.%d.%d.%d", + addr >> 24, (addr >> 16) & 0xff, + (addr >> 8) & 0xff, addr & 0xff); + + desc->Add(buf); break; + } case TYPE_TIME: case TYPE_INTERVAL: diff --git a/src/main.cc b/src/main.cc index e224910db4..c101e54e74 100644 --- a/src/main.cc +++ b/src/main.cc @@ -290,6 +290,7 @@ void terminate_bro() if ( remote_serializer ) remote_serializer->LogStats(); + log_mgr->Terminate(); thread_mgr->Terminate(); delete timer_mgr; diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 4d51c3c4e4..51c4f7a3bc 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -2,9 +2,14 @@ #include #include +#include "config.h" #include "BasicThread.h" #include "Manager.h" +#ifdef HAVE_LINUX +#include +#endif + using namespace threading; uint64_t BasicThread::thread_counter = 0; @@ -25,6 +30,8 @@ BasicThread::BasicThread() BasicThread::~BasicThread() { + if ( buf ) + free(buf); } void BasicThread::SetName(const string& arg_name) @@ -35,8 +42,8 @@ void BasicThread::SetName(const string& arg_name) void BasicThread::SetOSName(const string& name) { -#ifdef LINUX - pthread_setname_np(pthread_self(), name.c_str()); +#ifdef HAVE_LINUX + prctl(PR_SET_NAME, name.c_str(), 0, 0, 0); #endif #ifdef __APPLE__ diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index 6d2f739620..cc87ae03bc 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -82,7 +82,7 @@ public: void Stop(); /** - * Returns true if Terminate() has been called. + * Returns true if Stop() has been called. * * This method is safe to call from any thread. */ diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d07311bbe8..d008d2e5e8 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -7,9 +7,10 @@ Manager::Manager() { DBG_LOG(DBG_THREADING, "Creating thread manager ..."); - did_process = false; + did_process = true; next_beat = 0; terminating = false; + idle = false; } Manager::~Manager() @@ -41,6 +42,7 @@ void Manager::Terminate() all_threads.clear(); msg_threads.clear(); + idle = true; terminating = false; } @@ -70,18 +72,22 @@ void Manager::GetFds(int* read, int* write, int* except) double Manager::NextTimestamp(double* network_time) { - if ( did_process || ! next_beat == 0 ) - // If we had something to process last time (or haven't had a - // chance to check yet), we want to check for more asap. + if ( ::network_time && ! next_beat ) + next_beat = ::network_time + HEART_BEAT_INTERVAL; + +// fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat); + + if ( did_process || ::network_time > next_beat ) + // If we had something to process last time (or out heartbeat + // is due), we want to check for more asap. return timer_mgr->Time(); - // Else we assume we don't have much to do at all and wait for the next heart beat. - return next_beat; + return -1.0; } void Manager::Process() { - bool do_beat = (next_beat == 0 || network_time >= next_beat); + bool do_beat = (next_beat && network_time > next_beat); did_process = false; @@ -90,14 +96,17 @@ void Manager::Process() MsgThread* t = *i; if ( do_beat ) + { t->Heartbeat(); + next_beat = 0; + } if ( ! t->HasOut() ) continue; Message* msg = t->RetrieveOut(); - if ( msg->Process() ) + if ( msg->Process() && network_time ) did_process = true; else @@ -110,15 +119,14 @@ void Manager::Process() delete msg; } - if ( do_beat ) - next_beat = network_time + HEART_BEAT_INTERVAL; +// fprintf(stderr, "P %.6f %.6f do_beat=%d did_process=%d next_next=%.6f\n", network_time, timer_mgr->Time(), do_beat, (int)did_process, next_beat); } const threading::Manager::msg_stats_list& threading::Manager::GetMsgThreadStats() { stats.clear(); - for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) { MsgThread* t = *i; diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index f41b20ddf9..b7782b9a05 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -142,12 +142,19 @@ void MsgThread::OnStop() void MsgThread::Heartbeat() { SendIn(new HeartbeatMessage(this, network_time, current_time())); + } - string name = Fmt("%s (%d/%d)", name.c_str(), - cnt_sent_in - queue_in.Size(), - cnt_sent_out - queue_out.Size()); +bool MsgThread::DoHeartbeat(double network_time, double current_time) + { + string n = Name(); - SetOSName(name.c_str()); + n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", n.c_str(), + cnt_sent_in - queue_in.Size(), + cnt_sent_out - queue_out.Size()); + + SetOSName(n.c_str()); + + return true; } void MsgThread::Info(const char* msg) @@ -197,7 +204,10 @@ void MsgThread::Debug(DebugStream stream, const char* msg) void MsgThread::SendIn(BasicInputMessage* msg, bool force) { if ( Terminating() && ! force ) + { + delete msg; return; + } DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str()); @@ -209,7 +219,10 @@ void MsgThread::SendIn(BasicInputMessage* msg, bool force) void MsgThread::SendOut(BasicOutputMessage* msg, bool force) { if ( Terminating() && ! force ) + { + delete msg; return; + } queue_out.Put(msg); diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 459ac6c603..28c7690dfa 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -184,7 +184,10 @@ protected: * This is method is called regularly by the threading::Manager. * * Can be overriden in derived classed to hook into the heart beat, - * but must call the parent implementation. + * but must call the parent implementation. Note that this method is + * always called by the main thread and must not access data of the + * child thread directly. See DoHeartbeat() if you want to do + * something on the child-side. */ virtual void Heartbeat(); @@ -206,7 +209,7 @@ protected: * current_time: Wall clock when the heartbeat was trigger by the * main thread. */ - virtual bool DoHeartbeat(double network_time, double current_time) { return true; } + virtual bool DoHeartbeat(double network_time, double current_time); private: /** From 7fcb7b5f17a966b6f384f7c34aa234fb60a45483 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 12 Feb 2012 13:04:47 -0800 Subject: [PATCH 19/42] Save CPU when idle. This needs a bit more testing. It may also with the general problem of high CPU usage with low traffic. --- src/Net.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Net.cc b/src/Net.cc index 2d8ee85353..d2b505544e 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -454,6 +454,7 @@ void net_run() // date on timers and events. network_time = ct; expire_timers(); + usleep(1); // Just yield. } } From 1f8b299aaf37c5d03994c8ed9f6f7acaaba9a98b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 14 Feb 2012 10:12:09 -0800 Subject: [PATCH 20/42] Shortening file names a bit. --- src/Attr.cc | 2 +- src/CMakeLists.txt | 2 +- src/RemoteSerializer.cc | 2 +- src/logging/Manager.cc | 4 ++-- src/logging/WriterBackend.cc | 2 +- src/logging/WriterFrontend.cc | 2 +- src/logging/writers/Ascii.cc | 4 ++-- src/threading/{SerializationTypes.cc => SerialTypes.cc} | 2 +- src/threading/{SerializationTypes.h => SerialTypes.h} | 0 9 files changed, 10 insertions(+), 10 deletions(-) rename src/threading/{SerializationTypes.cc => SerialTypes.cc} (99%) rename src/threading/{SerializationTypes.h => SerialTypes.h} (100%) diff --git a/src/Attr.cc b/src/Attr.cc index 40c6c1a75c..82d9c9ddc7 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -5,7 +5,7 @@ #include "Attr.h" #include "Expr.h" #include "Serializer.h" -#include "threading/SerializationTypes.h" +#include "threading/SerialTypes.h" const char* attr_name(attr_tag t) { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7a3cc4babf..67d82c577a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -411,7 +411,7 @@ set(bro_SRCS threading/BasicThread.cc threading/Manager.cc threading/MsgThread.cc - threading/SerializationTypes.cc + threading/SerialTypes.cc logging/Manager.cc logging/WriterBackend.cc diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index ba2598c018..948dfddaff 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -184,7 +184,7 @@ #include "File.h" #include "Conn.h" #include "Reporter.h" -#include "threading/SerializationTypes.h" +#include "threading/SerialTypes.h" #include "logging/Manager.h" extern "C" { diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 5f9f1c4222..6078a1e566 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -7,6 +7,8 @@ #include "../NetVar.h" #include "../Net.h" +#include "threading/SerialTypes.h" + #include "Manager.h" #include "WriterFrontend.h" #include "WriterBackend.h" @@ -14,8 +16,6 @@ #include "writers/Ascii.h" #include "writers/None.h" -#include "threading/SerializationTypes.h" - using namespace logging; using threading::Value; using threading::Field; diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 0ffedf073c..f4e48ebaef 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -1,10 +1,10 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "util.h" +#include "threading/SerialTypes.h" #include "WriterBackend.h" #include "WriterFrontend.h" -#include "../threading/SerializationTypes.h" // Messages sent from backend to frontend (i.e., "OutputMessages"). diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index b0e780f27d..02f1a188d8 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -1,9 +1,9 @@ #include "Net.h" +#include "threading/SerialTypes.h" #include "WriterFrontend.h" #include "WriterBackend.h" -#include "../threading/SerializationTypes.h" using threading::Value; using threading::Field; diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index c974877175..0a101feb79 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -3,10 +3,10 @@ #include #include -#include "../../NetVar.h" +#include "NetVar.h" +#include "threading/SerialTypes.h" #include "Ascii.h" -#include "../../threading/SerializationTypes.h" using namespace logging; using namespace writer; diff --git a/src/threading/SerializationTypes.cc b/src/threading/SerialTypes.cc similarity index 99% rename from src/threading/SerializationTypes.cc rename to src/threading/SerialTypes.cc index f74de6ce57..f35d1fc6b0 100644 --- a/src/threading/SerializationTypes.cc +++ b/src/threading/SerialTypes.cc @@ -1,7 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "SerializationTypes.h" +#include "SerialTypes.h" #include "../RemoteSerializer.h" diff --git a/src/threading/SerializationTypes.h b/src/threading/SerialTypes.h similarity index 100% rename from src/threading/SerializationTypes.h rename to src/threading/SerialTypes.h From bc6ebe53cc40f28aebff0c744e33e92fdc6328ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 16 Feb 2012 11:31:30 -0800 Subject: [PATCH 21/42] remove unnecessary error function from manger --- src/logging/Manager.cc | 6 ------ src/logging/Manager.h | 3 --- 2 files changed, 9 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 5ab2e5bc77..b47a2f9eff 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -1181,12 +1181,6 @@ void Manager::Terminate() } } -void Manager::Error(WriterFrontend* writer, const char* msg) - { - reporter->Error("error with writer for %s: %s", - writer->Path().c_str(), msg); - } - // Timer which on dispatching rotates the filter. class RotationTimer : public Timer { public: diff --git a/src/logging/Manager.h b/src/logging/Manager.h index b65d22e3c0..d931bfaef8 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -172,9 +172,6 @@ protected: bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, double open, double close, bool terminating); - // Reports an error for the given writer. - void Error(WriterFrontend* writer, const char* msg); - // Deletes the values as passed into Write(). void DeleteVals(int num_fields, threading::Value** vals); From 14916b43f634c9b0dc989c8b89ffcd272951b633 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 27 Feb 2012 21:39:56 -0800 Subject: [PATCH 22/42] Readding deleted functions. These are needed in debug mode in turns out. --- src/net_util.cc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/net_util.cc b/src/net_util.cc index a65afb1a25..f78246c634 100644 --- a/src/net_util.cc +++ b/src/net_util.cc @@ -147,6 +147,26 @@ char addr_to_class(uint32 addr) return 'A'; } +const char* fmt_conn_id(const IPAddr& src_addr, uint32 src_port, + const IPAddr& dst_addr, uint32 dst_port) + { + static char buffer[512]; + + safe_snprintf(buffer, sizeof(buffer), "%s:%d > %s:%d", + string(src_addr).c_str(), src_port, + string(dst_addr).c_str(), dst_port); + + return buffer; + } + +const char* fmt_conn_id(const uint32* src_addr, uint32 src_port, + const uint32* dst_addr, uint32 dst_port) + { + IPAddr src(IPAddr::IPv6, src_addr, IPAddr::Network); + IPAddr dst(IPAddr::IPv6, dst_addr, IPAddr::Network); + return fmt_conn_id(src, src_port, dst, dst_port); + } + uint32 extract_uint32(const u_char* data) { uint32 val; From edc9bb14af6b2fe56318e5dbb07847d4413cc408 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 28 Feb 2012 15:12:35 -0800 Subject: [PATCH 23/42] Making exchange of addresses between threads thread-safe. As we can't use the IPAddr class (because it's not thread-safe), this involved a bit manual address manipulation and also shuffling some things around a bit. Not fully working yet, the tests for remote logging still fail. --- src/Anon.cc | 2 +- src/CompHash.cc | 2 +- src/ConnCompressor.cc | 16 ++++---- src/DCE_RPC.cc | 4 +- src/DNS_Mgr.cc | 4 +- src/Desc.cc | 10 +++++ src/Desc.h | 7 ++-- src/IPAddr.cc | 4 +- src/IPAddr.h | 45 ++++++++++++++++++-- src/RemoteSerializer.cc | 4 +- src/RuleMatcher.cc | 2 +- src/SerializationFormat.cc | 80 +++++++++++++++++++++++++++++++++++- src/SerializationFormat.h | 15 +++++++ src/Val.cc | 26 ++++++++---- src/Val.h | 8 +--- src/bro.bif | 22 +++++----- src/logging/Manager.cc | 4 +- src/logging/WriterBackend.cc | 36 ++++++++++++++++ src/logging/WriterBackend.h | 15 +++++++ src/logging/writers/Ascii.cc | 4 +- src/net_util.cc | 4 +- src/net_util.h | 10 ++++- src/threading/SerialTypes.cc | 63 +++++++++++++++++++--------- src/threading/SerialTypes.h | 22 ++++++++-- 24 files changed, 325 insertions(+), 84 deletions(-) diff --git a/src/Anon.cc b/src/Anon.cc index 4c4146ac3e..d2a28a0e08 100644 --- a/src/Anon.cc +++ b/src/Anon.cc @@ -154,7 +154,7 @@ void AnonymizeIPAddr_A50::init() int AnonymizeIPAddr_A50::PreservePrefix(ipaddr32_t input, int num_bits) { DEBUG_MSG("%s/%d\n", - IPAddr(IPAddr::IPv4, &input, IPAddr::Network).AsString().c_str(), + IPAddr(IPv4, &input, IPAddr::Network).AsString().c_str(), num_bits); if ( ! before_anonymization ) diff --git a/src/CompHash.cc b/src/CompHash.cc index ae0e082216..86677f9719 100644 --- a/src/CompHash.cc +++ b/src/CompHash.cc @@ -709,7 +709,7 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, const uint32* const kp = AlignType(kp0); kp1 = reinterpret_cast(kp+4); - IPAddr addr(IPAddr::IPv6, kp, IPAddr::Network); + IPAddr addr(IPv6, kp, IPAddr::Network); switch ( tag ) { case TYPE_ADDR: diff --git a/src/ConnCompressor.cc b/src/ConnCompressor.cc index 29e24457f5..e2b297220a 100644 --- a/src/ConnCompressor.cc +++ b/src/ConnCompressor.cc @@ -236,7 +236,7 @@ Connection* ConnCompressor::NextPacket(double t, HashKey* key, const IP_Hdr* ip, } else if ( ip->SrcAddr() == - IPAddr(IPAddr::IPv6, SrcAddr(pending), IPAddr::Network) && + IPAddr(IPv6, SrcAddr(pending), IPAddr::Network) && tp->th_sport == SrcPort(pending) ) // Another packet from originator. tc = NextFromOrig(pending, t, key, ip, tp); @@ -508,8 +508,8 @@ Connection* ConnCompressor::Instantiate(HashKey* key, PendingConn* pending) { // Instantantiate a Connection. ConnID conn_id; - conn_id.src_addr = IPAddr(IPAddr::IPv6, SrcAddr(pending), IPAddr::Network); - conn_id.dst_addr = IPAddr(IPAddr::IPv6, DstAddr(pending), IPAddr::Network); + conn_id.src_addr = IPAddr(IPv6, SrcAddr(pending), IPAddr::Network); + conn_id.dst_addr = IPAddr(IPv6, DstAddr(pending), IPAddr::Network); conn_id.src_port = SrcPort(pending); conn_id.dst_port = DstPort(pending); @@ -608,7 +608,7 @@ void ConnCompressor::PktHdrToPendingConn(double time, const HashKey* key, memcpy(&c->key, key->Key(), key->Size()); c->hash = key->Hash(); - IPAddr ip1(IPAddr::IPv6, c->key.ip1, IPAddr::Network); + IPAddr ip1(IPv6, c->key.ip1, IPAddr::Network); c->ip1_is_src = ip1 == ip->SrcAddr() && c->key.port1 == tp->th_sport; c->time = time; @@ -658,10 +658,10 @@ const IP_Hdr* ConnCompressor::PendingConnToPacket(const PendingConn* c) tp->th_urp = 0; } - IPAddr ip1(IPAddr::IPv6, c->key.ip1, IPAddr::Network); - IPAddr ip2(IPAddr::IPv6, c->key.ip2, IPAddr::Network); - if ( ip1.GetFamily() == IPAddr::IPv6 || - ip2.GetFamily() == IPAddr::IPv6 ) + IPAddr ip1(IPv6, c->key.ip1, IPAddr::Network); + IPAddr ip2(IPv6, c->key.ip2, IPAddr::Network); + if ( ip1.GetFamily() == IPv6 || + ip2.GetFamily() == IPv6 ) reporter->InternalError("IPv6 snuck into connection compressor"); else { diff --git a/src/DCE_RPC.cc b/src/DCE_RPC.cc index 88cdb48e80..21cb3be9a0 100644 --- a/src/DCE_RPC.cc +++ b/src/DCE_RPC.cc @@ -137,7 +137,7 @@ static bool is_mapped_dce_rpc_endpoint(const dce_rpc_endpoint_addr& addr) bool is_mapped_dce_rpc_endpoint(const ConnID* id, TransportProto proto) { - if ( id->dst_addr.GetFamily() == IPAddr::IPv6 ) + if ( id->dst_addr.GetFamily() == IPv6 ) // TODO: Does the protocol support v6 addresses? #773 return false; @@ -414,7 +414,7 @@ void DCE_RPC_Session::DeliverEpmapperMapResponse( case binpac::DCE_RPC_Simple::EPM_PROTOCOL_IP: uint32 hostip = floor->rhs()->data()->ip(); - mapped.addr.addr = IPAddr(IPAddr::IPv4, &hostip, IPAddr::Host); + mapped.addr.addr = IPAddr(IPv4, &hostip, IPAddr::Host); break; } } diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index a80de42704..9e65d3c9a9 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -321,10 +321,10 @@ void DNS_Mapping::Init(struct hostent* h) addrs = new IPAddr[num_addrs]; for ( int i = 0; i < num_addrs; ++i ) if ( h->h_addrtype == AF_INET ) - addrs[i] = IPAddr(IPAddr::IPv4, (uint32*)h->h_addr_list[i], + addrs[i] = IPAddr(IPv4, (uint32*)h->h_addr_list[i], IPAddr::Network); else if ( h->h_addrtype == AF_INET6 ) - addrs[i] = IPAddr(IPAddr::IPv6, (uint32*)h->h_addr_list[i], + addrs[i] = IPAddr(IPv6, (uint32*)h->h_addr_list[i], IPAddr::Network); } else diff --git a/src/Desc.cc b/src/Desc.cc index 12b4a524eb..9d94321427 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -157,6 +157,16 @@ void ODesc::Add(double d) } } +void ODesc::Add(const IPAddr& addr) + { + Add(addr.AsString()); + } + +void ODesc::Add(const IPPrefix& prefix) + { + Add(prefix.AsString()); + } + void ODesc::AddCS(const char* s) { int n = strlen(s); diff --git a/src/Desc.h b/src/Desc.h index 9f8d7ab109..9c60c68106 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -8,7 +8,6 @@ #include #include "BroString.h" -#include "IPAddr.h" typedef enum { DESC_READABLE, @@ -23,6 +22,8 @@ typedef enum { } desc_style; class BroFile; +class IPAddr; +class IPPrefix; class ODesc { public: @@ -76,8 +77,8 @@ public: void Add(int64 i); void Add(uint64 u); void Add(double d); - void Add(const IPAddr& addr) { Add(addr.AsString()); } - void Add(const IPPrefix& prefix) { Add(prefix.AsString()); } + void Add(const IPAddr& addr); + void Add(const IPPrefix& prefix); // Add s as a counted string. void AddCS(const char* s); diff --git a/src/IPAddr.cc b/src/IPAddr.cc index ff124025f9..29c8f1b8cf 100644 --- a/src/IPAddr.cc +++ b/src/IPAddr.cc @@ -250,7 +250,7 @@ IPPrefix::IPPrefix(const in6_addr& in6, uint8_t length) IPPrefix::IPPrefix(const IPAddr& addr, uint8_t length) : prefix(addr) { - if ( prefix.GetFamily() == IPAddr::IPv4 ) + if ( prefix.GetFamily() == IPv4 ) { if ( length > 32 ) reporter->InternalError("Bad IPAddr(v4) IPPrefix length : %d", @@ -275,7 +275,7 @@ string IPPrefix::AsString() const { char l[16]; - if ( prefix.GetFamily() == IPAddr::IPv4 ) + if ( prefix.GetFamily() == IPv4 ) modp_uitoa10(length - 96, l); else modp_uitoa10(length, l); diff --git a/src/IPAddr.h b/src/IPAddr.h index f0c0ac12c8..67d6f2112e 100644 --- a/src/IPAddr.h +++ b/src/IPAddr.h @@ -10,6 +10,8 @@ #include "BroString.h" #include "Hash.h" #include "util.h" +#include "Type.h" +#include "threading/SerialTypes.h" struct ConnID; class ExpectedConn; @@ -25,7 +27,7 @@ public: /** * Address family. */ - enum Family { IPv4, IPv6 }; + typedef IPFamily Family; /** * Byte order. @@ -318,14 +320,19 @@ public: return memcmp(&addr1.in6, &addr2.in6, sizeof(in6_addr)) < 0; } + /** Converts the address into the type used internally by the + * inter-thread communication. + */ + void ConvertToThreadingValue(threading::Value::addr_t* v) const; + friend HashKey* BuildConnIDHashKey(const ConnID& id); friend HashKey* BuildExpectedConnHashKey(const ExpectedConn& c); - friend class IPPrefix; - unsigned int MemoryAllocation() const { return padded_sizeof(*this); } private: + friend class IPPrefix; + /** * Initializes an address instance from a string representation. * @@ -384,6 +391,25 @@ inline bool IPAddr::IsLoopback() const && (in6.s6_addr[14] == 0) && (in6.s6_addr[15] == 1)); } +inline void IPAddr::ConvertToThreadingValue(threading::Value::addr_t* v) const + { + v->family = GetFamily(); + + switch ( v->family ) { + + case IPv4: + CopyIPv4(&v->in.in4); + return; + + case IPv6: + CopyIPv6(&v->in.in6); + return; + + // Can't be reached. + abort(); + } + } + /** * Returns a hash key for a given ConnID. Passes ownership to caller. */ @@ -459,7 +485,7 @@ public: */ uint8_t Length() const { - return prefix.GetFamily() == IPAddr::IPv4 ? length - 96 : length; + return prefix.GetFamily() == IPv4 ? length - 96 : length; } /** @@ -497,6 +523,8 @@ public: */ string AsString() const; + /** Converts the address into the type used internally by the inter-thread communicastion. + */ operator std::string() const { return AsString(); } /** @@ -516,6 +544,15 @@ public: return new HashKey(&key, sizeof(key)); } + /** Converts the prefix into the type used internally by the + * inter-thread communication. + */ + void ConvertToThreadingValue(threading::Value::subnet_t* v) const + { + v->length = length; + prefix.ConvertToThreadingValue(&v->prefix); + } + unsigned int MemoryAllocation() const { return padded_sizeof(*this); } /** diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 0d89e5ce99..4b8f527f2b 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -681,7 +681,7 @@ RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip, if ( ! initialized ) reporter->InternalError("remote serializer not initialized"); - if ( ip.GetFamily() == IPAddr::IPv6 ) + if ( ip.GetFamily() == IPv6 ) Error("inter-Bro communication not supported over IPv6"); const uint32* bytes; @@ -1238,7 +1238,7 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl) if ( ! initialized ) reporter->InternalError("remote serializer not initialized"); - if ( ip.GetFamily() == IPAddr::IPv6 ) + if ( ip.GetFamily() == IPv6 ) Error("inter-Bro communication not supported over IPv6"); const uint32* bytes; diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index a90bc83293..da12b1b679 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -1082,7 +1082,7 @@ static bool val_to_maskedval(Val* v, maskedvalue_list* append_to) bool is_v4_mask = m[0] == 0xffffffff && m[1] == m[0] && m[2] == m[0]; - if ( v->AsSubNet().Prefix().GetFamily() == IPAddr::IPv4 && + if ( v->AsSubNet().Prefix().GetFamily() == IPv4 && is_v4_mask ) { mval->val = ntohl(*n); diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index af3d9b44c2..ef2dc80cd7 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -250,9 +250,9 @@ bool BinarySerializationFormat::Read(IPAddr* addr, const char* tag) } if ( n == 1 ) - *addr = IPAddr(IPAddr::IPv4, raw, IPAddr::Network); + *addr = IPAddr(IPv4, raw, IPAddr::Network); else - *addr = IPAddr(IPAddr::IPv6, raw, IPAddr::Network); + *addr = IPAddr(IPv6, raw, IPAddr::Network); return true; } @@ -269,6 +269,33 @@ bool BinarySerializationFormat::Read(IPPrefix* prefix, const char* tag) return true; } +bool BinarySerializationFormat::Read(struct in_addr* addr, const char* tag) + { + uint32_t* bytes = (uint32_t*) &addr->s_addr; + + if ( ! Read(&bytes[0], "addr4") ) + return false; + + bytes[0] = htonl(bytes[0]); + return true; + } + +bool BinarySerializationFormat::Read(struct in6_addr* addr, const char* tag) + { + uint32_t* bytes = (uint32_t*) &addr->s6_addr; + + for ( int i = 0; i < 4; ++i ) + { + if ( ! Read(&bytes[i], "addr6-part") ) + return false; + + bytes[i] = htonl(bytes[i]); + } + + return true; + } + + bool BinarySerializationFormat::Write(char v, const char* tag) { DBG_LOG(DBG_SERIAL, "Write char %s [%s]", fmt_bytes(&v, 1), tag); @@ -362,6 +389,31 @@ bool BinarySerializationFormat::Write(const IPPrefix& prefix, const char* tag) return Write(prefix.Prefix(), "prefix") && Write(prefix.Length(), "width"); } +bool BinarySerializationFormat::Write(struct in_addr& addr, const char* tag) + { + const uint32_t* bytes; + bytes = (uint32_t*) &addr.s_addr; + + if ( ! Write(ntohl(bytes[0]), "addr4") ) + return false; + + return true; + } + +bool BinarySerializationFormat::Write(struct in6_addr& addr, const char* tag) + { + const uint32_t* bytes; + bytes = (uint32_t*) &addr.s6_addr; + + for ( int i = 0; i < 4; ++i ) + { + if ( ! Write(ntohl(bytes[i]), "addr6-part") ) + return false; + } + + return true; + } + bool BinarySerializationFormat::WriteOpenTag(const char* tag) { return true; @@ -464,6 +516,18 @@ bool XMLSerializationFormat::Read(IPPrefix* prefix, const char* tag) return false; } +bool XMLSerializationFormat::Read(struct in_addr* addr, const char* tag) + { + reporter->InternalError("no reading of xml"); + return false; + } + +bool XMLSerializationFormat::Read(struct in6_addr* addr, const char* tag) + { + reporter->InternalError("no reading of xml"); + return false; + } + bool XMLSerializationFormat::Write(char v, const char* tag) { return WriteElem(tag, "char", &v, 1); @@ -556,6 +620,18 @@ bool XMLSerializationFormat::Write(const IPPrefix& prefix, const char* tag) return false; } +bool XMLSerializationFormat::Write(struct in_addr& addr, const char* tag) + { + reporter->InternalError("XML output of in_addr not implemented"); + return false; + } + +bool XMLSerializationFormat::Write(struct in6_addr& addr, const char* tag) + { + reporter->InternalError("XML output of in6_addr not implemented"); + return false; + } + bool XMLSerializationFormat::WriteEncodedString(const char* s, int len) { while ( len-- ) diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index f5eb77c608..ba5ad195a2 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -9,6 +9,9 @@ using namespace std; #include "util.h" +class IPAddr; +class IPPrefix; + // Abstract base class. class SerializationFormat { public: @@ -30,6 +33,8 @@ public: virtual bool Read(string* s, const char* tag) = 0; virtual bool Read(IPAddr* addr, const char* tag) = 0; virtual bool Read(IPPrefix* prefix, const char* tag) = 0; + virtual bool Read(struct in_addr* addr, const char* tag) = 0; + virtual bool Read(struct in6_addr* addr, const char* tag) = 0; // Returns number of raw bytes read since last call to StartRead(). int BytesRead() const { return bytes_read; } @@ -54,6 +59,8 @@ public: virtual bool Write(const string& s, const char* tag) = 0; virtual bool Write(const IPAddr& addr, const char* tag) = 0; virtual bool Write(const IPPrefix& prefix, const char* tag) = 0; + virtual bool Write(struct in_addr& addr, const char* tag) = 0; + virtual bool Write(struct in6_addr& addr, const char* tag) = 0; virtual bool WriteOpenTag(const char* tag) = 0; virtual bool WriteCloseTag(const char* tag) = 0; @@ -96,6 +103,8 @@ public: virtual bool Read(string* s, const char* tag); virtual bool Read(IPAddr* addr, const char* tag); virtual bool Read(IPPrefix* prefix, const char* tag); + virtual bool Read(struct in_addr* addr, const char* tag); + virtual bool Read(struct in6_addr* addr, const char* tag); virtual bool Write(int v, const char* tag); virtual bool Write(uint16 v, const char* tag); virtual bool Write(uint32 v, const char* tag); @@ -109,6 +118,8 @@ public: virtual bool Write(const string& s, const char* tag); virtual bool Write(const IPAddr& addr, const char* tag); virtual bool Write(const IPPrefix& prefix, const char* tag); + virtual bool Write(struct in_addr& addr, const char* tag); + virtual bool Write(struct in6_addr& addr, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -133,6 +144,8 @@ public: virtual bool Write(const string& s, const char* tag); virtual bool Write(const IPAddr& addr, const char* tag); virtual bool Write(const IPPrefix& prefix, const char* tag); + virtual bool Write(struct in_addr& addr, const char* tag); + virtual bool Write(struct in6_addr& addr, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -150,6 +163,8 @@ public: virtual bool Read(string* s, const char* tag); virtual bool Read(IPAddr* addr, const char* tag); virtual bool Read(IPPrefix* prefix, const char* tag); + virtual bool Read(struct in_addr* addr, const char* tag); + virtual bool Read(struct in6_addr* addr, const char* tag); private: // Encodes non-printable characters. diff --git a/src/Val.cc b/src/Val.cc index db6e9eb23a..83bbc59b9d 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -606,7 +606,7 @@ ID* MutableVal::Bind() const ip = htonl(0x7f000001); // 127.0.0.1 safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#", - IPAddr(IPAddr::IPv4, &ip, IPAddr::Network)->AsString().c_str(), + IPAddr(IPv4, &ip, IPAddr::Network)->AsString().c_str(), getpid()); #else safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#", host, getpid()); @@ -864,12 +864,12 @@ AddrVal::AddrVal(const char* text) : Val(TYPE_ADDR) AddrVal::AddrVal(uint32 addr) : Val(TYPE_ADDR) { // ### perhaps do gethostbyaddr here? - val.addr_val = new IPAddr(IPAddr::IPv4, &addr, IPAddr::Network); + val.addr_val = new IPAddr(IPv4, &addr, IPAddr::Network); } AddrVal::AddrVal(const uint32 addr[4]) : Val(TYPE_ADDR) { - val.addr_val = new IPAddr(IPAddr::IPv6, addr, IPAddr::Network); + val.addr_val = new IPAddr(IPv6, addr, IPAddr::Network); } AddrVal::AddrVal(const IPAddr& addr) : Val(TYPE_ADDR) @@ -889,7 +889,7 @@ unsigned int AddrVal::MemoryAllocation() const Val* AddrVal::SizeVal() const { - if ( val.addr_val->GetFamily() == IPAddr::IPv4 ) + if ( val.addr_val->GetFamily() == IPv4 ) return new Val(32, TYPE_COUNT); else return new Val(128, TYPE_COUNT); @@ -933,13 +933,13 @@ SubNetVal::SubNetVal(const char* text, int width) : Val(TYPE_SUBNET) SubNetVal::SubNetVal(uint32 addr, int width) : Val(TYPE_SUBNET) { - IPAddr a(IPAddr::IPv4, &addr, IPAddr::Network); + IPAddr a(IPv4, &addr, IPAddr::Network); val.subnet_val = new IPPrefix(a, width); } SubNetVal::SubNetVal(const uint32* addr, int width) : Val(TYPE_SUBNET) { - IPAddr a(IPAddr::IPv6, addr, IPAddr::Network); + IPAddr a(IPv6, addr, IPAddr::Network); val.subnet_val = new IPPrefix(a, width); } @@ -953,6 +953,16 @@ SubNetVal::~SubNetVal() delete val.subnet_val; } +const IPAddr& SubNetVal::Prefix() const + { + return val.subnet_val->Prefix(); + } + +int SubNetVal::Width() const + { + return val.subnet_val->Length(); + } + unsigned int SubNetVal::MemoryAllocation() const { return padded_sizeof(*this) + val.subnet_val->MemoryAllocation(); @@ -978,7 +988,7 @@ IPAddr SubNetVal::Mask() const uint32 m[4]; for ( unsigned int i = 0; i < 4; ++i ) m[i] = 0; - IPAddr rval(IPAddr::IPv6, m, IPAddr::Host); + IPAddr rval(IPv6, m, IPAddr::Host); return rval; } @@ -994,7 +1004,7 @@ IPAddr SubNetVal::Mask() const while ( ++mp < m + 4 ) *mp = 0; - IPAddr rval(IPAddr::IPv6, m, IPAddr::Host); + IPAddr rval(IPv6, m, IPAddr::Host); return rval; } diff --git a/src/Val.h b/src/Val.h index 415996d97a..e939e51cdc 100644 --- a/src/Val.h +++ b/src/Val.h @@ -513,10 +513,6 @@ protected: #define UDP_PORT_MASK 0x20000 #define ICMP_PORT_MASK 0x30000 -typedef enum { - TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP, -} TransportProto; - class PortVal : public Val { public: // Constructors - both take the port number in host order. @@ -588,8 +584,8 @@ public: Val* SizeVal() const; - const IPAddr& Prefix() const { return val.subnet_val->Prefix(); } - int Width() const { return val.subnet_val->Length(); } + const IPAddr& Prefix() const; + int Width() const; IPAddr Mask() const; bool Contains(const IPAddr& addr) const; diff --git a/src/bro.bif b/src/bro.bif index ff06288940..684b888202 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1949,7 +1949,7 @@ function is_local_interface%(ip: addr%) : bool if ( ent ) { for ( unsigned int len = 0; ent->h_addr_list[len]; ++len ) - addrs.push_back(IPAddr(IPAddr::IPv4, (uint32*)ent->h_addr_list[len], + addrs.push_back(IPAddr(IPv4, (uint32*)ent->h_addr_list[len], IPAddr::Network)); } @@ -1958,7 +1958,7 @@ function is_local_interface%(ip: addr%) : bool if ( ent ) { for ( unsigned int len = 0; ent->h_addr_list[len]; ++len ) - addrs.push_back(IPAddr(IPAddr::IPv6, (uint32*)ent->h_addr_list[len], + addrs.push_back(IPAddr(IPv6, (uint32*)ent->h_addr_list[len], IPAddr::Network)); } @@ -2024,7 +2024,7 @@ function gethostname%(%) : string ## Returns: true if *a* is an IPv4 address, else false. function is_v4_addr%(a: addr%): bool %{ - if ( a->AsAddr().GetFamily() == IPAddr::IPv4 ) + if ( a->AsAddr().GetFamily() == IPv4 ) return new Val(1, TYPE_BOOL); else return new Val(0, TYPE_BOOL); @@ -2037,7 +2037,7 @@ function is_v4_addr%(a: addr%): bool ## Returns: true if *a* is an IPv6 address, else false. function is_v6_addr%(a: addr%): bool %{ - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) return new Val(1, TYPE_BOOL); else return new Val(0, TYPE_BOOL); @@ -3522,7 +3522,7 @@ function lookup_location%(a: addr%) : geo_location } #ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( geoip_v6 && a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( geoip_v6 && a->AsAddr().GetFamily() == IPv6 ) { geoipv6_t ga; a->AsAddr().CopyIPv6(&ga); @@ -3534,7 +3534,7 @@ function lookup_location%(a: addr%) : geo_location else #endif - if ( geoip && a->AsAddr().GetFamily() == IPAddr::IPv4 ) + if ( geoip && a->AsAddr().GetFamily() == IPv4 ) { const uint32* bytes; a->AsAddr().GetBytes(&bytes); @@ -3617,7 +3617,7 @@ function lookup_asn%(a: addr%) : count { // IPv6 support showed up in 1.4.5. #ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) { geoipv6_t ga; a->AsAddr().CopyIPv6(&ga); @@ -3626,7 +3626,7 @@ function lookup_asn%(a: addr%) : count else #endif - if ( a->AsAddr().GetFamily() == IPAddr::IPv4 ) + if ( a->AsAddr().GetFamily() == IPv4 ) { const uint32* bytes; a->AsAddr().GetBytes(&bytes); @@ -5353,7 +5353,7 @@ function preserve_prefix%(a: addr, width: count%): any AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; if ( ip_anon ) { - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) builtin_error("preserve_prefix() not supported for IPv6 addresses"); else { @@ -5382,7 +5382,7 @@ function preserve_subnet%(a: subnet%): any AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; if ( ip_anon ) { - if ( a->AsSubNet().Prefix().GetFamily() == IPAddr::IPv6 ) + if ( a->AsSubNet().Prefix().GetFamily() == IPv6 ) builtin_error("preserve_subnet() not supported for IPv6 addresses"); else { @@ -5418,7 +5418,7 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr if ( anon_class < 0 || anon_class >= NUM_ADDR_ANONYMIZATION_CLASSES ) builtin_error("anonymize_addr(): invalid ip addr anonymization class"); - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) { builtin_error("anonymize_addr() not supported for IPv6 addresses"); return 0; diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index ca9ec1c3c4..0753296cb4 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -862,11 +862,11 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty) break; case TYPE_SUBNET: - lval->val.subnet_val = new IPPrefix(val->AsSubNet()); + val->AsSubNet().ConvertToThreadingValue(&lval->val.subnet_val); break; case TYPE_ADDR: - lval->val.addr_val = new IPAddr(val->AsAddr()); + val->AsAddr().ConvertToThreadingValue(&lval->val.addr_val); break; case TYPE_DOUBLE: diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index f4e48ebaef..7c71c09604 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -242,4 +242,40 @@ bool WriterBackend::DoHeartbeat(double network_time, double current_time) return true; } +string WriterBackend::Render(const threading::Value::addr_t& addr) const + { + if ( addr.family == IPv4 ) + { + char s[INET_ADDRSTRLEN]; + + if ( inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) == NULL ) + return ""; + else + return s; + } + else + { + char s[INET6_ADDRSTRLEN]; + + if ( inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) == NULL ) + return ""; + else + return s; + } + } + +string WriterBackend::Render(const threading::Value::subnet_t& subnet) const + { + char l[16]; + + if ( subnet.prefix.family == IPv4 ) + modp_uitoa10(subnet.length - 96, l); + else + modp_uitoa10(subnet.length, l); + + string s = Render(subnet.prefix) + "/" + l; + + return s; + } + diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index f9653d6b69..efb3b5d95e 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -158,6 +158,21 @@ public: bool FinishedRotation(string new_name, string old_name, double open, double close, bool terminating); + /** Helper method to render an IP address as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + string Render(const threading::Value::addr_t& addr) const; + + /** Helper method to render an subnet value as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + string Render(const threading::Value::subnet_t& subnet) const; protected: /** diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index e5bfc205be..0759e60a82 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -177,11 +177,11 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) break; case TYPE_SUBNET: - desc->Add(*val->val.subnet_val); + desc->Add(Render(val->val.subnet_val)); break; case TYPE_ADDR: - desc->Add(*val->val.addr_val); + desc->Add(Render(val->val.addr_val)); break; case TYPE_TIME: diff --git a/src/net_util.cc b/src/net_util.cc index f78246c634..5e403a349f 100644 --- a/src/net_util.cc +++ b/src/net_util.cc @@ -162,8 +162,8 @@ const char* fmt_conn_id(const IPAddr& src_addr, uint32 src_port, const char* fmt_conn_id(const uint32* src_addr, uint32 src_port, const uint32* dst_addr, uint32 dst_port) { - IPAddr src(IPAddr::IPv6, src_addr, IPAddr::Network); - IPAddr dst(IPAddr::IPv6, dst_addr, IPAddr::Network); + IPAddr src(IPv6, src_addr, IPAddr::Network); + IPAddr dst(IPv6, dst_addr, IPAddr::Network); return fmt_conn_id(src, src_port, dst, dst_port); } diff --git a/src/net_util.h b/src/net_util.h index 8787340328..f61340869a 100644 --- a/src/net_util.h +++ b/src/net_util.h @@ -5,6 +5,13 @@ #include "config.h" +// Define first. +typedef enum { + TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP, +} TransportProto; + +typedef enum { IPv4, IPv6 } IPFamily; + #include #include @@ -21,7 +28,6 @@ #include #include "util.h" -#include "IPAddr.h" #ifdef HAVE_NETINET_IP6_H #include @@ -58,6 +64,8 @@ inline int seq_delta(uint32 a, uint32 b) return int(a-b); } +class IPAddr; + // Returns the ones-complement checksum of a chunk of b short-aligned bytes. extern int ones_complement_checksum(const void* p, int b, uint32 sum); extern int ones_complement_checksum(const IPAddr& a, uint32 sum); diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index d3735e34f3..32569a5442 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -30,12 +30,6 @@ Value::~Value() && present ) delete val.string_val; - if ( type == TYPE_ADDR && present ) - delete val.addr_val; - - if ( type == TYPE_SUBNET && present ) - delete val.subnet_val; - if ( type == TYPE_TABLE && present ) { for ( int i = 0; i < val.set_val.size; i++ ) @@ -130,8 +124,8 @@ bool Value::Read(SerializationFormat* fmt) if ( ! (fmt->Read(&val.port_val.port, "port") && fmt->Read(&proto, "proto") ) ) { return false; } - - switch (proto) { + + switch ( proto ) { case 0: val.port_val.proto = TRANSPORT_UNKNOWN; break; @@ -147,20 +141,35 @@ bool Value::Read(SerializationFormat* fmt) default: return false; } + return true; } - - case TYPE_SUBNET: - { - val.subnet_val = new IPPrefix; - return fmt->Read(val.subnet_val, "subnet"); - } - case TYPE_ADDR: { - val.addr_val = new IPAddr; - return fmt->Read(val.addr_val, "addr"); + int family; + + if ( ! fmt->Read(&family, "addr-family") ) + return false; + + switch ( family ) { + case 4: + val.addr_val.family = IPv4; + return fmt->Read(&val.addr_val.in.in4, "addr-in4"); + + case 6: + val.addr_val.family = IPv6; + return fmt->Read(&val.addr_val.in.in6, "addr-in6"); + + } + + // Can't be reached. + abort(); + } + + case TYPE_SUBNET: + { + // FIXME. } case TYPE_DOUBLE: @@ -239,13 +248,27 @@ bool Value::Write(SerializationFormat* fmt) const return fmt->Write(val.uint_val, "uint"); case TYPE_PORT: - return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); + return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); case TYPE_SUBNET: - return fmt->Write(*val.subnet_val, "subnet"); + return false; // FIXME. case TYPE_ADDR: - return fmt->Write(*val.addr_val, "addr"); + { + switch ( val.addr_val.family ) { + case IPv4: + return fmt->Write((int)4, "addr-family") + && fmt->Write(val.addr_val.in.in4, "addr-in4"); + + case IPv6: + return fmt->Write((int)6, "addr-family") + && fmt->Write(val.addr_val.in.in6, "addr-in6"); + break; + } + + // Can't be reached. + abort(); + } case TYPE_DOUBLE: case TYPE_TIME: diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index c53ca37dc0..adff2035d7 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -2,10 +2,13 @@ #ifndef THREADING_SERIALIZATIONTYPES_H #define THREADING_SERIALIZATIONTYPES_H -#include "../RemoteSerializer.h" - using namespace std; +#include "Type.h" +#include "net_util.h" + +class SerializationFormat; + namespace threading { /** @@ -62,6 +65,16 @@ struct Value { typedef set_t vec_t; struct port_t { bro_uint_t port; TransportProto proto; }; + struct addr_t { + IPFamily family; + union { + struct in_addr in4; + struct in6_addr in6; + } in; + }; + + struct subnet_t { addr_t prefix; uint8_t length; }; + /** * This union is a subset of BroValUnion, including only the types we * can log directly. See IsCompatibleType(). @@ -73,8 +86,8 @@ struct Value { double double_val; set_t set_val; vec_t vector_val; - IPAddr* addr_val; - IPPrefix* subnet_val; + addr_t addr_val; + subnet_t subnet_val; string* string_val; } val; @@ -120,6 +133,7 @@ struct Value { static bool IsCompatibleType(BroType* t, bool atomic_only=false); private: +friend class ::IPAddr; Value(const Value& other) { } // Disabled. }; From df874f0f6284ea8c916633754430a0e37f45c69b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 29 Feb 2012 14:32:29 -0800 Subject: [PATCH 24/42] A number of bugfixes for the recent threading updates. All tests pass now except one: scripts.base.frameworks.metrics.cluster-intermediate-update Couldn't figure out yet why that still fails. --- src/IPAddr.h | 6 ++--- src/SerializationFormat.cc | 15 +++++------ src/SerializationFormat.h | 12 ++++----- src/threading/Manager.cc | 7 +++-- src/threading/SerialTypes.cc | 51 +++++++++++++++++++++++++++++++----- 5 files changed, 63 insertions(+), 28 deletions(-) diff --git a/src/IPAddr.h b/src/IPAddr.h index 67d6f2112e..8e1921e07b 100644 --- a/src/IPAddr.h +++ b/src/IPAddr.h @@ -47,7 +47,7 @@ public: * * @param in6 The IPv6 address. */ - IPAddr(const in4_addr& in4) + explicit IPAddr(const in4_addr& in4) { memcpy(in6.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix)); memcpy(&in6.s6_addr[12], &in4.s_addr, sizeof(in4.s_addr)); @@ -58,7 +58,7 @@ public: * * @param in6 The IPv6 address. */ - IPAddr(const in6_addr& arg_in6) : in6(arg_in6) { } + explicit IPAddr(const in6_addr& arg_in6) : in6(arg_in6) { } /** * Constructs an address instance from a string representation. @@ -523,8 +523,6 @@ public: */ string AsString() const; - /** Converts the address into the type used internally by the inter-thread communicastion. - */ operator std::string() const { return AsString(); } /** diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index ef2dc80cd7..10dd4f29ea 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -295,7 +295,6 @@ bool BinarySerializationFormat::Read(struct in6_addr* addr, const char* tag) return true; } - bool BinarySerializationFormat::Write(char v, const char* tag) { DBG_LOG(DBG_SERIAL, "Write char %s [%s]", fmt_bytes(&v, 1), tag); @@ -389,10 +388,9 @@ bool BinarySerializationFormat::Write(const IPPrefix& prefix, const char* tag) return Write(prefix.Prefix(), "prefix") && Write(prefix.Length(), "width"); } -bool BinarySerializationFormat::Write(struct in_addr& addr, const char* tag) +bool BinarySerializationFormat::Write(const struct in_addr& addr, const char* tag) { - const uint32_t* bytes; - bytes = (uint32_t*) &addr.s_addr; + const uint32_t* bytes = (uint32_t*) &addr.s_addr; if ( ! Write(ntohl(bytes[0]), "addr4") ) return false; @@ -400,10 +398,9 @@ bool BinarySerializationFormat::Write(struct in_addr& addr, const char* tag) return true; } -bool BinarySerializationFormat::Write(struct in6_addr& addr, const char* tag) +bool BinarySerializationFormat::Write(const struct in6_addr& addr, const char* tag) { - const uint32_t* bytes; - bytes = (uint32_t*) &addr.s6_addr; + const uint32_t* bytes = (uint32_t*) &addr.s6_addr; for ( int i = 0; i < 4; ++i ) { @@ -620,13 +617,13 @@ bool XMLSerializationFormat::Write(const IPPrefix& prefix, const char* tag) return false; } -bool XMLSerializationFormat::Write(struct in_addr& addr, const char* tag) +bool XMLSerializationFormat::Write(const struct in_addr& addr, const char* tag) { reporter->InternalError("XML output of in_addr not implemented"); return false; } -bool XMLSerializationFormat::Write(struct in6_addr& addr, const char* tag) +bool XMLSerializationFormat::Write(const struct in6_addr& addr, const char* tag) { reporter->InternalError("XML output of in6_addr not implemented"); return false; diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index ba5ad195a2..f270b61bae 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -59,8 +59,8 @@ public: virtual bool Write(const string& s, const char* tag) = 0; virtual bool Write(const IPAddr& addr, const char* tag) = 0; virtual bool Write(const IPPrefix& prefix, const char* tag) = 0; - virtual bool Write(struct in_addr& addr, const char* tag) = 0; - virtual bool Write(struct in6_addr& addr, const char* tag) = 0; + virtual bool Write(const struct in_addr& addr, const char* tag) = 0; + virtual bool Write(const struct in6_addr& addr, const char* tag) = 0; virtual bool WriteOpenTag(const char* tag) = 0; virtual bool WriteCloseTag(const char* tag) = 0; @@ -118,8 +118,8 @@ public: virtual bool Write(const string& s, const char* tag); virtual bool Write(const IPAddr& addr, const char* tag); virtual bool Write(const IPPrefix& prefix, const char* tag); - virtual bool Write(struct in_addr& addr, const char* tag); - virtual bool Write(struct in6_addr& addr, const char* tag); + virtual bool Write(const struct in_addr& addr, const char* tag); + virtual bool Write(const struct in6_addr& addr, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -144,8 +144,8 @@ public: virtual bool Write(const string& s, const char* tag); virtual bool Write(const IPAddr& addr, const char* tag); virtual bool Write(const IPPrefix& prefix, const char* tag); - virtual bool Write(struct in_addr& addr, const char* tag); - virtual bool Write(struct in6_addr& addr, const char* tag); + virtual bool Write(const struct in_addr& addr, const char* tag); + virtual bool Write(const struct in6_addr& addr, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d008d2e5e8..24e100fe37 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -106,8 +106,11 @@ void Manager::Process() Message* msg = t->RetrieveOut(); - if ( msg->Process() && network_time ) - did_process = true; + if ( msg->Process() ) + { + if ( network_time ) + did_process = true; + } else { diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index 32569a5442..a5692b2ffd 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -147,7 +147,7 @@ bool Value::Read(SerializationFormat* fmt) case TYPE_ADDR: { - int family; + char family; if ( ! fmt->Read(&family, "addr-family") ) return false; @@ -169,7 +169,27 @@ bool Value::Read(SerializationFormat* fmt) case TYPE_SUBNET: { - // FIXME. + char length; + char family; + + if ( ! (fmt->Read(&length, "subnet-len") && fmt->Read(&family, "subnet-family")) ) + return false; + + switch ( family ) { + case 4: + val.subnet_val.length = (uint8_t)length; + val.subnet_val.prefix.family = IPv4; + return fmt->Read(&val.subnet_val.prefix.in.in4, "subnet-in4"); + + case 6: + val.subnet_val.length = (uint8_t)length; + val.subnet_val.prefix.family = IPv6; + return fmt->Read(&val.subnet_val.prefix.in.in6, "subnet-in6"); + + } + + // Can't be reached. + abort(); } case TYPE_DOUBLE: @@ -250,18 +270,15 @@ bool Value::Write(SerializationFormat* fmt) const case TYPE_PORT: return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); - case TYPE_SUBNET: - return false; // FIXME. - case TYPE_ADDR: { switch ( val.addr_val.family ) { case IPv4: - return fmt->Write((int)4, "addr-family") + return fmt->Write((char)4, "addr-family") && fmt->Write(val.addr_val.in.in4, "addr-in4"); case IPv6: - return fmt->Write((int)6, "addr-family") + return fmt->Write((char)6, "addr-family") && fmt->Write(val.addr_val.in.in6, "addr-in6"); break; } @@ -270,6 +287,26 @@ bool Value::Write(SerializationFormat* fmt) const abort(); } + case TYPE_SUBNET: + { + if ( ! fmt->Write((char)val.subnet_val.length, "subnet-length") ) + return false; + + switch ( val.subnet_val.prefix.family ) { + case IPv4: + return fmt->Write((char)4, "subnet-family") + && fmt->Write(val.subnet_val.prefix.in.in4, "subnet-in4"); + + case IPv6: + return fmt->Write((char)6, "subnet-family") + && fmt->Write(val.subnet_val.prefix.in.in6, "subnet-in6"); + break; + } + + // Can't be reached. + abort(); + } + case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: From 6df9004423dc0ea06aebe8201f4ca4bb937eda09 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 29 Feb 2012 16:06:54 -0800 Subject: [PATCH 25/42] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index 3b63c3f1e7..4d2dde5573 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 3b63c3f1e7d915b1bda16862bfa4a8593ffc38f6 +Subproject commit 4d2dde55733ed86ea3f2db8df5b78b0bcfbb54c4 From 355c447698dd86fc6a7e78694318a911bb7ee3e4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 29 Feb 2012 16:08:37 -0800 Subject: [PATCH 26/42] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index 4d2dde5573..d6e36c95e0 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 4d2dde55733ed86ea3f2db8df5b78b0bcfbb54c4 +Subproject commit d6e36c95e0335f7cc081191c8612085bd12706f9 From 56dd7918d0475df364b47b6133745bbb598536fc Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 29 Feb 2012 16:46:21 -0800 Subject: [PATCH 27/42] Adding missing includes needed on FreeBSD. --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- src/threading/SerialTypes.h | 4 ++++ 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/aux/binpac b/aux/binpac index 43308aab47..3034da8f08 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 43308aab47a3357ca1885e1b6954154a2744d821 +Subproject commit 3034da8f082b61157e234237993ffd7a95be6e62 diff --git a/aux/bro-aux b/aux/bro-aux index 139cc2e1e0..f53bcb2b49 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 139cc2e1e049c4e1cc7e95f20866102be1d3d599 +Subproject commit f53bcb2b492cb0db3dd288384040abc2ab711767 diff --git a/aux/broccoli b/aux/broccoli index d6e36c95e0..2602eb53e7 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit d6e36c95e0335f7cc081191c8612085bd12706f9 +Subproject commit 2602eb53e70d7f0afae8fac58d7636b9291974a4 diff --git a/aux/broctl b/aux/broctl index e908ba686d..954538514d 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit e908ba686dceb56065bdf569c18dd0f67f662f6b +Subproject commit 954538514d71983e7ef3f0e109960466096e1c1d diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index adff2035d7..db7dc837bd 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -4,6 +4,10 @@ using namespace std; +#include +#include +#include + #include "Type.h" #include "net_util.h" From 6429d1248adf0caa275187d8e204b52638ab3332 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 1 Mar 2012 16:00:30 -0800 Subject: [PATCH 28/42] Prevent manager from busy looping. I saw this with the new threading code but I'm wondering if it also helps with the "high CPU usage with low traffiv volume" problem. --- src/Net.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Net.cc b/src/Net.cc index d93f1e1a85..2ebbfb20ed 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -486,6 +486,8 @@ void net_run() // since Bro timers are not high-precision anyway.) if ( ! using_communication ) usleep(100000); + else + usleep(1000); // Flawfinder says about usleep: // From 554a29b3edd5e1504c5e611c7b7ff9287abae330 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 1 Mar 2012 16:04:34 -0800 Subject: [PATCH 29/42] Preventing busy looping when no threads have been spawned. --- src/threading/Manager.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 24e100fe37..f5770e2dd8 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -10,7 +10,7 @@ Manager::Manager() did_process = true; next_beat = 0; terminating = false; - idle = false; + idle = true; } Manager::~Manager() @@ -58,6 +58,7 @@ void Manager::AddThread(BasicThread* thread) { DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str()); all_threads.push_back(thread); + idle = false; } void Manager::AddMsgThread(MsgThread* thread) From c0678e7e1f649d4c69701eb7d03fd28893e12dd4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 17:14:58 -0800 Subject: [PATCH 30/42] Fixing problem logging remotely when local logging was turned off. For that, moved the remote logging from the Manager to the WriterFrontend. That also simplifies the Manager a bit. --- src/RemoteSerializer.cc | 13 +++++- src/logging/Manager.cc | 63 ++++++---------------------- src/logging/Manager.h | 3 +- src/logging/WriterFrontend.cc | 79 ++++++++++++++++++++++++++++++----- src/logging/WriterFrontend.h | 18 +++++++- src/threading/BasicThread.cc | 15 ++++--- 6 files changed, 116 insertions(+), 75 deletions(-) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 4b8f527f2b..c6b9623096 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -234,7 +234,7 @@ static const int PRINT_BUFFER_SIZE = 10 * 1024; static const int SOCKBUF_SIZE = 1024 * 1024; // Buffer size for remote-log data. -static const int LOG_BUFFER_SIZE = 50 * 1024; +static const int LOG_BUFFER_SIZE = 512; struct ping_args { uint32 seq; @@ -2587,7 +2587,10 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) || (network_time - last_flush > 1.0) ) { if ( ! FlushLogBuffer(peer) ) + { + delete [] data; return false; + } } // If the data is actually larger than our complete buffer, just send it out. @@ -2631,6 +2634,12 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( current_peer->state == Peer::CLOSING ) return false; +#ifdef USE_PERFTOOLS + // Don't track allocations here, they'll be released only after the + // main loop exists. And it's just a tiny amount anyway. + HeapLeakChecker::Disabler disabler; +#endif + assert(current_args); EnumVal* id_val = 0; @@ -2666,7 +2675,7 @@ bool RemoteSerializer::ProcessLogCreateWriter() id_val = new EnumVal(id, BifType::Enum::Log::ID); writer_val = new EnumVal(writer, BifType::Enum::Log::Writer); - if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields) ) + if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields, true, false) ) goto error; Unref(id_val); diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 0753296cb4..14fb3428fe 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -753,64 +753,25 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) for ( int j = 0; j < filter->num_fields; ++j ) arg_fields[j] = new Field(*filter->fields[j]); - if ( filter->remote ) - remote_serializer->SendLogCreateWriter(stream->id, - filter->writer, - path, - filter->num_fields, - arg_fields); + writer = CreateWriter(stream->id, filter->writer, + path, filter->num_fields, + arg_fields, filter->local, filter->remote); - if ( filter->local ) + if ( ! writer ) { - writer = CreateWriter(stream->id, filter->writer, - path, filter->num_fields, - arg_fields); - - if ( ! writer ) - { - Unref(columns); - return false; - } + Unref(columns); + return false; } - else - { - // Insert a null pointer into the map to make - // sure we don't try creating it again. - stream->writers.insert(Stream::WriterMap::value_type( - Stream::WriterPathPair(filter->writer->AsEnum(), path), 0)); - for( int i = 0; i < filter->num_fields; ++i) - delete arg_fields[i]; - - delete [] arg_fields; - } } // Alright, can do the write now. - if ( filter->local || filter->remote ) - { - threading::Value** vals = RecordToFilterVals(stream, filter, columns); - - if ( filter->remote ) - remote_serializer->SendLogWrite(stream->id, - filter->writer, - path, - filter->num_fields, - vals); - - if ( filter->local ) - { - // Write takes ownership of vals. - assert(writer); - writer->Write(filter->num_fields, vals); - } - - else - DeleteVals(filter->num_fields, vals); - - } + threading::Value** vals = RecordToFilterVals(stream, filter, columns); + // Write takes ownership of vals. + assert(writer); + writer->Write(filter->num_fields, vals); #ifdef DEBUG DBG_LOG(DBG_LOGGING, "Wrote record to filter '%s' on stream '%s'", @@ -976,7 +937,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, } WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, const Field* const* fields) + int num_fields, const Field* const* fields, bool local, bool remote) { Stream* stream = FindStream(id); @@ -992,7 +953,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, // return it. return w->second->writer; - WriterFrontend* writer_obj = new WriterFrontend(writer->AsEnum()); + WriterFrontend* writer_obj = new WriterFrontend(id, writer, local, remote); assert(writer_obj); writer_obj->Init(path, num_fields, fields); diff --git a/src/logging/Manager.h b/src/logging/Manager.h index d931bfaef8..bf097c5e1a 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -159,7 +159,8 @@ protected: // Takes ownership of fields. WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, const threading::Field* const* fields); + int num_fields, const threading::Field* const* fields, + bool local, bool remote); // Takes ownership of values.. bool Write(EnumVal* id, EnumVal* writer, string path, diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 02f1a188d8..26e8eaf22e 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -99,21 +99,36 @@ public: using namespace logging; -WriterFrontend::WriterFrontend(bro_int_t type) +WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote) { + stream = arg_stream; + writer = arg_writer; + Ref(stream); + Ref(writer); + disabled = initialized = false; buf = true; + local = arg_local; + remote = arg_remote; write_buffer = 0; write_buffer_pos = 0; ty_name = ""; - backend = log_mgr->CreateBackend(this, type); - assert(backend); - backend->Start(); + if ( local ) + { + backend = log_mgr->CreateBackend(this, writer->AsEnum()); + assert(backend); + backend->Start(); + } + + else + backend = 0; } WriterFrontend::~WriterFrontend() { + Unref(stream); + Unref(writer); } string WriterFrontend::Name() const @@ -128,7 +143,9 @@ void WriterFrontend::Stop() { FlushWriteBuffer(); SetDisable(); - backend->Stop(); + + if ( backend ) + backend->Stop(); } void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* const * arg_fields) @@ -144,7 +161,17 @@ void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* cons fields = arg_fields; initialized = true; - backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + + if ( backend ) + backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + + if ( remote ) + remote_serializer->SendLogCreateWriter(stream, + writer, + arg_path, + arg_num_fields, + arg_fields); + } void WriterFrontend::Write(int num_fields, Value** vals) @@ -152,6 +179,19 @@ void WriterFrontend::Write(int num_fields, Value** vals) if ( disabled ) return; + if ( remote ) + remote_serializer->SendLogWrite(stream, + writer, + path, + num_fields, + vals); + + if ( ! backend ) + { + DeleteVals(vals); + return; + } + if ( ! write_buffer ) { // Need new buffer. @@ -173,7 +213,8 @@ void WriterFrontend::FlushWriteBuffer() // Nothing to do. return; - backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer)); + if ( backend ) + backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer)); // Clear buffer (no delete, we pass ownership to child thread.) write_buffer = 0; @@ -187,7 +228,8 @@ void WriterFrontend::SetBuf(bool enabled) buf = enabled; - backend->SendIn(new SetBufMessage(backend, enabled)); + if ( backend ) + backend->SendIn(new SetBufMessage(backend, enabled)); if ( ! buf ) // Make sure no longer buffer any still queued data. @@ -200,7 +242,9 @@ void WriterFrontend::Flush() return; FlushWriteBuffer(); - backend->SendIn(new FlushMessage(backend)); + + if ( backend ) + backend->SendIn(new FlushMessage(backend)); } void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating) @@ -209,7 +253,9 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool return; FlushWriteBuffer(); - backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); + + if ( backend ) + backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); } void WriterFrontend::Finish() @@ -218,7 +264,18 @@ void WriterFrontend::Finish() return; FlushWriteBuffer(); - backend->SendIn(new FinishMessage(backend)); + + if ( backend ) + backend->SendIn(new FinishMessage(backend)); + } + +void WriterFrontend::DeleteVals(Value** vals) + { + // Note this code is duplicated in Manager::DeleteVals(). + for ( int i = 0; i < num_fields; i++ ) + delete vals[i]; + + delete [] vals; } diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index 4386a15f64..3e05d17c9e 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -25,14 +25,21 @@ public: /** * Constructor. * - * type: The backend writer type, with the value corresponding to the + * stream: The logging stream. + * + * writer: The backend writer type, with the value corresponding to the * script-level \c Log::Writer enum (e.g., \a WRITER_ASCII). The * frontend will internally instantiate a WriterBackend of the * corresponding type. + * + * local: If true, the writer will instantiate a local backend. + * + * remote: If true, the writer will forward all data to remote + * clients. * * Frontends must only be instantiated by the main thread. */ - WriterFrontend(bro_int_t type); + WriterFrontend(EnumVal* stream, EnumVal* writer, bool local, bool remote); /** * Destructor. @@ -187,10 +194,17 @@ public: protected: friend class Manager; + void DeleteVals(threading::Value** vals); + + EnumVal* stream; + EnumVal* writer; + WriterBackend* backend; // The backend we have instanatiated. bool disabled; // True if disabled. bool initialized; // True if initialized. bool buf; // True if buffering is enabled (default). + bool local; // True if logging locally. + bool remote; // True if loggin remotely. string ty_name; // Name of the backend type. Set by the manager. string path; // The log path. diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 51c4f7a3bc..e590b13434 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -20,8 +20,8 @@ BasicThread::BasicThread() terminating = false; pthread = 0; - buf = 0; - buf_len = 1024; + buf_len = 2048; + buf = (char*) malloc(buf_len); name = Fmt("thread-%d", ++thread_counter); @@ -57,9 +57,6 @@ void BasicThread::SetOSName(const string& name) const char* BasicThread::Fmt(const char* format, ...) { - if ( ! buf ) - buf = (char*) malloc(buf_len); - va_list al; va_start(al, format); int n = safe_vsnprintf(buf, buf_len, format, al); @@ -67,13 +64,15 @@ const char* BasicThread::Fmt(const char* format, ...) if ( (unsigned int) n >= buf_len ) { // Not enough room, grow the buffer. - buf_len = n + 32; - buf = (char*) realloc(buf, buf_len); + int tmp_len = n + 32; + char* tmp = (char*) malloc(tmp_len); // Is it portable to restart? va_start(al, format); - n = safe_vsnprintf(buf, buf_len, format, al); + n = safe_vsnprintf(tmp, tmp_len, format, al); va_end(al); + + free(tmp); } return buf; From bf14bd91d7332384931ebd1b42401b8dd0c14754 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 17:16:51 -0800 Subject: [PATCH 31/42] Removing some no longer needed checks. --- src/logging/Manager.cc | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 14fb3428fe..4e97351e57 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -105,9 +105,6 @@ Manager::Stream::~Stream() { WriterInfo* winfo = i->second; - if ( ! winfo ) - continue; - if ( winfo->rotation_timer ) timer_mgr->Cancel(winfo->rotation_timer); @@ -207,7 +204,7 @@ Manager::WriterInfo* Manager::FindWriter(WriterFrontend* writer) { WriterInfo* winfo = i->second; - if ( winfo && winfo->writer == writer ) + if ( winfo->writer == writer ) return winfo; } } @@ -221,7 +218,7 @@ void Manager::RemoveDisabledWriters(Stream* stream) for ( Stream::WriterMap::iterator j = stream->writers.begin(); j != stream->writers.end(); j++ ) { - if ( j->second && j->second->writer->Disabled() ) + if ( j->second->writer->Disabled() ) { j->second->writer->Stop(); delete j->second; @@ -740,7 +737,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) if ( w != stream->writers.end() ) // We know this writer already. - writer = w->second ? w->second->writer : 0; + writer = w->second->writer; else { @@ -948,7 +945,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path)); - if ( w != stream->writers.end() && w->second ) + if ( w != stream->writers.end() ) // If we already have a writer for this. That's fine, we just // return it. return w->second->writer; @@ -1050,8 +1047,7 @@ bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, return false; } - if ( w->second ) - w->second->writer->Write(num_fields, vals); + w->second->writer->Write(num_fields, vals); DBG_LOG(DBG_LOGGING, "Wrote pre-filtered record to path '%s' on stream '%s'", @@ -1072,9 +1068,6 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) { - if ( ! i->second ) - continue; - WriterFrontend* writer = i->second->writer; EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer); @@ -1095,10 +1088,7 @@ bool Manager::SetBuf(EnumVal* id, bool enabled) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - { - if ( i->second ) - i->second->writer->SetBuf(enabled); - } + i->second->writer->SetBuf(enabled); RemoveDisabledWriters(stream); @@ -1116,10 +1106,7 @@ bool Manager::Flush(EnumVal* id) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - { - if ( i->second ) - i->second->writer->Flush(); - } + i->second->writer->Flush(); RemoveDisabledWriters(stream); From 83038d78e0511a0fbed649e514a27904f5071c2f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 17:29:23 -0800 Subject: [PATCH 32/42] Adding new leak tests involving remote logging. --- .../manager-1.metrics.log | 10 +++ .../core.leaks.remote/sender.test.failure.log | 10 +++ .../core.leaks.remote/sender.test.log | 12 +++ .../core.leaks.remote/sender.test.success.log | 9 +++ testing/btest/core/leaks/basic-cluster.bro | 39 +++++++++ testing/btest/core/leaks/remote.bro | 79 +++++++++++++++++++ .../external/scripts/perftools-adapt-paths | 2 +- 7 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log create mode 100644 testing/btest/Baseline/core.leaks.remote/sender.test.failure.log create mode 100644 testing/btest/Baseline/core.leaks.remote/sender.test.log create mode 100644 testing/btest/Baseline/core.leaks.remote/sender.test.success.log create mode 100644 testing/btest/core/leaks/basic-cluster.bro create mode 100644 testing/btest/core/leaks/remote.bro diff --git a/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log new file mode 100644 index 0000000000..42fcd6a526 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path metrics +#fields ts metric_id filter_name index.host index.str index.network value +#types time enum string addr string subnet count +1331256494.591966 TEST_METRIC foo-bar 6.5.4.3 - - 4 +1331256494.591966 TEST_METRIC foo-bar 7.2.1.5 - - 2 +1331256494.591966 TEST_METRIC foo-bar 1.2.3.4 - - 6 diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log new file mode 100644 index 0000000000..5a26f322f4 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test.failure +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.log b/testing/btest/Baseline/core.leaks.remote/sender.test.log new file mode 100644 index 0000000000..9d2ba26f48 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.success.log b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log new file mode 100644 index 0000000000..1b2ed452a0 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log @@ -0,0 +1,9 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test.success +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro new file mode 100644 index 0000000000..a82f52c8b2 --- /dev/null +++ b/testing/btest/core/leaks/basic-cluster.bro @@ -0,0 +1,39 @@ +# Needs perftools support. +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT +# @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run worker-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT +# @TEST-EXEC: btest-bg-run worker-2 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT +# @TEST-EXEC: btest-bg-wait -k 30 +# @TEST-EXEC: btest-diff manager-1/metrics.log + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")], + ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"], +}; +@TEST-END-FILE + +redef Log::default_rotation_interval = 0secs; + +redef enum Metrics::ID += { + TEST_METRIC, +}; + +event bro_init() &priority=5 + { + Metrics::add_filter(TEST_METRIC, + [$name="foo-bar", + $break_interval=3secs]); + + if ( Cluster::local_node_type() == Cluster::WORKER ) + { + Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); + Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); + Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); + } + } diff --git a/testing/btest/core/leaks/remote.bro b/testing/btest/core/leaks/remote.bro new file mode 100644 index 0000000000..fa72ce6024 --- /dev/null +++ b/testing/btest/core/leaks/remote.bro @@ -0,0 +1,79 @@ +# Needs perftools support. +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-wait -k 10 +# @TEST-EXEC: btest-diff sender/test.log +# @TEST-EXEC: btest-diff sender/test.failure.log +# @TEST-EXEC: btest-diff sender/test.success.log +# @TEST-EXEC: cmp receiver/test.log sender/test.log +# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log +# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log + +# This is the common part loaded by both sender and receiver. +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { LOG }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); + Log::add_filter(Test::LOG, [$name="f1", $path="test.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); +} + +##### + +@TEST-START-FILE sender.bro + +module Test; + +@load frameworks/communication/listen + +function fail(rec: Log): bool + { + return rec$status != "success"; + } + +event remote_connection_handshake_done(p: event_peer) + { + Log::add_filter(Test::LOG, [$name="f2", $path="test.failure", $pred=fail]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + + # Log something. + Log::write(Test::LOG, r); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + disconnect(p); + } +@TEST-END-FILE + +@TEST-START-FILE receiver.bro + +##### + +redef Communication::nodes += { + ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T] +}; + +@TEST-END-FILE diff --git a/testing/external/scripts/perftools-adapt-paths b/testing/external/scripts/perftools-adapt-paths index 2eda2477c7..cfecd39993 100755 --- a/testing/external/scripts/perftools-adapt-paths +++ b/testing/external/scripts/perftools-adapt-paths @@ -7,4 +7,4 @@ cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1 -grep -q "No leaks found" $1 +grep -qv "detected leaks of" $1 From 51009b73bcd23c812f8244f55ee5c6e5172598fd Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 18:13:17 -0800 Subject: [PATCH 33/42] Finetuning communication CPU usage. --- .../base/frameworks/cluster/setup-connections.bro | 2 +- src/RemoteSerializer.cc | 14 ++++++++++++++ src/RemoteSerializer.h | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index b5a0d25e1f..20646525be 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -44,7 +44,7 @@ event bro_init() &priority=9 { if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events]; + [$host=n$ip, $connect=F, $class=i, $sync=F, $auth=T, $events=worker2proxy_events]; # accepts connections from the previous one. # (This is not ideal for setups with many proxies) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index c6b9623096..56e27c2104 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -532,6 +532,7 @@ RemoteSerializer::RemoteSerializer() terminating = false; in_sync = 0; last_flush = 0; + received_logs = 0; } RemoteSerializer::~RemoteSerializer() @@ -1353,6 +1354,14 @@ double RemoteSerializer::NextTimestamp(double* local_network_time) { Poll(false); + if ( received_logs > 0 ) + { + // If we processed logs last time, assume there's more. + idle = false; + received_logs = 0; + return timer_mgr->Time(); + } + double et = events.length() ? events[0]->time : -1; double pt = packets.length() ? packets[0]->time : -1; @@ -2744,6 +2753,8 @@ bool RemoteSerializer::ProcessLogWrite() fmt.EndRead(); + ++received_logs; + return true; error: @@ -3385,6 +3396,9 @@ void SocketComm::Run() small_timeout.tv_usec = io->CanWrite() || io->CanRead() ? 1 : 10; + if ( ! io->CanWrite() ) + usleep(10); + int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except, &small_timeout); diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index eabcb18a38..05d25ca525 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -338,6 +338,7 @@ private: int propagate_accesses; bool ignore_accesses; bool terminating; + int received_logs; Peer* source_peer; PeerID id_counter; // Keeps track of assigned IDs. uint32 current_sync_point; From 8eaf40ec18222d54ae4a76f535008951a73d7fca Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 20:24:12 -0800 Subject: [PATCH 34/42] Reverting accidental commit. Thanks, Seth! --- scripts/base/frameworks/cluster/setup-connections.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index 20646525be..b5a0d25e1f 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -44,7 +44,7 @@ event bro_init() &priority=9 { if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, $class=i, $sync=F, $auth=T, $events=worker2proxy_events]; + [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events]; # accepts connections from the previous one. # (This is not ideal for setups with many proxies) From 89a3bb33c8003bd0be1369e756eef31af23cb18e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 20:22:39 -0800 Subject: [PATCH 35/42] Don't assert during shutdown. --- src/RemoteSerializer.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 56e27c2104..f29e907790 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -2561,7 +2561,9 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st if ( ! peer->logs_requested ) return false; - assert(peer->log_buffer); + if ( ! peer->log_buffer ) + // Peer shutting down. + return false; // Serialize the log record entry. From e3f5cbb670a0a59ab80ed58224f23cb053e4cc9f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 16 Mar 2012 09:11:31 -0700 Subject: [PATCH 36/42] Small fixes and tweaks. - Fixing tiny leak. - Fixing threads stat output. --- src/logging/Manager.cc | 5 ++--- src/threading/Manager.cc | 30 +++++++++++++++--------------- src/threading/MsgThread.cc | 4 ++-- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 4e97351e57..74220ecde4 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -677,11 +677,11 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) Val* path_arg; if ( filter->path_val ) - path_arg = filter->path_val; + path_arg = filter->path_val->Ref(); else path_arg = new StringVal(""); - vl.append(path_arg->Ref()); + vl.append(path_arg); Val* rec_arg; BroType* rt = filter->path_func->FType()->Args()->FieldType("rec"); @@ -715,7 +715,6 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) if ( ! filter->path_val ) { - Unref(path_arg); filter->path = v->AsString()->CheckString(); filter->path_val = v->Ref(); } diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index f5770e2dd8..43eb0313f4 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -102,25 +102,25 @@ void Manager::Process() next_beat = 0; } - if ( ! t->HasOut() ) - continue; - - Message* msg = t->RetrieveOut(); - - if ( msg->Process() ) + while ( t->HasOut() ) { - if ( network_time ) - did_process = true; + Message* msg = t->RetrieveOut(); + + if ( msg->Process() ) + { + if ( network_time ) + did_process = true; + } + + else + { + string s = msg->Name() + " failed, terminating thread"; + reporter->Error("%s", s.c_str()); + t->Stop(); } - else - { - string s = msg->Name() + " failed, terminating thread"; - reporter->Error("%s", s.c_str()); - t->Stop(); + delete msg; } - - delete msg; } // fprintf(stderr, "P %.6f %.6f do_beat=%d did_process=%d next_next=%.6f\n", network_time, timer_mgr->Time(), do_beat, (int)did_process, next_beat); diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index b7782b9a05..145e16c57b 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -281,7 +281,7 @@ void MsgThread::GetStats(Stats* stats) { stats->sent_in = cnt_sent_in; stats->sent_out = cnt_sent_out; - stats->pending_in = cnt_sent_in - queue_in.Size(); - stats->pending_out = cnt_sent_out - queue_out.Size(); + stats->pending_in = queue_in.Size(); + stats->pending_out = queue_out.Size(); } From d7c9471818ed60453fc319388277ebaf43939b27 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 23 Mar 2012 15:57:25 -0700 Subject: [PATCH 37/42] Extending queue statistics. --- .../frameworks/cluster/setup-connections.bro | 2 +- src/Stats.cc | 9 ++++- src/logging/WriterFrontend.h | 2 +- src/threading/MsgThread.cc | 2 + src/threading/MsgThread.h | 4 ++ src/threading/Queue.h | 39 +++++++++++++++++++ 6 files changed, 54 insertions(+), 4 deletions(-) diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index b5a0d25e1f..20646525be 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -44,7 +44,7 @@ event bro_init() &priority=9 { if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events]; + [$host=n$ip, $connect=F, $class=i, $sync=F, $auth=T, $events=worker2proxy_events]; # accepts connections from the previous one. # (This is not ideal for setups with many proxies) diff --git a/src/Stats.cc b/src/Stats.cc index a2e7496c5f..c3035231e9 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -210,11 +210,16 @@ void ProfileLogger::Log() i != thread_stats.end(); ++i ) { threading::MsgThread::Stats s = i->second; - file->Write(fmt("%0.6f %-25s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", + file->Write(fmt("%0.6f %-25s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 + " (#queue r/w: in=%" PRIu64 "/%" PRIu64 " out=%" PRIu64 "/%" PRIu64 ")" + "\n", network_time, i->first.c_str(), s.sent_in, s.sent_out, - s.pending_in, s.pending_out)); + s.pending_in, s.pending_out, + s.queue_in_stats.num_reads, s.queue_in_stats.num_writes, + s.queue_out_stats.num_reads, s.queue_out_stats.num_writes + )); } // Script-level state. diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index 3e05d17c9e..4d22bd9b1f 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -212,7 +212,7 @@ protected: const threading::Field* const* fields; // The log fields. // Buffer for bulk writes. - static const int WRITER_BUFFER_SIZE = 50; + static const int WRITER_BUFFER_SIZE = 1000; int write_buffer_pos; // Position of next write in buffer. threading::Value*** write_buffer; // Buffer of size WRITER_BUFFER_SIZE. }; diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 145e16c57b..ddcd3df1dd 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -283,5 +283,7 @@ void MsgThread::GetStats(Stats* stats) stats->sent_out = cnt_sent_out; stats->pending_in = queue_in.Size(); stats->pending_out = queue_out.Size(); + queue_in.GetStats(&stats->queue_in_stats); + queue_out.GetStats(&stats->queue_out_stats); } diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 28c7690dfa..5ac1c0f780 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -154,6 +154,10 @@ public: uint64_t sent_out; //! Number of messages sent from the child thread to the main thread uint64_t pending_in; //! Number of messages sent to the child but not yet processed. uint64_t pending_out; //! Number of messages sent from the child but not yet processed by the main thread. + + /// Statistics from our queues. + Queue::Stats queue_in_stats; + Queue::Stats queue_out_stats; }; /** diff --git a/src/threading/Queue.h b/src/threading/Queue.h index a25f897d23..985ba31714 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -58,6 +58,22 @@ public: */ uint64_t Size(); + /** + * Statistics about inter-thread communication. + */ + struct Stats + { + uint64_t num_reads; //! Number of messages read from the queue. + uint64_t num_writes; //! Number of messages written to the queue. + }; + + /** + * Returns statistics about the queue's usage. + * + * @param stats A pointer to a structure that will be filled with + * current numbers. */ + void GetStats(Stats* stats); + private: static const int NUM_QUEUES = 8; @@ -67,6 +83,10 @@ private: int read_ptr; // Where the next operation will read from int write_ptr; // Where the next operation will write to + + // Statistics. + uint64_t num_reads; + uint64_t num_writes; }; inline static void safe_lock(pthread_mutex_t* mutex) @@ -86,6 +106,7 @@ inline Queue::Queue() { read_ptr = 0; write_ptr = 0; + num_reads = num_writes = 0; for( int i = 0; i < NUM_QUEUES; ++i ) { @@ -121,6 +142,7 @@ inline T Queue::Get() messages[read_ptr].pop(); read_ptr = (read_ptr + 1) % NUM_QUEUES; + ++num_reads; safe_unlock(&mutex[old_read_ptr]); @@ -142,6 +164,7 @@ inline void Queue::Put(T data) pthread_cond_signal(&has_data[write_ptr]); write_ptr = (write_ptr + 1) % NUM_QUEUES; + ++num_writes; safe_unlock(&mutex[old_write_ptr]); } @@ -177,7 +200,23 @@ inline uint64_t Queue::Size() return size; } +template +inline void Queue::GetStats(Stats* stats) + { + // To be safe, we look all queues. That's probably unneccessary, but + // doesn't really hurt. + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_lock(&mutex[i]); + + stats->num_reads = num_reads; + stats->num_writes = num_writes; + + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_unlock(&mutex[i]); + } + } + #endif From 42066cc1fd35ca7b63daeaf588271ec3c040385e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 28 Mar 2012 14:53:59 -0700 Subject: [PATCH 38/42] Teaching cmake to always link in tcmalloc if it finds it. Also renaming --enable-perftools to --enable-perftool-debug to indicate that the switch is only relevant for debugging the heap. It's not needed to pick up tcmalloc for better performance. --with-perftools can still (and always) be used to give a hint where to find the libraries. With the threading, using tcmalloc improves memory usage on FreeBSD significantly when running on a trace. If it fixes the live problems, remains to be seen ... --- CMakeLists.txt | 20 +++++++++++++++----- cmake | 2 +- configure | 9 ++++----- src/DPM.cc | 2 +- src/File.cc | 4 ++-- src/ID.cc | 2 +- src/Login.cc | 2 +- src/PersistenceSerializer.cc | 2 +- src/RemoteSerializer.cc | 4 ++-- src/RuleMatcher.cc | 2 +- src/StateAccess.cc | 2 +- src/main.cc | 18 +++++++++--------- src/util.h | 2 +- 13 files changed, 40 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de3138c20c..febc2d6ec1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,11 +89,20 @@ if (LIBGEOIP_FOUND) endif () set(USE_PERFTOOLS false) -if (ENABLE_PERFTOOLS) - find_package(GooglePerftools) - if (GOOGLEPERFTOOLS_FOUND) - set(USE_PERFTOOLS true) - include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR}) +set(USE_PERFTOOLS_DEBUG false) + +find_package(GooglePerftools) + +if (GOOGLEPERFTOOLS_FOUND) + include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR}) + set(USE_PERFTOOLS true) + + if (ENABLE_PERFTOOLS_DEBUG) + # Enable heap debugging with perftools. + set(USE_PERFTOOLS_DEBUG true) + list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG}) + else () + # Link in tcmalloc for better performance. list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES}) endif () endif () @@ -183,6 +192,7 @@ message( "\n" "\nGeoIP: ${USE_GEOIP}" "\nGoogle perftools: ${USE_PERFTOOLS}" + "\n debugging: ${USE_PERFTOOLS_DEBUG}" "\n" "\n================================================================\n" ) diff --git a/cmake b/cmake index 2cc1055770..4b573ed849 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 2cc105577044a2d214124568f3f2496ed2ccbb34 +Subproject commit 4b573ed849f131ebb8e34fa24786d56f9805e444 diff --git a/configure b/configure index 43afb4ae99..05aa12815b 100755 --- a/configure +++ b/configure @@ -27,7 +27,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... Optional Features: --enable-debug compile in debugging mode - --enable-perftools use Google's perftools + --enable-perftools-debug use Google's perftools for debugging --disable-broccoli don't build or install the Broccoli library --disable-broctl don't install Broctl --disable-auxtools don't build or install auxilliary tools @@ -91,7 +91,7 @@ append_cache_entry BRO_ROOT_DIR PATH /usr/local/bro append_cache_entry PY_MOD_INSTALL_DIR PATH /usr/local/bro/lib/broctl append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING /usr/local/bro/share/bro append_cache_entry ENABLE_DEBUG BOOL false -append_cache_entry ENABLE_PERFTOOLS BOOL false +append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false append_cache_entry BinPAC_SKIP_INSTALL BOOL true append_cache_entry BUILD_SHARED_LIBS BOOL true append_cache_entry INSTALL_AUX_TOOLS BOOL true @@ -132,8 +132,8 @@ while [ $# -ne 0 ]; do --enable-debug) append_cache_entry ENABLE_DEBUG BOOL true ;; - --enable-perftools) - append_cache_entry ENABLE_PERFTOOLS BOOL true + --enable-perftools-debug) + append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL true ;; --disable-broccoli) append_cache_entry INSTALL_BROCCOLI BOOL false @@ -178,7 +178,6 @@ while [ $# -ne 0 ]; do append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg ;; --with-perftools=*) - append_cache_entry ENABLE_PERFTOOLS BOOL true append_cache_entry GooglePerftools_ROOT_DIR PATH $optarg ;; --with-python=*) diff --git a/src/DPM.cc b/src/DPM.cc index 595ee42ec8..0902ae9a45 100644 --- a/src/DPM.cc +++ b/src/DPM.cc @@ -74,7 +74,7 @@ void DPM::PostScriptInit() void DPM::AddConfig(const Analyzer::Config& cfg) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif diff --git a/src/File.cc b/src/File.cc index 080923ad37..d4e31bcc16 100644 --- a/src/File.cc +++ b/src/File.cc @@ -232,7 +232,7 @@ BroFile::~BroFile() delete [] access; delete [] cipher_buffer; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->UnIgnoreObject(this); #endif } @@ -255,7 +255,7 @@ void BroFile::Init() cipher_ctx = 0; cipher_buffer = 0; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->IgnoreObject(this); #endif } diff --git a/src/ID.cc b/src/ID.cc index 3f5c76ca1d..a70aa3fd0e 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -372,7 +372,7 @@ ID* ID::Unserialize(UnserialInfo* info) Ref(id); global_scope()->Insert(id->Name(), id); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->IgnoreObject(id); #endif } diff --git a/src/Login.cc b/src/Login.cc index 56efd12f53..e626fb3a0a 100644 --- a/src/Login.cc +++ b/src/Login.cc @@ -38,7 +38,7 @@ Login_Analyzer::Login_Analyzer(AnalyzerTag::Tag tag, Connection* conn) if ( ! re_skip_authentication ) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif re_skip_authentication = init_RE(skip_authentication); diff --git a/src/PersistenceSerializer.cc b/src/PersistenceSerializer.cc index c757467f90..d9baad05bb 100644 --- a/src/PersistenceSerializer.cc +++ b/src/PersistenceSerializer.cc @@ -137,7 +137,7 @@ bool PersistenceSerializer::CheckForFile(UnserialInfo* info, const char* file, bool PersistenceSerializer::ReadAll(bool is_init, bool delete_files) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index f29e907790..e9fbe0aab8 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -2645,7 +2645,7 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( current_peer->state == Peer::CLOSING ) return false; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG // Don't track allocations here, they'll be released only after the // main loop exists. And it's just a tiny amount anyway. HeapLeakChecker::Disabler disabler; @@ -2866,7 +2866,7 @@ void RemoteSerializer::GotID(ID* id, Val* val) (desc && *desc) ? desc : "not set"), current_peer); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG // May still be cached, but we don't care. heap_checker->IgnoreObject(id); #endif diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index da12b1b679..c9cf1f5c11 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -191,7 +191,7 @@ void RuleMatcher::Delete(RuleHdrTest* node) bool RuleMatcher::ReadFiles(const name_list& files) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 7abef72c46..2d0a8dfc5a 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -678,7 +678,7 @@ bool StateAccess::DoUnserialize(UnserialInfo* info) target.id = new ID(name, SCOPE_GLOBAL, true); Ref(target.id); global_scope()->Insert(name, target.id); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->IgnoreObject(target.id); #endif } diff --git a/src/main.cc b/src/main.cc index e484b58fe2..17a798ea81 100644 --- a/src/main.cc +++ b/src/main.cc @@ -65,7 +65,7 @@ extern "C" { #include "setsignal.h" }; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker* heap_checker = 0; int perftools_leaks = 0; int perftools_profile = 0; @@ -177,7 +177,7 @@ void usage() fprintf(stderr, " -W|--watchdog | activate watchdog timer\n"); fprintf(stderr, " -Z|--doc-scripts | generate documentation for all loaded scripts\n"); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG fprintf(stderr, " -m|--mem-leaks | show leaks [perftools]\n"); fprintf(stderr, " -M|--mem-profile | record heap [perftools]\n"); #endif @@ -244,7 +244,7 @@ void done_with_network() net_finish(1); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG if ( perftools_profile ) { @@ -424,7 +424,7 @@ int main(int argc, char** argv) #ifdef USE_IDMEF {"idmef-dtd", required_argument, 0, 'n'}, #endif -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG {"mem-leaks", no_argument, 0, 'm'}, {"mem-profile", no_argument, 0, 'M'}, #endif @@ -466,7 +466,7 @@ int main(int argc, char** argv) safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLOPSWbdghvZ", sizeof(opts)); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG strncat(opts, "mM", 2); #endif @@ -622,7 +622,7 @@ int main(int argc, char** argv) exit(0); break; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG case 'm': perftools_leaks = 1; break; @@ -758,14 +758,14 @@ int main(int argc, char** argv) // nevertheless reported; see perftools docs), thus // we suppress some messages here. -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG { HeapLeakChecker::Disabler disabler; #endif yyparse(); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG } #endif @@ -1019,7 +1019,7 @@ int main(int argc, char** argv) if ( profiling_logger ) profiling_logger->Log(); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG if ( perftools_leaks ) heap_checker = new HeapLeakChecker("net_run"); diff --git a/src/util.h b/src/util.h index 498bdf00e4..a4e3aa71b8 100644 --- a/src/util.h +++ b/src/util.h @@ -37,7 +37,7 @@ #endif -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG #include #include extern HeapLeakChecker* heap_checker; From c382439079755f1ca613881b1699ba77bfe9f246 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 28 Mar 2012 15:39:56 -0700 Subject: [PATCH 39/42] Switching log buffer size back to normal --- src/RemoteSerializer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index e9fbe0aab8..017f260bdf 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -234,7 +234,7 @@ static const int PRINT_BUFFER_SIZE = 10 * 1024; static const int SOCKBUF_SIZE = 1024 * 1024; // Buffer size for remote-log data. -static const int LOG_BUFFER_SIZE = 512; +static const int LOG_BUFFER_SIZE = 50 * 1024; struct ping_args { uint32 seq; From 6e7faafdb7bbf46306a186a033a85e3b34dd364d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 30 Mar 2012 12:40:31 -0500 Subject: [PATCH 40/42] Fix compile errors due to now-explicit IPAddr ctors and global IPFamily enum. --- src/IP.cc | 6 +++--- src/bro.bif | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/IP.cc b/src/IP.cc index 7f616fbbb0..bb60d17f15 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -74,8 +74,8 @@ RecordVal* IPv6_Hdr::BuildRecordVal(VectorVal* chain) const rv->Assign(2, new Val(ntohs(ip6->ip6_plen), TYPE_COUNT)); rv->Assign(3, new Val(ip6->ip6_nxt, TYPE_COUNT)); rv->Assign(4, new Val(ip6->ip6_hlim, TYPE_COUNT)); - rv->Assign(5, new AddrVal(ip6->ip6_src)); - rv->Assign(6, new AddrVal(ip6->ip6_dst)); + rv->Assign(5, new AddrVal(IPAddr(ip6->ip6_src))); + rv->Assign(6, new AddrVal(IPAddr(ip6->ip6_dst))); if ( ! chain ) chain = new VectorVal(new VectorType( hdrType(ip6_ext_hdr_type, "ip6_ext_hdr")->Ref())); @@ -314,7 +314,7 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, bool set_next, uint16 next) if ( ((const struct ip6_rthdr*)hdrs)->ip6r_segleft > 0 ) { const in6_addr* a = (const in6_addr*)(hdrs+len-16); - reporter->Weird(src, *a, "routing0_segleft"); + reporter->Weird(src, IPAddr(*a), "routing0_segleft"); } else { diff --git a/src/bro.bif b/src/bro.bif index 6766a89142..fa6766a7bf 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2070,7 +2070,7 @@ function routing0_data_to_addrs%(s: string%): addr_vec while ( len > 0 ) { - IPAddr a(IPAddr::IPv6, (const uint32*) bytes, IPAddr::Network); + IPAddr a(IPv6, (const uint32*) bytes, IPAddr::Network); rval->Assign(rval->Size(), new AddrVal(a), 0); bytes += 16; len -= 16; From c381da9ccfd956be506b17267e92077a0dc1868c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 3 Apr 2012 22:18:35 -0700 Subject: [PATCH 41/42] This could be fixing the memory problems finally. Keeping fingers crossed ... --- src/threading/Manager.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 43eb0313f4..7481e83192 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -73,14 +73,11 @@ void Manager::GetFds(int* read, int* write, int* except) double Manager::NextTimestamp(double* network_time) { - if ( ::network_time && ! next_beat ) - next_beat = ::network_time + HEART_BEAT_INTERVAL; - // fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat); - if ( did_process || ::network_time > next_beat ) + if ( did_process || ::network_time > next_beat || ! next_beat ) // If we had something to process last time (or out heartbeat - // is due), we want to check for more asap. + // is due or not set yet), we want to check for more asap. return timer_mgr->Time(); return -1.0; @@ -88,7 +85,13 @@ double Manager::NextTimestamp(double* network_time) void Manager::Process() { - bool do_beat = (next_beat && network_time > next_beat); + bool do_beat = false; + + if ( network_time && (network_time > next_beat || ! next_beat) ) + { + do_beat = true; + next_beat = ::network_time + HEART_BEAT_INTERVAL; + } did_process = false; @@ -97,10 +100,7 @@ void Manager::Process() MsgThread* t = *i; if ( do_beat ) - { t->Heartbeat(); - next_beat = 0; - } while ( t->HasOut() ) { From fe61d02a9f9d32620a9783f1c61a0fe56880235f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Apr 2012 10:42:50 -0700 Subject: [PATCH 42/42] Two more tweaks to reliably terminate when reading from trace. --- src/threading/Manager.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 7481e83192..abdbf849b5 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -43,6 +43,7 @@ void Manager::Terminate() msg_threads.clear(); idle = true; + closed = true; terminating = false; } @@ -75,7 +76,7 @@ double Manager::NextTimestamp(double* network_time) { // fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat); - if ( did_process || ::network_time > next_beat || ! next_beat ) + if ( ::network_time && (did_process || ::network_time > next_beat || ! next_beat) ) // If we had something to process last time (or out heartbeat // is due or not set yet), we want to check for more asap. return timer_mgr->Time();