From ca17a1cf46ce50a84038eecfa5da401351777d4b Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 16:18:54 -0800 Subject: [PATCH 01/56] make logging framework send the protocol to the writer. for use in future writers, that have a special type for port, which includes the protocol. --- src/LogMgr.cc | 32 +++++++++++++++++++++++++++++--- src/LogMgr.h | 4 ++++ src/LogWriterAscii.cc | 5 ++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 0b706f6417..8873b22b46 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -114,6 +114,10 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } + +// if ( type == TYPE_PORT && present ) +// delete val.port_val.proto; + } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -186,9 +190,12 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Read(&val.uint_val, "uint"); + case TYPE_PORT: + val.port_val.proto = new string; + return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -301,9 +308,11 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Write(val.uint_val, "uint"); + case TYPE_PORT: + return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -1062,6 +1071,22 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } +string LogMgr::TransportProtoToString(TransportProto p) { + switch ( p ) { + case TRANSPORT_UNKNOWN: + return "unknown"; + case TRANSPORT_TCP: + return "tcp"; + case TRANSPORT_UDP: + return "udp"; + case TRANSPORT_ICMP: + return "icmp"; + } + + assert(false); + return ""; +} + LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1093,7 +1118,8 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; case TYPE_PORT: - lval->val.uint_val = val->AsPortVal()->Port(); + lval->val.port_val.port = val->AsPortVal()->Port(); + lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..52acd04be5 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -34,10 +34,12 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; + struct port_t { bro_uint_t port; string* proto; }; union _val { bro_int_t int_val; bro_uint_t uint_val; + port_t port_val; uint32 addr_val[NUM_ADDR_WORDS]; subnet_type subnet_val; double double_val; @@ -132,6 +134,8 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); + string TransportProtoToString(TransportProto p); + vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9b1fda3b62..c449c1a788 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -169,10 +169,13 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: desc->Add(val->val.uint_val); break; + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + case TYPE_SUBNET: desc->Add(dotted_addr(val->val.subnet_val.net)); desc->Add("/"); From a0da991030836d53bc669b64ad3de9b4dba34070 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:56:26 -0800 Subject: [PATCH 02/56] memleak fix. --- src/LogMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 8873b22b46..729979b4ef 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -115,8 +115,8 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } -// if ( type == TYPE_PORT && present ) -// delete val.port_val.proto; + if ( type == TYPE_PORT && present ) + delete val.port_val.proto; } From 4b3cc95f7206d61614a0193508acc0f60828e3df Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 7 Dec 2011 12:43:15 -0800 Subject: [PATCH 03/56] send enum instead of string --- src/LogMgr.cc | 50 ++++++++++++++++++++++++++------------------------ src/LogMgr.h | 4 +--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 729979b4ef..58581e2943 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -115,9 +115,6 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } - if ( type == TYPE_PORT && present ) - delete val.port_val.proto; - } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -192,9 +189,30 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNTER: return fmt->Read(&val.uint_val, "uint"); - case TYPE_PORT: - val.port_val.proto = new string; - return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_PORT: { + int proto; + if ( ! (fmt->Read(&val.port_val.port, "port") && fmt->Read(&proto, "proto") ) ) { + return false; + } + + switch (proto) { + case 0: + val.port_val.proto = TRANSPORT_UNKNOWN; + break; + case 1: + val.port_val.proto = TRANSPORT_TCP; + break; + case 2: + val.port_val.proto = TRANSPORT_UDP; + break; + case 3: + val.port_val.proto = TRANSPORT_ICMP; + break; + default: + return false; + } + return true; + } case TYPE_SUBNET: { @@ -311,7 +329,7 @@ bool LogVal::Write(SerializationFormat* fmt) const return fmt->Write(val.uint_val, "uint"); case TYPE_PORT: - return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); case TYPE_SUBNET: { @@ -1071,22 +1089,6 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } -string LogMgr::TransportProtoToString(TransportProto p) { - switch ( p ) { - case TRANSPORT_UNKNOWN: - return "unknown"; - case TRANSPORT_TCP: - return "tcp"; - case TRANSPORT_UDP: - return "udp"; - case TRANSPORT_ICMP: - return "icmp"; - } - - assert(false); - return ""; -} - LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1119,7 +1121,7 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) case TYPE_PORT: lval->val.port_val.port = val->AsPortVal()->Port(); - lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); + lval->val.port_val.proto = val->AsPortVal()->PortType(); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 52acd04be5..8c2c8250f8 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -34,7 +34,7 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; - struct port_t { bro_uint_t port; string* proto; }; + struct port_t { bro_uint_t port; TransportProto proto; }; union _val { bro_int_t int_val; @@ -134,8 +134,6 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); - string TransportProtoToString(TransportProto p); - vector streams; // Indexed by stream enum. }; From e4e770d47517ad6b9c730a7132037feb265ffe30 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 26 Jan 2012 17:47:36 -0800 Subject: [PATCH 04/56] Threaded logging framework. This is based on Gilbert's code but I ended up refactoring it quite a bit. That's why I didn't do a direct merge but started with a new branch and copied things over to adapt. It looks quite a bit different now as I tried to generalize things a bit more to also support the Input Framework. The larger changes code are: - Moved all logging code into subdirectory src/logging/. Code here is in namespace "logging". - Moved all threading code into subdirectory src/threading/. Code here is in namespace "threading". - Introduced a central thread manager that tracks threads and is in charge of termination and (eventually) statistics. - Refactored logging independent threading code into base classes BasicThread and MsgThread. The former encapsulates all the pthread code with simple start/stop methods and provides a single Run() method to override. The latter is derived from BasicThread and adds bi-directional message passing between main and child threads. The hope is that the Input Framework can reuse this part quite directly. - A log writer is now split into a general WriterFrontend (LogEmissary in Gilbert's code) and a type-specific WriterBackend. Specific writers are implemented by deriving from the latter. (The plugin interface is almost unchanged compared to the 2.0 version.). Frontend and backend communicate via MsgThread's message passing. - MsgThread (and thus WriterBackend) has a Heartbeat() method that a thread can override to execute code on a regular basis. It's triggered roughly once a second by the main thread. - Integration into "the rest of Bro". Threads can send messages to the reporter and do debugging output; they are hooked into the I/O loop for sending messages back; and there's a new debugging stream "threading" that logs, well, threading activity. This all seems to work for the most part, but it's not done yet. TODO list: - Not all tests pass yet. In particular, diffs for the external tests seem to indicate some memory problem (no crashes, just an occasional weird character). - Only tested in --enable-debug mode. - Only tested on Linux. - Needs leak check. - Each log write is currently a single inter-thread message. Bring Gilbert's bulk writes back. - Code needs further cleanup. - Document the class API. - Document the internal structure of the logging framework. - Check for robustness: live traffic, aborting, signals, etc. - Add thread statistics to profile.log (most of the code is there). - Customize the OS-visible thread names on platforms that support it. --- src/Attr.cc | 4 +- src/CMakeLists.txt | 19 +- src/DebugLogger.cc | 2 +- src/DebugLogger.h | 1 + src/LogWriter.cc | 158 ---------- src/LogWriterNone.cc | 16 - src/LogWriterNone.h | 30 -- src/RemoteSerializer.cc | 22 +- src/RemoteSerializer.h | 15 +- src/logging.bif | 3 +- src/{LogMgr.cc => logging/Manager.cc} | 289 +++++++++--------- src/{LogMgr.h => logging/Manager.h} | 75 +++-- src/logging/WriterBackend.cc | 161 ++++++++++ src/{LogWriter.h => logging/WriterBackend.h} | 123 ++++---- src/logging/WriterFrontend.cc | 175 +++++++++++ src/logging/WriterFrontend.h | 66 ++++ .../writers/Ascii.cc} | 41 +-- .../writers/Ascii.h} | 34 ++- src/logging/writers/None.cc | 19 ++ src/logging/writers/None.h | 35 +++ src/main.cc | 20 +- src/threading/BasicThread.cc | 129 ++++++++ src/threading/BasicThread.h | 63 ++++ src/threading/Manager.cc | 104 +++++++ src/threading/Manager.h | 52 ++++ src/threading/MsgThread.cc | 285 +++++++++++++++++ src/threading/MsgThread.h | 157 ++++++++++ src/threading/Queue.h | 150 +++++++++ 28 files changed, 1745 insertions(+), 503 deletions(-) delete mode 100644 src/LogWriter.cc delete mode 100644 src/LogWriterNone.cc delete mode 100644 src/LogWriterNone.h rename src/{LogMgr.cc => logging/Manager.cc} (87%) rename src/{LogMgr.h => logging/Manager.h} (68%) create mode 100644 src/logging/WriterBackend.cc rename src/{LogWriter.h => logging/WriterBackend.h} (70%) create mode 100644 src/logging/WriterFrontend.cc create mode 100644 src/logging/WriterFrontend.h rename src/{LogWriterAscii.cc => logging/writers/Ascii.cc} (89%) rename src/{LogWriterAscii.h => logging/writers/Ascii.h} (57%) create mode 100644 src/logging/writers/None.cc create mode 100644 src/logging/writers/None.h create mode 100644 src/threading/BasicThread.cc create mode 100644 src/threading/BasicThread.h create mode 100644 src/threading/Manager.cc create mode 100644 src/threading/Manager.h create mode 100644 src/threading/MsgThread.cc create mode 100644 src/threading/MsgThread.h create mode 100644 src/threading/Queue.h diff --git a/src/Attr.cc b/src/Attr.cc index aed9165182..b877250f52 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -5,7 +5,7 @@ #include "Attr.h" #include "Expr.h" #include "Serializer.h" -#include "LogMgr.h" +#include "logging/Manager.h" const char* attr_name(attr_tag t) { @@ -416,7 +416,7 @@ void Attributes::CheckAttr(Attr* a) break; case ATTR_LOG: - if ( ! LogVal::IsCompatibleType(type) ) + if ( ! logging::Value::IsCompatibleType(type) ) Error("&log applied to a type that cannot be logged"); break; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0e29082db3..61a4847b70 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -213,6 +213,8 @@ binpac_target(syslog.pac ######################################################################## ## bro target +find_package (Threads) + # This macro stores associated headers for any C/C++ source files given # as arguments (past _var) as a list in the CMake variable named "_var". macro(COLLECT_HEADERS _var) @@ -334,10 +336,6 @@ set(bro_SRCS IRC.cc List.cc Reporter.cc - LogMgr.cc - LogWriter.cc - LogWriterAscii.cc - LogWriterNone.cc Login.cc MIME.cc NCP.cc @@ -409,6 +407,17 @@ set(bro_SRCS PacketDumper.cc strsep.c modp_numtoa.c + + threading/BasicThread.cc + threading/Manager.cc + threading/MsgThread.cc + + logging/Manager.cc + logging/WriterBackend.cc + logging/WriterFrontend.cc + logging/writers/Ascii.cc + logging/writers/None.cc + ${dns_SRCS} ${openssl_SRCS} ) @@ -421,7 +430,7 @@ add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}") add_executable(bro ${bro_SRCS} ${bro_HEADERS}) -target_link_libraries(bro ${brodeps}) +target_link_libraries(bro ${brodeps} ${CMAKE_THREAD_LIBS_INIT}) install(TARGETS bro DESTINATION bin) install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base) diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index d60fdd70c8..c41a0552c6 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -15,7 +15,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, - { "logging", 0, false } + { "logging", 0, false }, { "threading", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index a2dece5b3c..71e21bfa26 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -24,6 +24,7 @@ enum DebugStream { DBG_DPD, // Dynamic application detection framework DBG_TM, // Time-machine packet input via Brocolli DBG_LOGGING, // Logging streams + DBG_THREADING, // Threading system NUM_DBGS // Has to be last }; diff --git a/src/LogWriter.cc b/src/LogWriter.cc deleted file mode 100644 index 8584a0b0b5..0000000000 --- a/src/LogWriter.cc +++ /dev/null @@ -1,158 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "util.h" -#include "LogWriter.h" - -LogWriter::LogWriter() - { - buf = 0; - buf_len = 1024; - buffering = true; - disabled = false; - } - -LogWriter::~LogWriter() - { - if ( buf ) - free(buf); - - for(int i = 0; i < num_fields; ++i) - delete fields[i]; - - delete [] fields; - } - -bool LogWriter::Init(string arg_path, int arg_num_fields, - const LogField* const * arg_fields) - { - path = arg_path; - num_fields = arg_num_fields; - fields = arg_fields; - - if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) - { - disabled = true; - return false; - } - - return true; - } - -bool LogWriter::Write(int arg_num_fields, LogVal** vals) - { - // Double-check that the arguments match. If we get this from remote, - // something might be mixed up. - if ( num_fields != arg_num_fields ) - { - DBG_LOG(DBG_LOGGING, "Number of fields don't match in LogWriter::Write() (%d vs. %d)", - arg_num_fields, num_fields); - - DeleteVals(vals); - return false; - } - - for ( int i = 0; i < num_fields; ++i ) - { - if ( vals[i]->type != fields[i]->type ) - { - DBG_LOG(DBG_LOGGING, "Field type doesn't match in LogWriter::Write() (%d vs. %d)", - vals[i]->type, fields[i]->type); - DeleteVals(vals); - return false; - } - } - - bool result = DoWrite(num_fields, fields, vals); - - DeleteVals(vals); - - if ( ! result ) - disabled = true; - - return result; - } - -bool LogWriter::SetBuf(bool enabled) - { - if ( enabled == buffering ) - // No change. - return true; - - buffering = enabled; - - if ( ! DoSetBuf(enabled) ) - { - disabled = true; - return false; - } - - return true; - } - -bool LogWriter::Rotate(string rotated_path, double open, - double close, bool terminating) - { - if ( ! DoRotate(rotated_path, open, close, terminating) ) - { - disabled = true; - return false; - } - - return true; - } - -bool LogWriter::Flush() - { - if ( ! DoFlush() ) - { - disabled = true; - return false; - } - - return true; - } - -void LogWriter::Finish() - { - DoFinish(); - } - -const char* LogWriter::Fmt(const char* format, ...) - { - if ( ! buf ) - buf = (char*) malloc(buf_len); - - va_list al; - va_start(al, format); - int n = safe_vsnprintf(buf, buf_len, format, al); - va_end(al); - - if ( (unsigned int) n >= buf_len ) - { // Not enough room, grow the buffer. - buf_len = n + 32; - buf = (char*) realloc(buf, buf_len); - - // Is it portable to restart? - va_start(al, format); - n = safe_vsnprintf(buf, buf_len, format, al); - va_end(al); - } - - return buf; - } - -void LogWriter::Error(const char *msg) - { - log_mgr->Error(this, msg); - } - -void LogWriter::DeleteVals(LogVal** vals) - { - log_mgr->DeleteVals(num_fields, vals); - } - -bool LogWriter::FinishedRotation(string new_name, string old_name, double open, - double close, bool terminating) - { - return log_mgr->FinishedRotation(this, new_name, old_name, open, close, terminating); - } diff --git a/src/LogWriterNone.cc b/src/LogWriterNone.cc deleted file mode 100644 index 592772afdb..0000000000 --- a/src/LogWriterNone.cc +++ /dev/null @@ -1,16 +0,0 @@ - -#include "LogWriterNone.h" - -bool LogWriterNone::DoRotate(string rotated_path, double open, - double close, bool terminating) - { - if ( ! FinishedRotation(string("/dev/null"), Path(), open, close, terminating)) - { - Error(Fmt("error rotating %s", Path().c_str())); - return false; - } - - return true; - } - - diff --git a/src/LogWriterNone.h b/src/LogWriterNone.h deleted file mode 100644 index 3811a19469..0000000000 --- a/src/LogWriterNone.h +++ /dev/null @@ -1,30 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. -// -// Dummy log writer that just discards everything (but still pretends to rotate). - -#ifndef LOGWRITERNONE_H -#define LOGWRITERNONE_H - -#include "LogWriter.h" - -class LogWriterNone : public LogWriter { -public: - LogWriterNone() {} - ~LogWriterNone() {}; - - static LogWriter* Instantiate() { return new LogWriterNone; } - -protected: - virtual bool DoInit(string path, int num_fields, - const LogField* const * fields) { return true; } - - virtual bool DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals) { return true; } - virtual bool DoSetBuf(bool enabled) { return true; } - virtual bool DoRotate(string rotated_path, double open, double close, - bool terminating); - virtual bool DoFlush() { return true; } - virtual void DoFinish() {} -}; - -#endif diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index b72a6dcc1a..a75812b42b 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -183,8 +183,8 @@ #include "Sessions.h" #include "File.h" #include "Conn.h" -#include "LogMgr.h" #include "Reporter.h" +#include "logging/Manager.h" extern "C" { #include "setsignal.h" @@ -2476,7 +2476,7 @@ bool RemoteSerializer::ProcessRemotePrint() return true; } -bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields) +bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields) { loop_over_list(peers, i) { @@ -2486,7 +2486,7 @@ bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string return true; } -bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields) +bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields) { SetErrorDescr("logging"); @@ -2540,7 +2540,7 @@ error: return false; } -bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals) +bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals) { loop_over_list(peers, i) { @@ -2550,7 +2550,7 @@ bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, i return true; } -bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals) +bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals) { if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) return false; @@ -2641,7 +2641,7 @@ bool RemoteSerializer::ProcessLogCreateWriter() EnumVal* id_val = 0; EnumVal* writer_val = 0; - LogField** fields = 0; + logging::Field** fields = 0; BinarySerializationFormat fmt; fmt.StartRead(current_args->data, current_args->len); @@ -2658,11 +2658,11 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( ! success ) goto error; - fields = new LogField* [num_fields]; + fields = new logging::Field* [num_fields]; for ( int i = 0; i < num_fields; i++ ) { - fields[i] = new LogField; + fields[i] = new logging::Field; if ( ! fields[i]->Read(&fmt) ) goto error; } @@ -2703,7 +2703,7 @@ bool RemoteSerializer::ProcessLogWrite() // Unserialize one entry. EnumVal* id_val = 0; EnumVal* writer_val = 0; - LogVal** vals = 0; + logging::Value** vals = 0; int id, writer; string path; @@ -2717,11 +2717,11 @@ bool RemoteSerializer::ProcessLogWrite() if ( ! success ) goto error; - vals = new LogVal* [num_fields]; + vals = new logging::Value* [num_fields]; for ( int i = 0; i < num_fields; i++ ) { - vals[i] = new LogVal; + vals[i] = new logging::Value; if ( ! vals[i]->Read(&fmt) ) goto error; } diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index b64fdcbe66..ba0bde7d41 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -14,8 +14,11 @@ // FIXME: Change this to network byte order class IncrementalSendTimer; -class LogField; -class LogVal; + +namespace logging { + class Field; + class Value; +} // This class handles the communication done in Bro's main loop. class RemoteSerializer : public Serializer, public IOSource { @@ -99,13 +102,13 @@ public: bool SendPrintHookEvent(BroFile* f, const char* txt, size_t len); // Send a request to create a writer on a remote side. - bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields); + bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields); // Broadcasts a request to create a writer. - bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields); + bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields); // Broadcast a log entry to everybody interested. - bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals); + bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals); // Synchronzizes time with all connected peers. Returns number of // current sync-point, or -1 on error. @@ -300,7 +303,7 @@ protected: bool SendID(SerialInfo* info, Peer* peer, const ID& id); bool SendCapabilities(Peer* peer); bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p); - bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals); + bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals); void UnregisterHandlers(Peer* peer); void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0); diff --git a/src/logging.bif b/src/logging.bif index 31e1bebacd..c8960b4e38 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -3,8 +3,9 @@ module Log; %%{ -#include "LogMgr.h" #include "NetVar.h" + +#include "logging/Manager.h" %%} type Filter: record; diff --git a/src/LogMgr.cc b/src/logging/Manager.cc similarity index 87% rename from src/LogMgr.cc rename to src/logging/Manager.cc index 28e9a2ac1f..09c5030fdc 100644 --- a/src/LogMgr.cc +++ b/src/logging/Manager.cc @@ -2,33 +2,38 @@ #include -#include "LogMgr.h" -#include "Event.h" -#include "EventHandler.h" -#include "NetVar.h" -#include "Net.h" +#include "../Event.h" +#include "../EventHandler.h" +#include "../NetVar.h" +#include "../Net.h" -#include "LogWriterAscii.h" -#include "LogWriterNone.h" +#include "Manager.h" +#include "WriterFrontend.h" +#include "WriterBackend.h" + +#include "writers/Ascii.h" +#include "writers/None.h" + +using namespace logging; // Structure describing a log writer type. -struct LogWriterDefinition { +struct WriterDefinition { bro_int_t type; // The type. const char *name; // Descriptive name for error messages. bool (*init)(); // An optional one-time initialization function. - LogWriter* (*factory)(); // A factory function creating instances. + WriterBackend* (*factory)(); // A factory function creating instances. }; // Static table defining all availabel log writers. -LogWriterDefinition log_writers[] = { - { BifEnum::Log::WRITER_NONE, "None", 0, LogWriterNone::Instantiate }, - { BifEnum::Log::WRITER_ASCII, "Ascii", 0, LogWriterAscii::Instantiate }, +WriterDefinition log_writers[] = { + { BifEnum::Log::WRITER_NONE, "None", 0, writer::None::Instantiate }, + { BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate }, // End marker, don't touch. - { BifEnum::Log::WRITER_DEFAULT, "None", 0, (LogWriter* (*)())0 } + { BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)())0 } }; -struct LogMgr::Filter { +struct Manager::Filter { string name; EnumVal* id; Func* pred; @@ -42,7 +47,7 @@ struct LogMgr::Filter { Func* postprocessor; int num_fields; - LogField** fields; + Field** fields; // Vector indexed by field number. Each element is a list of record // indices defining a path leading to the value across potential @@ -52,16 +57,16 @@ struct LogMgr::Filter { ~Filter(); }; -struct LogMgr::WriterInfo { +struct Manager::WriterInfo { EnumVal* type; double open_time; Timer* rotation_timer; double interval; Func* postprocessor; - LogWriter* writer; + WriterFrontend* writer; }; -struct LogMgr::Stream { +struct Manager::Stream { EnumVal* id; bool enabled; string name; @@ -78,7 +83,7 @@ struct LogMgr::Stream { ~Stream(); }; -bool LogField::Read(SerializationFormat* fmt) +bool Field::Read(SerializationFormat* fmt) { int t; int st; @@ -90,12 +95,12 @@ bool LogField::Read(SerializationFormat* fmt) return success; } -bool LogField::Write(SerializationFormat* fmt) const +bool Field::Write(SerializationFormat* fmt) const { return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); } -LogVal::~LogVal() +Value::~Value() { if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC) && present ) @@ -118,7 +123,7 @@ LogVal::~LogVal() } } -bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) +bool Value::IsCompatibleType(BroType* t, bool atomic_only) { if ( ! t ) return false; @@ -169,7 +174,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) return false; } -bool LogVal::Read(SerializationFormat* fmt) +bool Value::Read(SerializationFormat* fmt) { int ty; @@ -249,11 +254,11 @@ bool LogVal::Read(SerializationFormat* fmt) if ( ! fmt->Read(&val.set_val.size, "set_size") ) return false; - val.set_val.vals = new LogVal* [val.set_val.size]; + val.set_val.vals = new Value* [val.set_val.size]; for ( int i = 0; i < val.set_val.size; ++i ) { - val.set_val.vals[i] = new LogVal; + val.set_val.vals[i] = new Value; if ( ! val.set_val.vals[i]->Read(fmt) ) return false; @@ -267,11 +272,11 @@ bool LogVal::Read(SerializationFormat* fmt) if ( ! fmt->Read(&val.vector_val.size, "vector_size") ) return false; - val.vector_val.vals = new LogVal* [val.vector_val.size]; + val.vector_val.vals = new Value* [val.vector_val.size]; for ( int i = 0; i < val.vector_val.size; ++i ) { - val.vector_val.vals[i] = new LogVal; + val.vector_val.vals[i] = new Value; if ( ! val.vector_val.vals[i]->Read(fmt) ) return false; @@ -281,13 +286,13 @@ bool LogVal::Read(SerializationFormat* fmt) } default: - reporter->InternalError("unsupported type %s in LogVal::Write", type_name(type)); + reporter->InternalError("unsupported type %s in Value::Write", type_name(type)); } return false; } -bool LogVal::Write(SerializationFormat* fmt) const +bool Value::Write(SerializationFormat* fmt) const { if ( ! (fmt->Write((int)type, "type") && fmt->Write(present, "present")) ) @@ -382,13 +387,13 @@ bool LogVal::Write(SerializationFormat* fmt) const } default: - reporter->InternalError("unsupported type %s in LogVal::REad", type_name(type)); + reporter->InternalError("unsupported type %s in Value::REad", type_name(type)); } return false; } -LogMgr::Filter::~Filter() +Manager::Filter::~Filter() { for ( int i = 0; i < num_fields; ++i ) delete fields[i]; @@ -398,7 +403,7 @@ LogMgr::Filter::~Filter() Unref(path_val); } -LogMgr::Stream::~Stream() +Manager::Stream::~Stream() { Unref(columns); @@ -421,17 +426,64 @@ LogMgr::Stream::~Stream() delete *f; } -LogMgr::LogMgr() +Manager::Manager() { } -LogMgr::~LogMgr() +Manager::~Manager() { for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) delete *s; } -LogMgr::Stream* LogMgr::FindStream(EnumVal* id) +WriterBackend* Manager::CreateBackend(bro_int_t type) + { + WriterDefinition* ld = log_writers; + + while ( true ) + { + if ( ld->type == BifEnum::Log::WRITER_DEFAULT ) + { + reporter->Error("unknow writer when creating writer"); + return 0; + } + + if ( ld->type == type ) + break; + + if ( ! ld->factory ) + // Oops, we can't instantiate this guy. + return 0; + + // If the writer has an init function, call it. + if ( ld->init ) + { + if ( (*ld->init)() ) + // Clear the init function so that we won't + // call it again later. + ld->init = 0; + else + // Init failed, disable by deleting factory + // function. + ld->factory = 0; + + DBG_LOG(DBG_LOGGING, "failed to init writer class %s", + ld->name); + + return false; + } + + ++ld; + } + + assert(ld->factory); + + WriterBackend* backend = (*ld->factory)(); + assert(backend); + return backend; + } + +Manager::Stream* Manager::FindStream(EnumVal* id) { unsigned int idx = id->AsEnum(); @@ -441,7 +493,7 @@ LogMgr::Stream* LogMgr::FindStream(EnumVal* id) return streams[idx]; } -LogMgr::WriterInfo* LogMgr::FindWriter(LogWriter* writer) +Manager::WriterInfo* Manager::FindWriter(WriterFrontend* writer) { for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) { @@ -460,7 +512,7 @@ LogMgr::WriterInfo* LogMgr::FindWriter(LogWriter* writer) return 0; } -void LogMgr::RemoveDisabledWriters(Stream* stream) +void Manager::RemoveDisabledWriters(Stream* stream) { list disabled; @@ -468,6 +520,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) { if ( j->second && j->second->writer->Disabled() ) { + j->second->writer->Stop(); delete j->second; disabled.push_back(j->first); } @@ -477,7 +530,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) +bool Manager::CreateStream(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); @@ -497,7 +550,7 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) if ( ! (columns->FieldDecl(i)->FindAttr(ATTR_LOG)) ) continue; - if ( ! LogVal::IsCompatibleType(columns->FieldType(i)) ) + if ( ! Value::IsCompatibleType(columns->FieldType(i)) ) { reporter->Error("type of field '%s' is not support for logging output", columns->FieldName(i)); @@ -569,7 +622,7 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) return true; } -bool LogMgr::EnableStream(EnumVal* id) +bool Manager::EnableStream(EnumVal* id) { Stream* stream = FindStream(id); @@ -585,7 +638,7 @@ bool LogMgr::EnableStream(EnumVal* id) return true; } -bool LogMgr::DisableStream(EnumVal* id) +bool Manager::DisableStream(EnumVal* id) { Stream* stream = FindStream(id); @@ -602,7 +655,7 @@ bool LogMgr::DisableStream(EnumVal* id) } // Helper for recursive record field unrolling. -bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, +bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices) { for ( int i = 0; i < rt->NumFields(); ++i ) @@ -696,9 +749,9 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, filter->indices.push_back(new_indices); - filter->fields = (LogField**) + filter->fields = (Field**) realloc(filter->fields, - sizeof(LogField) * ++filter->num_fields); + sizeof(Field) * ++filter->num_fields); if ( ! filter->fields ) { @@ -706,14 +759,14 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, return false; } - LogField* field = new LogField(); + Field* field = new Field(); field->name = new_path; field->type = t->Tag(); - if ( field->type == TYPE_TABLE ) + if ( field->type == TYPE_TABLE ) { field->subtype = t->AsSetType()->Indices()->PureType()->Tag(); - } - else if ( field->type == TYPE_VECTOR ) + } + else if ( field->type == TYPE_VECTOR ) { field->subtype = t->AsVectorType()->YieldType()->Tag(); } @@ -723,7 +776,7 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, return true; } -bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) +bool Manager::AddFilter(EnumVal* id, RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); @@ -819,7 +872,7 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) for ( int i = 0; i < filter->num_fields; i++ ) { - LogField* field = filter->fields[i]; + Field* field = filter->fields[i]; DBG_LOG(DBG_LOGGING, " field %10s: %s", field->name.c_str(), type_name(field->type)); } @@ -828,12 +881,12 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) return true; } -bool LogMgr::RemoveFilter(EnumVal* id, StringVal* name) +bool Manager::RemoveFilter(EnumVal* id, StringVal* name) { return RemoveFilter(id, name->AsString()->CheckString()); } -bool LogMgr::RemoveFilter(EnumVal* id, string name) +bool Manager::RemoveFilter(EnumVal* id, string name) { Stream* stream = FindStream(id); if ( ! stream ) @@ -860,7 +913,7 @@ bool LogMgr::RemoveFilter(EnumVal* id, string name) return true; } -bool LogMgr::Write(EnumVal* id, RecordVal* columns) +bool Manager::Write(EnumVal* id, RecordVal* columns) { bool error = false; @@ -980,7 +1033,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(filter->writer->AsEnum(), path)); - LogWriter* writer = 0; + WriterFrontend* writer = 0; if ( w != stream->writers.end() ) // We know this writer already. @@ -990,12 +1043,12 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) { // No, need to create one. - // Copy the fields for LogWriter::Init() as it will take - // ownership. - LogField** arg_fields = new LogField*[filter->num_fields]; + // Copy the fields for WriterFrontend::Init() as it + // will take ownership. + Field** arg_fields = new Field*[filter->num_fields]; for ( int j = 0; j < filter->num_fields; ++j ) - arg_fields[j] = new LogField(*filter->fields[j]); + arg_fields[j] = new Field(*filter->fields[j]); if ( filter->remote ) remote_serializer->SendLogCreateWriter(stream->id, @@ -1034,7 +1087,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) if ( filter->local || filter->remote ) { - LogVal** vals = RecordToFilterVals(stream, filter, columns); + Value** vals = RecordToFilterVals(stream, filter, columns); if ( filter->remote ) remote_serializer->SendLogWrite(stream->id, @@ -1045,11 +1098,9 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) if ( filter->local ) { - assert(writer); - // Write takes ownership of vals. - if ( ! writer->Write(filter->num_fields, vals) ) - error = true; + assert(writer); + writer->Write(filter->num_fields, vals); } else @@ -1072,15 +1123,15 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } -LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) +Value* Manager::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) ty = val->Type(); if ( ! val ) - return new LogVal(ty->Tag(), false); + return new Value(ty->Tag(), false); - LogVal* lval = new LogVal(ty->Tag()); + Value* lval = new Value(ty->Tag()); switch ( lval->type ) { case TYPE_BOOL: @@ -1160,7 +1211,7 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) set = new ListVal(TYPE_INT); lval->val.set_val.size = set->Length(); - lval->val.set_val.vals = new LogVal* [lval->val.set_val.size]; + lval->val.set_val.vals = new Value* [lval->val.set_val.size]; for ( int i = 0; i < lval->val.set_val.size; i++ ) lval->val.set_val.vals[i] = ValToLogVal(set->Index(i)); @@ -1174,7 +1225,7 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) VectorVal* vec = val->AsVectorVal(); lval->val.vector_val.size = vec->Size(); lval->val.vector_val.vals = - new LogVal* [lval->val.vector_val.size]; + new Value* [lval->val.vector_val.size]; for ( int i = 0; i < lval->val.vector_val.size; i++ ) { @@ -1193,10 +1244,10 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) return lval; } -LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, +Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns) { - LogVal** vals = new LogVal*[filter->num_fields]; + Value** vals = new Value*[filter->num_fields]; for ( int i = 0; i < filter->num_fields; ++i ) { @@ -1215,7 +1266,7 @@ LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, if ( ! val ) { // Value, or any of its parents, is not set. - vals[i] = new LogVal(filter->fields[i]->type, false); + vals[i] = new Value(filter->fields[i]->type, false); break; } } @@ -1227,8 +1278,8 @@ LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, return vals; } -LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, LogField** fields) +WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, + int num_fields, Field** fields) { Stream* stream = FindStream(id); @@ -1244,56 +1295,10 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, // return it. return w->second->writer; - // Need to instantiate a new writer. + WriterFrontend* writer_obj = new WriterFrontend(writer->AsEnum()); + assert(writer_obj); - LogWriterDefinition* ld = log_writers; - - while ( true ) - { - if ( ld->type == BifEnum::Log::WRITER_DEFAULT ) - { - reporter->Error("unknow writer when creating writer"); - return 0; - } - - if ( ld->type == writer->AsEnum() ) - break; - - if ( ! ld->factory ) - // Oops, we can't instantiate this guy. - return 0; - - // If the writer has an init function, call it. - if ( ld->init ) - { - if ( (*ld->init)() ) - // Clear the init function so that we won't - // call it again later. - ld->init = 0; - else - // Init failed, disable by deleting factory - // function. - ld->factory = 0; - - DBG_LOG(DBG_LOGGING, "failed to init writer class %s", - ld->name); - - return false; - } - - ++ld; - } - - assert(ld->factory); - LogWriter* writer_obj = (*ld->factory)(); - - if ( ! writer_obj->Init(path, num_fields, fields) ) - { - DBG_LOG(DBG_LOGGING, "failed to init instance of writer %s", - ld->name); - - return 0; - } + writer_obj->Init(path, num_fields, fields); WriterInfo* winfo = new WriterInfo; winfo->type = writer->Ref()->AsEnumVal(); @@ -1338,16 +1343,17 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, return writer_obj; } -void LogMgr::DeleteVals(int num_fields, LogVal** vals) +void Manager::DeleteVals(int num_fields, Value** vals) { + // Note this code is duplicated in WriterBackend::DeleteVals(). for ( int i = 0; i < num_fields; i++ ) delete vals[i]; delete [] vals; } -bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, - LogVal** vals) +bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, + Value** vals) { Stream* stream = FindStream(id); @@ -1357,7 +1363,7 @@ bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, #ifdef DEBUG ODesc desc; id->Describe(&desc); - DBG_LOG(DBG_LOGGING, "unknown stream %s in LogMgr::Write()", + DBG_LOG(DBG_LOGGING, "unknown stream %s in Manager::Write()", desc.Description()); #endif DeleteVals(num_fields, vals); @@ -1379,23 +1385,24 @@ bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, #ifdef DEBUG ODesc desc; id->Describe(&desc); - DBG_LOG(DBG_LOGGING, "unknown writer %s in LogMgr::Write()", + DBG_LOG(DBG_LOGGING, "unknown writer %s in Manager::Write()", desc.Description()); #endif DeleteVals(num_fields, vals); return false; } - bool success = (w->second ? w->second->writer->Write(num_fields, vals) : true); + if ( w->second ) + w->second->writer->Write(num_fields, vals); DBG_LOG(DBG_LOGGING, - "Wrote pre-filtered record to path '%s' on stream '%s' [%s]", - path.c_str(), stream->name.c_str(), (success ? "ok" : "error")); + "Wrote pre-filtered record to path '%s' on stream '%s'", + path.c_str(), stream->name.c_str()); - return success; + return true; } -void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) +void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer) { for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) { @@ -1410,7 +1417,7 @@ void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) if ( ! i->second ) continue; - LogWriter* writer = i->second->writer; + WriterFrontend* writer = i->second->writer; EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer); remote_serializer->SendLogCreateWriter(peer, (*s)->id, @@ -1422,7 +1429,7 @@ void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) } } -bool LogMgr::SetBuf(EnumVal* id, bool enabled) +bool Manager::SetBuf(EnumVal* id, bool enabled) { Stream* stream = FindStream(id); if ( ! stream ) @@ -1440,7 +1447,7 @@ bool LogMgr::SetBuf(EnumVal* id, bool enabled) return true; } -bool LogMgr::Flush(EnumVal* id) +bool Manager::Flush(EnumVal* id) { Stream* stream = FindStream(id); if ( ! stream ) @@ -1461,7 +1468,7 @@ bool LogMgr::Flush(EnumVal* id) return true; } -void LogMgr::Error(LogWriter* writer, const char* msg) +void Manager::Error(WriterFrontend* writer, const char* msg) { reporter->Error("error with writer for %s: %s", writer->Path().c_str(), msg); @@ -1470,7 +1477,7 @@ void LogMgr::Error(LogWriter* writer, const char* msg) // Timer which on dispatching rotates the filter. class RotationTimer : public Timer { public: - RotationTimer(double t, LogMgr::WriterInfo* arg_winfo, bool arg_rotate) + RotationTimer(double t, Manager::WriterInfo* arg_winfo, bool arg_rotate) : Timer(t, TIMER_ROTATE) { winfo = arg_winfo; @@ -1482,7 +1489,7 @@ public: void Dispatch(double t, int is_expire); protected: - LogMgr::WriterInfo* winfo; + Manager::WriterInfo* winfo; bool rotate; }; @@ -1506,7 +1513,7 @@ void RotationTimer::Dispatch(double t, int is_expire) } } -void LogMgr::InstallRotationTimer(WriterInfo* winfo) +void Manager::InstallRotationTimer(WriterInfo* winfo) { if ( terminating ) return; @@ -1548,7 +1555,7 @@ void LogMgr::InstallRotationTimer(WriterInfo* winfo) } } -void LogMgr::Rotate(WriterInfo* winfo) +void Manager::Rotate(WriterInfo* winfo) { DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f", winfo->writer->Path().c_str(), network_time); @@ -1568,7 +1575,7 @@ void LogMgr::Rotate(WriterInfo* winfo) winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating); } -bool LogMgr::FinishedRotation(LogWriter* writer, string new_name, string old_name, +bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name, double open, double close, bool terminating) { DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s", diff --git a/src/LogMgr.h b/src/logging/Manager.h similarity index 68% rename from src/LogMgr.h rename to src/logging/Manager.h index 3eaba360d5..7fa2c271db 100644 --- a/src/LogMgr.h +++ b/src/logging/Manager.h @@ -2,24 +2,28 @@ // // A class managing log writers and filters. -#ifndef LOGMGR_H -#define LOGMGR_H +#ifndef LOGGING_MANAGER_H +#define LOGGING_MANAGER_H -#include "Val.h" -#include "EventHandler.h" -#include "RemoteSerializer.h" +#include "../Val.h" +#include "../EventHandler.h" +#include "../RemoteSerializer.h" class SerializationFormat; +class RemoteSerializer; +class RotationTimer; + +namespace logging { // Description of a log field. -struct LogField { +struct Field { string name; TypeTag type; // inner type of sets TypeTag subtype; - LogField() { subtype = TYPE_VOID; } - LogField(const LogField& other) + Field() { subtype = TYPE_VOID; } + Field(const Field& other) : name(other.name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. @@ -28,13 +32,13 @@ struct LogField { }; // Values as logged by a writer. -struct LogVal { +struct Value { TypeTag type; bool present; // False for unset fields. // The following union is a subset of BroValUnion, including only the // types we can log directly. - struct set_t { bro_int_t size; LogVal** vals; }; + struct set_t { bro_int_t size; Value** vals; }; typedef set_t vec_t; union _val { @@ -48,9 +52,9 @@ struct LogVal { vec_t vector_val; } val; - LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) + Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) : type(arg_type), present(arg_present) {} - ~LogVal(); + ~Value(); // (Un-)serialize. bool Read(SerializationFormat* fmt); @@ -61,17 +65,17 @@ struct LogVal { static bool IsCompatibleType(BroType* t, bool atomic_only=false); private: - LogVal(const LogVal& other) { } + Value(const Value& other) { } }; -class LogWriter; -class RemoteSerializer; -class RotationTimer; +class WriterBackend; +class WriterFrontend; +class RotationFinishedMessage; -class LogMgr { +class Manager { public: - LogMgr(); - ~LogMgr(); + Manager(); + ~Manager(); // These correspond to the BiFs visible on the scripting layer. The // actual BiFs just forward here. @@ -86,19 +90,24 @@ public: bool Flush(EnumVal* id); // Flushes all writers.. protected: - friend class LogWriter; - friend class RemoteSerializer; - friend class RotationTimer; + friend class WriterFrontend; + friend class RotationFinishedMessage; + friend class ::RemoteSerializer; + friend class ::RotationTimer; + + // Instantiates a new WriterBackend of the given type (note that + // doing so creates a new thread!). + WriterBackend* CreateBackend(bro_int_t type); //// Function also used by the RemoteSerializer. // Takes ownership of fields. - LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, LogField** fields); + WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path, + int num_fields, Field** fields); // Takes ownership of values.. bool Write(EnumVal* id, EnumVal* writer, string path, - int num_fields, LogVal** vals); + int num_fields, Value** vals); // Announces all instantiated writers to peer. void SendAllWritersTo(RemoteSerializer::PeerID peer); @@ -106,14 +115,14 @@ protected: //// Functions safe to use by writers. // Signals that a file has been rotated. - bool FinishedRotation(LogWriter* writer, string new_name, string old_name, + bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, double open, double close, bool terminating); // Reports an error for the given writer. - void Error(LogWriter* writer, const char* msg); + void Error(WriterFrontend* writer, const char* msg); // Deletes the values as passed into Write(). - void DeleteVals(int num_fields, LogVal** vals); + void DeleteVals(int num_fields, Value** vals); private: struct Filter; @@ -123,20 +132,22 @@ private: bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); - LogVal** RecordToFilterVals(Stream* stream, Filter* filter, + Value** RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns); - LogVal* ValToLogVal(Val* val, BroType* ty = 0); + Value* ValToLogVal(Val* val, BroType* ty = 0); Stream* FindStream(EnumVal* id); void RemoveDisabledWriters(Stream* stream); void InstallRotationTimer(WriterInfo* winfo); void Rotate(WriterInfo* info); Filter* FindFilter(EnumVal* id, StringVal* filter); - WriterInfo* FindWriter(LogWriter* writer); + WriterInfo* FindWriter(WriterFrontend* writer); vector streams; // Indexed by stream enum. }; -extern LogMgr* log_mgr; +} + +extern logging::Manager* log_mgr; #endif diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc new file mode 100644 index 0000000000..095490edc4 --- /dev/null +++ b/src/logging/WriterBackend.cc @@ -0,0 +1,161 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "util.h" + +#include "WriterBackend.h" +#include "WriterFrontend.h" + +// Messages sent from backend to frontend (i.e., "OutputMessages"). + +namespace logging { + +class RotationFinishedMessage : public threading::OutputMessage +{ +public: + RotationFinishedMessage(WriterFrontend* writer, string new_name, string old_name, + double open, double close, bool terminating) + : threading::OutputMessage("RotationFinished", writer), + new_name(new_name), old_name(old_name), open(open), + close(close), terminating(terminating) { } + + virtual bool Process() + { + return log_mgr->FinishedRotation(Object(), new_name, old_name, open, close, terminating); + } + +private: + string new_name; + string old_name; + double open; + double close; + bool terminating; +}; + +class DisableMessage : public threading::OutputMessage +{ +public: + DisableMessage(WriterFrontend* writer) + : threading::OutputMessage("Disable", writer) {} + + virtual bool Process() { Object()->SetDisable(); return true; } +}; + +} + +// Backend methods. + +using namespace logging; + +WriterBackend::WriterBackend(const string& name) : MsgThread(name) + { + path = ""; + num_fields = 0; + fields = 0; + buffering = true; + } + +WriterBackend::~WriterBackend() + { + if ( fields ) + { + for(int i = 0; i < num_fields; ++i) + delete fields[i]; + + delete [] fields; + } + } + +void WriterBackend::DeleteVals(Value** vals) + { + // Note this code is duplicated in Manager::DeleteVals(). + for ( int i = 0; i < num_fields; i++ ) + delete vals[i]; + + delete [] vals; + } + +bool WriterBackend::FinishedRotation(WriterFrontend* writer, string new_name, string old_name, + double open, double close, bool terminating) + { + SendOut(new RotationFinishedMessage(writer, new_name, old_name, open, close, terminating)); + return true; + } + +bool WriterBackend::Init(string arg_path, int arg_num_fields, + const Field* const * arg_fields) + { + path = arg_path; + num_fields = arg_num_fields; + fields = arg_fields; + + if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) + return false; + + return true; + } + +bool WriterBackend::Write(int arg_num_fields, Value** vals) + { + // Double-check that the arguments match. If we get this from remote, + // something might be mixed up. + if ( num_fields != arg_num_fields ) + { + +#ifdef DEBUG + const char* msg = Fmt("Number of fields don't match in WriterBackend::Write() (%d vs. %d)", + arg_num_fields, num_fields); + Debug(DBG_LOGGING, msg); +#endif + + DeleteVals(vals); + return false; + } + + for ( int i = 0; i < num_fields; ++i ) + { + if ( vals[i]->type != fields[i]->type ) + { +#ifdef DEBUG + const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)", + vals[i]->type, fields[i]->type); + Debug(DBG_LOGGING, msg); +#endif + + DeleteVals(vals); + return false; + } + } + + bool result = DoWrite(num_fields, fields, vals); + + DeleteVals(vals); + + return result; + } + +bool WriterBackend::SetBuf(bool enabled) + { + if ( enabled == buffering ) + // No change. + return true; + + buffering = enabled; + + return DoSetBuf(enabled); + } + +bool WriterBackend::Rotate(WriterFrontend* writer, string rotated_path, + double open, double close, bool terminating) + { + return DoRotate(writer, rotated_path, open, close, terminating); + } + +bool WriterBackend::Flush() + { + return DoFlush(); + } + +bool WriterBackend::Finish() + { + return DoFinish(); + } diff --git a/src/LogWriter.h b/src/logging/WriterBackend.h similarity index 70% rename from src/LogWriter.h rename to src/logging/WriterBackend.h index 1d2f9fa4b2..d1e4634e6d 100644 --- a/src/LogWriter.h +++ b/src/logging/WriterBackend.h @@ -1,32 +1,22 @@ // See the file "COPYING" in the main distribution directory for copyright. // -// Interface API for a log writer backend. The LogMgr creates a separate -// writer instance of pair of (writer type, output path). -// -// Note thay classes derived from LogWriter must be fully thread-safe and not -// use any non-thread-safe Bro functionality (which includes almost -// everything ...). In particular, do not use fmt() but LogWriter::Fmt()!. -// -// The one exception to this rule is the constructor: it is guaranteed to be -// executed inside the main thread and can thus in particular access global -// script variables. +// Bridge class between main process and writer threads. -#ifndef LOGWRITER_H -#define LOGWRITER_H +#ifndef LOGGING_WRITERBACKEND_H +#define LOGGING_WRITERBACKEND_H -#include "LogMgr.h" -#include "BroString.h" +#include "Manager.h" -class LogWriter { +#include "threading/MsgThread.h" + +namespace logging { + +// The backend runs in its own thread, separate from the main process. +class WriterBackend : public threading::MsgThread +{ public: - LogWriter(); - virtual ~LogWriter(); - - //// Interface methods to interact with the writer. Note that these - //// methods are not necessarily thread-safe and must be called only - //// from the main thread (which will typically mean only from the - //// LogMgr). In particular, they must not be called from the - //// writer's derived implementation. + WriterBackend(const string& name); + virtual ~WriterBackend(); // One-time initialization of the writer to define the logged fields. // Interpretation of "path" is left to the writer, and will be @@ -37,18 +27,18 @@ public: // // The new instance takes ownership of "fields", and will delete them // when done. - bool Init(string path, int num_fields, const LogField* const * fields); + bool Init(string path, int num_fields, const Field* const * fields); // Writes one log entry. The method takes ownership of "vals" and // will return immediately after queueing the write request, which is // potentially before output has actually been written out. // - // num_fields and the types of the LogVals must match what was passed + // num_fields and the types of the Values must match what was passed // to Init(). // // Returns false if an error occured, in which case the writer must // not be used any further. - bool Write(int num_fields, LogVal** vals); + bool Write(int num_fields, Value** vals); // Sets the buffering status for the writer, if the writer supports // that. (If not, it will be ignored). @@ -60,12 +50,12 @@ public: // Triggers rotation, if the writer supports that. (If not, it will // be ignored). - bool Rotate(string rotated_path, double open, double close, bool terminating); + bool Rotate(WriterFrontend* writer, string rotated_path, double open, double close, bool terminating); // Finishes writing to this logger regularly. Must not be called if // an error has been indicated earlier. After calling this, no // further writing must be performed. - void Finish(); + bool Finish(); //// Thread-safe methods that may be called from the writer //// implementation. @@ -73,24 +63,43 @@ public: // The following methods return the information as passed to Init(). const string Path() const { return path; } int NumFields() const { return num_fields; } - const LogField* const * Fields() const { return fields; } + const Field* const * Fields() const { return fields; } + + // Returns the current buffering state. + bool IsBuf() { return buffering; } + + // Signals to the log manager that a file has been rotated. + // + // writer: The frontend writer that triggered the rotation. This must + // be the value passed into DoRotate(). + // + // new_name: The filename of the rotated file. old_name: The filename + // of the origina file. + // + // open/close: The timestamps when the original file was opened and + // closed, respectively. + // + // terminating: True if rotation request occured due to the main Bro + // process shutting down. + bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, + double open, double close, bool terminating); protected: // Methods for writers to override. If any of these returs false, it // will be assumed that a fatal error has occured that prevents the // writer from further operation. It will then be disabled and - // deleted. When return false, the writer should also report the + // deleted. When returning false, the writer should also report the // error via Error(). Note that even if a writer does not support the // functionality for one these methods (like rotation), it must still // return true if that is not to be considered a fatal error. // // Called once for initialization of the writer. virtual bool DoInit(string path, int num_fields, - const LogField* const * fields) = 0; + const Field* const * fields) = 0; // Called once per log entry to record. - virtual bool DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals) = 0; + virtual bool DoWrite(int num_fields, const Field* const * fields, + Value** vals) = 0; // Called when the buffering status for this writer is changed. If // buffering is disabled, the writer should attempt to write out @@ -119,6 +128,11 @@ protected: // RotationDone() to signal the log manager that potential // postprocessors can now run. // + // "writer" is the frontend writer that triggered the rotation. The + // *only* purpose of this value is to be passed into + // FinishedRotation() once done. You must not otherwise access the + // frontend, it's running in a different thread. + // // "rotate_path" reflects the path to where the rotated output is to // be moved, with specifics depending on the writer. It should // generally be interpreted in a way consistent with that of "path" @@ -135,52 +149,31 @@ protected: // // A writer may ignore rotation requests if it doesn't fit with its // semantics (but must still return true in that case). - virtual bool DoRotate(string rotated_path, double open, double close, - bool terminating) = 0; + virtual bool DoRotate(WriterFrontend* writer, string rotated_path, + double open, double close, bool terminating) = 0; // Called once on termination. Not called when any of the other // methods has previously signaled an error, i.e., executing this // method signals a regular shutdown of the writer. - virtual void DoFinish() = 0; + virtual bool DoFinish() = 0; - //// Methods for writers to use. These are thread-safe. - - // A thread-safe version of fmt(). - const char* Fmt(const char* format, ...); - - // Returns the current buffering state. - bool IsBuf() { return buffering; } - - // Reports an error to the user. - void Error(const char *msg); - - // Signals to the log manager that a file has been rotated. - // - // new_name: The filename of the rotated file. old_name: The filename - // of the origina file. - // - // open/close: The timestamps when the original file was opened and - // closed, respectively. - // - // terminating: True if rotation request occured due to the main Bro - // process shutting down. - bool FinishedRotation(string new_name, string old_name, double open, - double close, bool terminating); + // Triggered by regular heartbeat messages from the main process. + virtual bool DoHeartbeat(double network_time, double current_time) { return true; }; private: - friend class LogMgr; + friend class Manager; // When an error occurs, we call this method to set a flag marking - // the writer as disabled. The LogMgr will check the flag later and + // the writer as disabled. The Manager will check the flag later and // remove the writer. bool Disabled() { return disabled; } // Deletes the values as passed into Write(). - void DeleteVals(LogVal** vals); + void DeleteVals(Value** vals); string path; int num_fields; - const LogField* const * fields; + const Field* const * fields; bool buffering; bool disabled; @@ -189,4 +182,8 @@ private: unsigned int buf_len; }; + +} + #endif + diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc new file mode 100644 index 0000000000..92c93c1c56 --- /dev/null +++ b/src/logging/WriterFrontend.cc @@ -0,0 +1,175 @@ + +#include "WriterFrontend.h" +#include "WriterBackend.h" + +namespace logging { + +// Messages sent from frontend to backend (i.e., "InputMessages"). + +class InitMessage : public threading::InputMessage +{ +public: + InitMessage(WriterBackend* backend, const string path, const int num_fields, const Field* const *fields) + : threading::InputMessage("Init", backend), + path(path), num_fields(num_fields), fields(fields) { } + + virtual bool Process() { return Object()->Init(path, num_fields, fields); } + +private: + const string path; + const int num_fields; + const Field * const* fields; +}; + +class RotateMessage : public threading::InputMessage +{ +public: + RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const string rotated_path, const double open, + const double close, const bool terminating) + : threading::InputMessage("Rotate", backend), + frontend(frontend), + rotated_path(rotated_path), open(open), + close(close), terminating(terminating) { } + + virtual bool Process() { return Object()->Rotate(frontend, rotated_path, open, close, terminating); } + +private: + WriterFrontend* frontend; + const string rotated_path; + const double open; + const double close; + const bool terminating; +}; + +class WriteMessage : public threading::InputMessage +{ +public: + WriteMessage(WriterBackend* backend, const int num_fields, Value **vals) + : threading::InputMessage("Write", backend), + num_fields(num_fields), fields(fields), vals(vals) {} + + virtual bool Process() { return Object()->Write(num_fields, vals); } + +private: + int num_fields; + Field* const* fields; + Value **vals; +}; + +class SetBufMessage : public threading::InputMessage +{ +public: + SetBufMessage(WriterBackend* backend, const bool enabled) + : threading::InputMessage("SetBuf", backend), + enabled(enabled) { } + + virtual bool Process() { return Object()->SetBuf(enabled); } + +private: + const bool enabled; +}; + +class FlushMessage : public threading::InputMessage +{ +public: + FlushMessage(WriterBackend* backend) + : threading::InputMessage("Flush", backend) {} + + virtual bool Process() { return Object()->Flush(); } +}; + +class FinishMessage : public threading::InputMessage +{ +public: + FinishMessage(WriterBackend* backend) + : threading::InputMessage("Finish", backend) {} + + virtual bool Process() { return Object()->Finish(); } +}; + +} + +// Frontend methods. + +using namespace logging; + +WriterFrontend::WriterFrontend(bro_int_t type) + { + disabled = initialized = false; + backend = log_mgr->CreateBackend(type); + + assert(backend); + backend->Start(); + } + +WriterFrontend::~WriterFrontend() + { + } + +void WriterFrontend::Stop() + { + SetDisable(); + backend->Stop(); + } + +void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* const * arg_fields) + { + if ( disabled ) + return; + + if ( initialized ) + reporter->InternalError("writer initialize twice"); + + path = arg_path; + num_fields = arg_num_fields; + fields = arg_fields; + + initialized = true; + backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + } + +void WriterFrontend::Write(int num_fields, Value** vals) + { + if ( disabled ) + return; + + backend->SendIn(new WriteMessage(backend, num_fields, vals)); + } + +void WriterFrontend::SetBuf(bool enabled) + { + if ( disabled ) + return; + + backend->SendIn(new SetBufMessage(backend, enabled)); + } + +void WriterFrontend::Flush() + { + if ( disabled ) + return; + + backend->SendIn(new FlushMessage(backend)); + } + +void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating) + { + if ( disabled ) + return; + + backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); + } + +void WriterFrontend::Finish() + { + if ( disabled ) + return; + + backend->SendIn(new FinishMessage(backend)); + } + + + + + + diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h new file mode 100644 index 0000000000..1998429d38 --- /dev/null +++ b/src/logging/WriterFrontend.h @@ -0,0 +1,66 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Bridge class between main process and writer threads. + +#ifndef LOGGING_WRITERFRONTEND_H +#define LOGGING_WRITERFRONTEND_H + +#include "Manager.h" + +#include "threading/MsgThread.h" + +namespace logging { + +class WriterBackend; + +class WriterFrontend { +public: + WriterFrontend(bro_int_t type); + virtual ~WriterFrontend(); + + // Disables the writers and stop the backend thread. + void Stop(); + + // Interface methods to interact with the writer from the main thread + // (and only from the main thread), typicalli from the log manager. + // All these methods forward (via inter-thread messaging) to the + // corresponding methods of an internally created WriterBackend. See + // there for documentation. + // + // If any of these operations fails, the writer will be automatically + // (but asynchronoulsy) disabled. + + void Init(string path, int num_fields, const Field* const * fields); + void Write(int num_fields, Value** vals); + void SetBuf(bool enabled); + void Flush(); + void Rotate(string rotated_path, double open, double close, bool terminating); + void Finish(); + + // Calling this disable the writer. All methods calls will be no-ops + // from now on. The Manager will eventually remove disabled writers. + void SetDisable() { disabled = true; } + bool Disabled() { return disabled; } + + const string Path() const { return path; } + int NumFields() const { return num_fields; } + const Field* const * Fields() const { return fields; } + +protected: + friend class Manager; + + + WriterBackend* backend; + bool disabled; + bool initialized; + + string path; + int num_fields; + const Field* const * fields; +}; + +} + + + +#endif diff --git a/src/LogWriterAscii.cc b/src/logging/writers/Ascii.cc similarity index 89% rename from src/LogWriterAscii.cc rename to src/logging/writers/Ascii.cc index d2c1d91370..70f513be3b 100644 --- a/src/LogWriterAscii.cc +++ b/src/logging/writers/Ascii.cc @@ -3,10 +3,14 @@ #include #include -#include "LogWriterAscii.h" -#include "NetVar.h" +#include "../../NetVar.h" -LogWriterAscii::LogWriterAscii() +#include "Ascii.h" + +using namespace logging; +using namespace writer; + +Ascii::Ascii() : WriterBackend("Ascii") { file = 0; @@ -42,7 +46,7 @@ LogWriterAscii::LogWriterAscii() desc.AddEscapeSequence(separator, separator_len); } -LogWriterAscii::~LogWriterAscii() +Ascii::~Ascii() { if ( file ) fclose(file); @@ -54,7 +58,7 @@ LogWriterAscii::~LogWriterAscii() delete [] header_prefix; } -bool LogWriterAscii::WriteHeaderField(const string& key, const string& val) +bool Ascii::WriteHeaderField(const string& key, const string& val) { string str = string(header_prefix, header_prefix_len) + key + string(separator, separator_len) + val + "\n"; @@ -62,8 +66,8 @@ bool LogWriterAscii::WriteHeaderField(const string& key, const string& val) return (fwrite(str.c_str(), str.length(), 1, file) == 1); } -bool LogWriterAscii::DoInit(string path, int num_fields, - const LogField* const * fields) +bool Ascii::DoInit(string path, int num_fields, + const Field* const * fields) { if ( output_to_stdout ) path = "/dev/stdout"; @@ -108,7 +112,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, types += string(separator, separator_len); } - const LogField* field = fields[i]; + const Field* field = fields[i]; names += field->name; types += type_name(field->type); if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) ) @@ -131,17 +135,18 @@ write_error: return false; } -bool LogWriterAscii::DoFlush() +bool Ascii::DoFlush() { fflush(file); return true; } -void LogWriterAscii::DoFinish() +bool Ascii::DoFinish() { + return true; } -bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) +bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) { if ( ! val->present ) { @@ -281,8 +286,8 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) return true; } -bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals) +bool Ascii::DoWrite(int num_fields, const Field* const * fields, + Value** vals) { if ( ! file ) DoInit(Path(), NumFields(), Fields()); @@ -312,8 +317,8 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, return true; } -bool LogWriterAscii::DoRotate(string rotated_path, double open, - double close, bool terminating) +bool Ascii::DoRotate(WriterFrontend* writer, string rotated_path, double open, + double close, bool terminating) { // Don't rotate special files or if there's not one currently open. if ( ! file || IsSpecial(Path()) ) @@ -325,7 +330,7 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open, string nname = rotated_path + "." + LogExt(); rename(fname.c_str(), nname.c_str()); - if ( ! FinishedRotation(nname, fname, open, close, terminating) ) + if ( ! FinishedRotation(writer, nname, fname, open, close, terminating) ) { Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str())); return false; @@ -334,13 +339,13 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open, return true; } -bool LogWriterAscii::DoSetBuf(bool enabled) +bool Ascii::DoSetBuf(bool enabled) { // Nothing to do. return true; } -string LogWriterAscii::LogExt() +string Ascii::LogExt() { const char* ext = getenv("BRO_LOG_SUFFIX"); if ( ! ext ) ext = "log"; diff --git a/src/LogWriterAscii.h b/src/logging/writers/Ascii.h similarity index 57% rename from src/LogWriterAscii.h rename to src/logging/writers/Ascii.h index 72127c8b1f..37fcfef267 100644 --- a/src/LogWriterAscii.h +++ b/src/logging/writers/Ascii.h @@ -2,33 +2,35 @@ // // Log writer for delimiter-separated ASCII logs. -#ifndef LOGWRITERASCII_H -#define LOGWRITERASCII_H +#ifndef LOGGING_WRITER_ASCII_H +#define LOGGING_WRITER_ASCII_H -#include "LogWriter.h" +#include "../WriterBackend.h" -class LogWriterAscii : public LogWriter { +namespace logging { namespace writer { + +class Ascii : public WriterBackend { public: - LogWriterAscii(); - ~LogWriterAscii(); + Ascii(); + ~Ascii(); - static LogWriter* Instantiate() { return new LogWriterAscii; } + static WriterBackend* Instantiate() { return new Ascii; } static string LogExt(); protected: virtual bool DoInit(string path, int num_fields, - const LogField* const * fields); - virtual bool DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals); + const Field* const * fields); + virtual bool DoWrite(int num_fields, const Field* const * fields, + Value** vals); virtual bool DoSetBuf(bool enabled); - virtual bool DoRotate(string rotated_path, double open, double close, - bool terminating); + virtual bool DoRotate(WriterFrontend* writer, string rotated_path, + double open, double close, bool terminating); virtual bool DoFlush(); - virtual void DoFinish(); + virtual bool DoFinish(); private: bool IsSpecial(string path) { return path.find("/dev/") == 0; } - bool DoWriteOne(ODesc* desc, LogVal* val, const LogField* field); + bool DoWriteOne(ODesc* desc, Value* val, const Field* field); bool WriteHeaderField(const string& key, const string& value); FILE* file; @@ -55,4 +57,8 @@ private: int header_prefix_len; }; +} +} + + #endif diff --git a/src/logging/writers/None.cc b/src/logging/writers/None.cc new file mode 100644 index 0000000000..e419d88a6b --- /dev/null +++ b/src/logging/writers/None.cc @@ -0,0 +1,19 @@ + +#include "None.h" + +using namespace logging; +using namespace writer; + +bool None::DoRotate(WriterFrontend* writer, string rotated_path, + double open, double close, bool terminating) + { + if ( ! FinishedRotation(writer, string("/dev/null"), Path(), open, close, terminating)) + { + Error(Fmt("error rotating %s", Path().c_str())); + return false; + } + + return true; + } + + diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h new file mode 100644 index 0000000000..9b2ab6c698 --- /dev/null +++ b/src/logging/writers/None.h @@ -0,0 +1,35 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Dummy log writer that just discards everything (but still pretends to rotate). + +#ifndef LOGGING_WRITER_NONE_H +#define LOGGING_WRITER_NONE_H + +#include "../WriterBackend.h" + +namespace logging { namespace writer { + +class None : public WriterBackend { +public: + None() : WriterBackend("None") {} + ~None() {}; + + static WriterBackend* Instantiate() { return new None; } + +protected: + virtual bool DoInit(string path, int num_fields, + const Field* const * fields) { return true; } + + virtual bool DoWrite(int num_fields, const Field* const * fields, + Value** vals) { return true; } + virtual bool DoSetBuf(bool enabled) { return true; } + virtual bool DoRotate(WriterFrontend* writer, string rotated_path, + double open, double close, bool terminating); + virtual bool DoFlush() { return true; } + virtual bool DoFinish() { return true; } +}; + +} +} + +#endif diff --git a/src/main.cc b/src/main.cc index bcc0498123..58a23e6c80 100644 --- a/src/main.cc +++ b/src/main.cc @@ -29,7 +29,6 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "Event.h" #include "File.h" #include "Reporter.h" -#include "LogMgr.h" #include "Net.h" #include "NetVar.h" #include "Var.h" @@ -48,7 +47,10 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "DPM.h" #include "BroDoc.h" #include "Brofiler.h" -#include "LogWriterAscii.h" + +#include "threading/Manager.h" +#include "logging/Manager.h" +#include "logging/writers/Ascii.h" #include "binpac_bro.h" @@ -75,7 +77,8 @@ char* writefile = 0; name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; -LogMgr* log_mgr; +logging::Manager* log_mgr = 0; +threading::Manager* thread_mgr = 0; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -197,7 +200,7 @@ void usage() fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes()); fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake()); fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); - fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", LogWriterAscii::LogExt().c_str()); + fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str()); fprintf(stderr, " $BRO_PROFILER_FILE | Output file for script execution statistics (not set)\n"); exit(1); @@ -287,6 +290,8 @@ void terminate_bro() if ( remote_serializer ) remote_serializer->LogStats(); + thread_mgr->Terminate(); + delete timer_mgr; delete dns_mgr; delete persistence_serializer; @@ -299,6 +304,7 @@ void terminate_bro() delete remote_serializer; delete dpm; delete log_mgr; + delete thread_mgr; delete reporter; } @@ -661,7 +667,9 @@ int main(int argc, char** argv) set_processing_status("INITIALIZING", "main"); bro_start_time = current_time(true); + reporter = new Reporter(); + thread_mgr = new threading::Manager(); #ifdef DEBUG if ( debug_streams ) @@ -727,7 +735,7 @@ int main(int argc, char** argv) persistence_serializer = new PersistenceSerializer(); remote_serializer = new RemoteSerializer(); event_registry = new EventRegistry(); - log_mgr = new LogMgr(); + log_mgr = new logging::Manager(); if ( events_file ) event_player = new EventPlayer(events_file); @@ -1001,6 +1009,8 @@ int main(int argc, char** argv) have_pending_timers = ! reading_traces && timer_mgr->Size() > 0; + io_sources.Register(thread_mgr, true); + if ( io_sources.Size() > 0 || have_pending_timers ) { if ( profiling_logger ) diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc new file mode 100644 index 0000000000..273a192de3 --- /dev/null +++ b/src/threading/BasicThread.cc @@ -0,0 +1,129 @@ + +#include +#include + +#include "BasicThread.h" +#include "Manager.h" + +using namespace threading; + +BasicThread::BasicThread(const string& arg_name) + { + started = false; + terminating = false; + pthread = 0; + + buf = 0; + buf_len = 1024; + + char tmp[128]; + snprintf(tmp, sizeof(tmp), "%s@%p", arg_name.c_str(), this); + name = string(tmp); + + thread_mgr->AddThread(this); + } + +BasicThread::~BasicThread() + { + } + +const char* BasicThread::Fmt(const char* format, ...) + { + if ( ! buf ) + buf = (char*) malloc(buf_len); + + va_list al; + va_start(al, format); + int n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + + if ( (unsigned int) n >= buf_len ) + { // Not enough room, grow the buffer. + buf_len = n + 32; + buf = (char*) realloc(buf, buf_len); + + // Is it portable to restart? + va_start(al, format); + n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + } + + return buf; + } + +void BasicThread::Start() + { + if ( sem_init(&terminate, 0, 0) != 0 ) + reporter->FatalError("Cannot create terminate semaphore for thread %s", name.c_str()); + + if ( pthread_create(&pthread, 0, BasicThread::launcher, this) != 0 ) + reporter->FatalError("Cannot create thread %s", name.c_str()); + + DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str()); + + started = true; + + OnStart(); + } + +void BasicThread::Stop() + { + if ( ! started ) + return; + + if ( terminating ) + return; + + DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str()); + + // Signal that it's ok for the thread to exit now. + if ( sem_post(&terminate) != 0 ) + reporter->FatalError("Failure flagging terminate condition for thread %s", name.c_str()); + + terminating = true; + + OnStop(); + } + +void BasicThread::Join() + { + if ( ! started ) + return; + + if ( ! terminating ) + Stop(); + + DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str()); + + if ( pthread_join(pthread, 0) != 0 ) + reporter->FatalError("Failure joining thread %s", name.c_str()); + + sem_destroy(&terminate); + + DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str()); + + pthread = 0; + } + +void* BasicThread::launcher(void *arg) + { + BasicThread* thread = (BasicThread *)arg; + + // Block signals in thread. We handle signals only in the main + // process. + sigset_t mask_set; + sigfillset(&mask_set); + int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0); + assert(res == 0); // + + // Run thread's main function. + thread->Run(); + + // Wait until somebody actually wants us to terminate. + + if ( sem_wait(&thread->terminate) != 0 ) + reporter->FatalError("Failure flagging terminate condition for thread %s", thread->Name().c_str()); + + return 0; + } + diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h new file mode 100644 index 0000000000..30a11b4505 --- /dev/null +++ b/src/threading/BasicThread.h @@ -0,0 +1,63 @@ + +#ifndef THREADING_BASICTHREAD_H +#define THREADING_BASICTHREAD_H + +#include +#include + +#include "Queue.h" +#include "util.h" + +using namespace std; + +namespace threading { + +class Manager; + +class BasicThread +{ +public: + BasicThread(const string& name); // Managed by manager, must not delete otherwise. + virtual ~BasicThread(); + + const string& Name() const { return name; } + + void Start(); // Spawns the thread and enters Run(). + void Stop(); // Signals the thread to terminate. + + bool Terminating() const { return terminating; } + + // A thread-safe version of fmt(). + const char* Fmt(const char* format, ...); + +protected: + virtual void Run() = 0; + + virtual void OnStart() {} + virtual void OnStop() {} + +private: + friend class Manager; + + static void* launcher(void *arg); + + // Used from the ThreadMgr. + void Join(); // Waits until the thread has terminated and then joins it. + + bool started; // Set to to true once running. + bool terminating; // Set to to true to signal termination. + string name; + + pthread_t pthread; + sem_t terminate; + + // For implementing Fmt(). + char* buf; + unsigned int buf_len; +}; + +} + +extern threading::Manager* thread_mgr; + +#endif diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc new file mode 100644 index 0000000000..ed4d9cf623 --- /dev/null +++ b/src/threading/Manager.cc @@ -0,0 +1,104 @@ + +#include "Manager.h" + +using namespace threading; + +Manager::Manager() + { + DBG_LOG(DBG_THREADING, "Creating thread manager ..."); + + did_process = false; + next_beat = 0; + } + +Manager::~Manager() + { + if ( all_threads.size() ) + Terminate(); + } + +void Manager::Terminate() + { + DBG_LOG(DBG_THREADING, "Terminating thread manager ..."); + + // First process remaining thread output for the message threads. + do Process(); while ( did_process ); + + // Signal all to stop. + for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ ) + (*i)->Stop(); + + // Then join them all. + for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ ) + { + (*i)->Join(); + delete *i; + } + + all_threads.clear(); + msg_threads.clear(); + } + +void Manager::AddThread(BasicThread* thread) + { + DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str()); + all_threads.push_back(thread); + } + +void Manager::AddMsgThread(MsgThread* thread) + { + DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name().c_str()); + msg_threads.push_back(thread); + } + +void Manager::GetFds(int* read, int* write, int* except) + { + } + +double Manager::NextTimestamp(double* network_time) + { + if ( did_process || ! next_beat == 0 ) + // If we had something to process last time (or haven't had a + // chance to check yet), we want to check for more asap. + return timer_mgr->Time(); + + // Else we assume we don't have much to do at all and wait for the next heart beat. + return next_beat; + } + +void Manager::Process() + { + bool do_beat = (next_beat == 0 || network_time >= next_beat); + + did_process = false; + + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + { + MsgThread* t = *i; + + if ( do_beat ) + t->Heartbeat(); + + if ( ! t->HasOut() ) + continue; + + Message* msg = t->RetrieveOut(); + + if ( msg->Process() ) + did_process = true; + + else + { + string s = msg->Name() + " failed, terminating thread"; + reporter->Error(s.c_str()); + t->Stop(); + } + + delete msg; + } + + if ( do_beat ) + next_beat = network_time + HEART_BEAT_INTERVAL; + } + + diff --git a/src/threading/Manager.h b/src/threading/Manager.h new file mode 100644 index 0000000000..aa7292ee81 --- /dev/null +++ b/src/threading/Manager.h @@ -0,0 +1,52 @@ + +#ifndef THREADING_MANAGER_H +#define THREADING_MANAGER_H + +#include + +#include "IOSource.h" + +#include "BasicThread.h" +#include "MsgThread.h" + +namespace threading { + +class Manager : public IOSource +{ +public: + Manager(); + ~Manager(); + + void Terminate(); + +protected: + friend class BasicThread; + friend class MsgThread; + + void AddThread(BasicThread* thread); + void AddMsgThread(MsgThread* thread); + + // IOSource interface. + virtual void GetFds(int* read, int* write, int* except); + virtual double NextTimestamp(double* network_time); + virtual void Process(); + virtual const char* Tag() { return "threading::Manager"; } + +private: + static const int HEART_BEAT_INTERVAL = 1; + + typedef std::list all_thread_list; + all_thread_list all_threads; + + typedef std::list msg_thread_list; + msg_thread_list msg_threads; + + bool did_process; + double next_beat; +}; + +} + +extern threading::Manager* thread_mgr; + +#endif diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc new file mode 100644 index 0000000000..e2d81cf47f --- /dev/null +++ b/src/threading/MsgThread.cc @@ -0,0 +1,285 @@ + +#include "DebugLogger.h" + +#include "MsgThread.h" +#include "Manager.h" + +using namespace threading; + +static void strreplace(const string& s, const string& o, const string& n) + { + string r = s; + + while ( true ) + { + size_t i = r.find(o); + + if ( i == std::string::npos ) + break; + + r.replace(i, o.size(), n); + } + } + +namespace threading { + +// Standard messages. + +class TerminateMessage : public InputMessage +{ +public: + TerminateMessage(MsgThread* thread) : InputMessage("Terminate", thread) { } + + virtual bool Process() { return true; } +}; + +class ReporterMessage : public OutputMessage +{ +public: + enum Type { + INFO, WARNING, ERROR, FATAL_ERROR, FATAL_ERROR_WITH_CORE, + INTERNAL_WARNING, INTERNAL_ERROR + }; + + ReporterMessage(Type arg_type, MsgThread* thread, const string& arg_msg) + : OutputMessage("ReporterMessage", thread) + { type = arg_type; msg = arg_msg; } + + virtual bool Process(); + +private: + string msg; + Type type; +}; + +class HeartbeatMessage : public InputMessage +{ +public: + HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time) + : InputMessage("Heartbeat", thread) + { network_time = arg_network_time; current_time = arg_current_time; } + + virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); } + +private: + double network_time; + double current_time; +}; + +#ifdef DEBUG +class DebugMessage : public OutputMessage +{ +public: + DebugMessage(DebugStream arg_stream, MsgThread* thread, const string& arg_msg) + : OutputMessage("DebugMessage", thread) + { stream = arg_stream; msg = arg_msg; } + + virtual bool Process() + { + string s = Object()->Name() + ": " + msg; + strreplace(s, "%", "%%"); + debug_logger.Log(stream, s.c_str()); + return true; + } +private: + string msg; + DebugStream stream; +}; +#endif + +} + +// Methods. + +Message::~Message() + { + } + +bool ReporterMessage::Process() + { + string s = Object()->Name() + ": " + msg; + strreplace(s, "%", "%%"); + + const char* cmsg = s.c_str(); + + switch ( type ) { + + case INFO: + reporter->Info(cmsg); + break; + + case WARNING: + reporter->Warning(cmsg); + break; + + case ERROR: + reporter->Error(cmsg); + break; + + case FATAL_ERROR: + reporter->FatalError(cmsg); + break; + + case FATAL_ERROR_WITH_CORE: + reporter->FatalErrorWithCore(cmsg); + break; + + case INTERNAL_WARNING: + reporter->InternalWarning(cmsg); + break; + + case INTERNAL_ERROR : + reporter->InternalError(cmsg); + break; + + default: + reporter->InternalError("unknown ReporterMessage type %d", type); + } + + return true; + } + +MsgThread::MsgThread(const string& name) : BasicThread(name) + { + cnt_sent_in = cnt_sent_out = 0; + thread_mgr->AddMsgThread(this); + } + +void MsgThread::OnStop() + { + // This is to unblock the current queue read operation. + SendIn(new TerminateMessage(this), true); + } + +void MsgThread::Heartbeat() + { + SendIn(new HeartbeatMessage(this, network_time, current_time())); + } + +void MsgThread::Info(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::INFO, this, msg)); + } + +void MsgThread::Warning(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::WARNING, this, msg)); + } + +void MsgThread::Error(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::ERROR, this, msg)); + } + +void MsgThread::FatalError(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::FATAL_ERROR, this, msg)); + } + +void MsgThread::FatalErrorWithCore(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::FATAL_ERROR_WITH_CORE, this, msg)); + } + +void MsgThread::InternalWarning(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::INTERNAL_WARNING, this, msg)); + } + +void MsgThread::InternalError(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::INTERNAL_ERROR, this, msg)); + } + +#ifdef DEBUG + +void MsgThread::Debug(DebugStream stream, const char* msg) + { + SendOut(new DebugMessage(stream, this, msg)); + } + +#endif + +void MsgThread::SendIn(BasicInputMessage* msg, bool force) + { + if ( Terminating() && ! force ) + return; + + DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str()); + + queue_in.Put(msg); + ++cnt_sent_in; + } + + +void MsgThread::SendOut(BasicOutputMessage* msg, bool force) + { + if ( Terminating() && ! force ) + return; + + queue_out.Put(msg); + ++cnt_sent_out; + } + +BasicOutputMessage* MsgThread::RetrieveOut() + { + BasicOutputMessage* msg = queue_out.Get(); + assert(msg); + +#ifdef DEBUG + if ( msg->Name() != "DebugMessage" ) // Avoid recursion. + { + string s = Fmt("Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str()); + Debug(DBG_THREADING, s.c_str()); + } +#endif + + return msg; + } + +BasicInputMessage* MsgThread::RetrieveIn() + { + BasicInputMessage* msg = queue_in.Get(); + assert(msg); + +#ifdef DEBUG + string s = Fmt("Retrieved '%s' in %s", msg->Name().c_str(), Name().c_str()); + Debug(DBG_THREADING, s.c_str()); +#endif + + return msg; + } + +void MsgThread::Run() + { + while ( true ) + { + // When requested to terminate, we only do so when + // all input has been processed. + if ( Terminating() && ! queue_in.Ready() ) + break; + + BasicInputMessage* msg = RetrieveIn(); + + bool result = msg->Process(); + + if ( ! result ) + { + string s = msg->Name() + " failed, terminating thread"; + Error(s.c_str()); + Stop(); + break; + } + + delete msg; + } + } + +void MsgThread::GetStats(Stats* stats) + { + stats->sent_in = cnt_sent_in; + stats->sent_out = cnt_sent_out; + stats->pending_in = cnt_sent_in - queue_in.Size(); + stats->pending_out = cnt_sent_out - queue_out.Size(); + } + diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h new file mode 100644 index 0000000000..2e976c1773 --- /dev/null +++ b/src/threading/MsgThread.h @@ -0,0 +1,157 @@ + +#ifndef THREADING_MSGTHREAD_H +#define THREADING_MSGTHREAD_H + +#include + +#include "DebugLogger.h" + +#include "BasicThread.h" +#include "Queue.h" + +namespace threading { + +class BasicInputMessage; +class BasicOutputMessage; +class HeartbeatMessage; + +class MsgThread : public BasicThread +{ +public: + MsgThread(const string& name); + + void SendIn(BasicInputMessage* msg) { return SendIn(msg, false); } + void SendOut(BasicOutputMessage* msg) { return SendOut(msg, false); } + + BasicOutputMessage* RetrieveOut(); + + // Report an informational message, nothing that needs specific + // attention. + void Info(const char* msg); + + // Report a warning that may indicate a problem. + void Warning(const char* msg); + + // Report a non-fatal error. Processing proceeds normally after the error + // has been reported. + void Error(const char* msg); + + // Report a fatal error. Bro will terminate after the message has been + // reported. + void FatalError(const char* msg); + + // Report a fatal error. Bro will terminate after the message has been + // reported and always generate a core dump. + void FatalErrorWithCore(const char* msg); + + // Report about a potential internal problem. Bro will continue + // normally. + void InternalWarning(const char* msg); + + // Report an internal program error. Bro will terminate with a core + // dump after the message has been reported. + void InternalError(const char* msg); + +#ifdef DEBUG + // Records a debug message for the given stream. + void Debug(DebugStream stream, const char* msg); +#endif + + void Heartbeat(); + + struct Stats + { + uint64_t sent_in; + uint64_t sent_out; + uint64_t pending_in; + uint64_t pending_out; + }; + + void GetStats(Stats* stats); + +protected: + friend class HeartbeatMessage; + + virtual void Run(); + virtual void OnStop(); + + virtual bool DoHeartbeat(double network_time, double current_time) { return true; } + +private: + friend class Manager; + + BasicInputMessage* RetrieveIn(); + + void SendIn(BasicInputMessage* msg, bool force); + void SendOut(BasicOutputMessage* msg, bool force); + + bool HasIn() { return queue_in.Ready(); } + bool HasOut() { return queue_out.Ready(); } + + Queue_ queue_in; + Queue_ queue_out; + + uint64_t cnt_sent_in; + uint64_t cnt_sent_out; +}; + +class Message +{ +public: + virtual ~Message(); + + const string& Name() const { return name; } + + virtual bool Process() = 0; // Thread will be terminated if returngin false. + +protected: + Message(const string& arg_name) { name = arg_name; } + +private: + string name; +}; + +class BasicInputMessage : public Message +{ +protected: + BasicInputMessage(const string& name) : Message(name) {} +}; + +class BasicOutputMessage : public Message +{ +protected: + BasicOutputMessage(const string& name) : Message(name) {} +}; + +template +class InputMessage : public BasicInputMessage +{ +public: + O* Object() const { return object; } + +protected: + InputMessage(const string& name, O* arg_object) : BasicInputMessage(name) + { object = arg_object; } + +private: + O* object; +}; + +template +class OutputMessage : public BasicOutputMessage +{ +public: + O* Object() const { return object; } + +protected: + OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name) + { object = arg_object; } + +private: + O* object; +}; + +} + + +#endif diff --git a/src/threading/Queue.h b/src/threading/Queue.h new file mode 100644 index 0000000000..49859dc051 --- /dev/null +++ b/src/threading/Queue.h @@ -0,0 +1,150 @@ + +#ifndef THREADING_QUEUE_H +#define THREADING_QUEUE_H + +#include +#include +#include +#include + +#include "Reporter.h" + +namespace threading { + +/** + * Just a simple threaded queue wrapper class. Uses multiple queues and reads / writes in rotary fashion in an attempt to limit contention. + * Due to locking granularity, bulk put / get is no faster than single put / get as long as FIFO guarantee is required. + */ + +template +class Queue_ +{ +public: + Queue_(); + ~Queue_(); + + T Get(); + void Put(T data); + bool Ready(); + uint64_t Size(); + +private: + static const int NUM_QUEUES = 8; + + pthread_mutex_t mutex[NUM_QUEUES]; // Mutex protected shared accesses. + pthread_cond_t has_data[NUM_QUEUES]; // Signals when data becomes available + std::queue messages[NUM_QUEUES]; // Actually holds the queued messages + + int read_ptr; // Where the next operation will read from + int write_ptr; // Where the next operation will write to + uint64_t size; +}; + +inline static void safe_lock(pthread_mutex_t* mutex) + { + if ( pthread_mutex_lock(mutex) != 0 ) + reporter->FatalErrorWithCore("cannot lock mutex"); + } + +inline static void safe_unlock(pthread_mutex_t* mutex) + { + if ( pthread_mutex_unlock(mutex) != 0 ) + reporter->FatalErrorWithCore("cannot unlock mutex"); + } + +template +inline Queue_::Queue_() + { + read_ptr = 0; + write_ptr = 0; + + for( int i = 0; i < NUM_QUEUES; ++i ) + { + if ( pthread_cond_init(&has_data[i], NULL) != 0 ) + reporter->FatalError("cannot init queue condition variable"); + + if ( pthread_mutex_init(&mutex[i], NULL) != 0 ) + reporter->FatalError("cannot init queue mutex"); + } + } + +template +inline Queue_::~Queue_() + { + for( int i = 0; i < NUM_QUEUES; ++i ) + { + pthread_cond_destroy(&has_data[i]); + pthread_mutex_destroy(&mutex[i]); + } + } + +template +inline T Queue_::Get() + { + safe_lock(&mutex[read_ptr]); + + int old_read_ptr = read_ptr; + + if ( messages[read_ptr].empty() ) + pthread_cond_wait(&has_data[read_ptr], &mutex[read_ptr]); + + T data = messages[read_ptr].front(); + messages[read_ptr].pop(); + --size; + + read_ptr = (read_ptr + 1) % NUM_QUEUES; + + safe_unlock(&mutex[old_read_ptr]); + + return data; + } + +template +inline void Queue_::Put(T data) + { + safe_lock(&mutex[write_ptr]); + + int old_write_ptr = write_ptr; + + bool need_signal = messages[write_ptr].empty(); + + messages[write_ptr].push(data); + ++size; + + if ( need_signal ) + pthread_cond_signal(&has_data[write_ptr]); + + write_ptr = (write_ptr + 1) % NUM_QUEUES; + + safe_unlock(&mutex[old_write_ptr]); + } + + +template +inline bool Queue_::Ready() + { + safe_lock(&mutex[read_ptr]); + + bool ret = (messages[read_ptr].size()); + + safe_unlock(&mutex[read_ptr]); + + return ret; + } + +template +inline uint64_t Queue_::Size() + { + safe_lock(&mutex[read_ptr]); + + uint64_t s = size; + + safe_unlock(&mutex[read_ptr]); + + return s; + } + +} + +#endif + From 6cc29a78328cf463a4706e36b77d63ba256b9bd6 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 30 Jan 2012 12:12:14 -0800 Subject: [PATCH 05/56] make logging with threads compile on mac os and fix a couple of string literal warnings. --- src/net_util.cc | 2 +- src/threading/Manager.cc | 2 +- src/threading/MsgThread.cc | 20 +++++++++----------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/net_util.cc b/src/net_util.cc index c0bacc98b2..ebe0392e2a 100644 --- a/src/net_util.cc +++ b/src/net_util.cc @@ -322,7 +322,7 @@ const uint32* mask_addr(const uint32* a, uint32 top_bits_to_keep) if ( top_bits_to_keep == 0 || top_bits_to_keep > max_bits ) { - reporter->Error("bad address mask value %s", top_bits_to_keep); + reporter->Error("bad address mask value %u", top_bits_to_keep); return addr; } diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index ed4d9cf623..d963876755 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -90,7 +90,7 @@ void Manager::Process() else { string s = msg->Name() + " failed, terminating thread"; - reporter->Error(s.c_str()); + reporter->Error("%s", s.c_str()); t->Stop(); } diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index e2d81cf47f..d78c7533a3 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -28,7 +28,7 @@ namespace threading { class TerminateMessage : public InputMessage { public: - TerminateMessage(MsgThread* thread) : InputMessage("Terminate", thread) { } + TerminateMessage(MsgThread* thread) : InputMessage("Terminate", thread) { } virtual bool Process() { return true; } }; @@ -56,7 +56,7 @@ class HeartbeatMessage : public InputMessage { public: HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time) - : InputMessage("Heartbeat", thread) + : InputMessage("Heartbeat", thread) { network_time = arg_network_time; current_time = arg_current_time; } virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); } @@ -98,38 +98,36 @@ Message::~Message() bool ReporterMessage::Process() { string s = Object()->Name() + ": " + msg; - strreplace(s, "%", "%%"); - const char* cmsg = s.c_str(); switch ( type ) { case INFO: - reporter->Info(cmsg); + reporter->Info("%s", cmsg); break; case WARNING: - reporter->Warning(cmsg); + reporter->Warning("%s", cmsg); break; case ERROR: - reporter->Error(cmsg); + reporter->Error("%s", cmsg); break; case FATAL_ERROR: - reporter->FatalError(cmsg); + reporter->FatalError("%s", cmsg); break; case FATAL_ERROR_WITH_CORE: - reporter->FatalErrorWithCore(cmsg); + reporter->FatalErrorWithCore("%s", cmsg); break; case INTERNAL_WARNING: - reporter->InternalWarning(cmsg); + reporter->InternalWarning("%s", cmsg); break; case INTERNAL_ERROR : - reporter->InternalError(cmsg); + reporter->InternalError("%s", cmsg); break; default: From a428645b2a88f20c6e9f48c573e88b74c3c03398 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 31 Jan 2012 23:47:33 -0800 Subject: [PATCH 06/56] Documenting the threading/* classes. Also switching from semaphores to mutexes as the former don't seem to be fully supported on MacOS. --- src/threading/BasicThread.cc | 29 ++-- src/threading/BasicThread.h | 119 ++++++++++++--- src/threading/Manager.h | 63 +++++++- src/threading/MsgThread.cc | 56 +++---- src/threading/MsgThread.h | 282 +++++++++++++++++++++++++++++++---- src/threading/Queue.h | 56 +++++-- 6 files changed, 499 insertions(+), 106 deletions(-) diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 273a192de3..f7bd2afbcd 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -7,6 +7,8 @@ using namespace threading; +uint64_t BasicThread::thread_counter = 0; + BasicThread::BasicThread(const string& arg_name) { started = false; @@ -16,9 +18,7 @@ BasicThread::BasicThread(const string& arg_name) buf = 0; buf_len = 1024; - char tmp[128]; - snprintf(tmp, sizeof(tmp), "%s@%p", arg_name.c_str(), this); - name = string(tmp); + name = Fmt("%s@%d", arg_name.c_str(), ++thread_counter); thread_mgr->AddThread(this); } @@ -53,8 +53,15 @@ const char* BasicThread::Fmt(const char* format, ...) void BasicThread::Start() { - if ( sem_init(&terminate, 0, 0) != 0 ) - reporter->FatalError("Cannot create terminate semaphore for thread %s", name.c_str()); + if ( started ) + return; + + if ( pthread_mutex_init(&terminate, 0) != 0 ) + reporter->FatalError("Cannot create terminate mutex for thread %s", name.c_str()); + + // We use this like a binary semaphore and acquire it immediately. + if ( pthread_mutex_lock(&terminate) != 0 ) + reporter->FatalError("Cannot aquire terminate mutex for thread %s", name.c_str()); if ( pthread_create(&pthread, 0, BasicThread::launcher, this) != 0 ) reporter->FatalError("Cannot create thread %s", name.c_str()); @@ -76,8 +83,9 @@ void BasicThread::Stop() DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str()); - // Signal that it's ok for the thread to exit now. - if ( sem_post(&terminate) != 0 ) + // Signal that it's ok for the thread to exit now by unlocking the + // mutex. + if ( pthread_mutex_unlock(&terminate) != 0 ) reporter->FatalError("Failure flagging terminate condition for thread %s", name.c_str()); terminating = true; @@ -98,7 +106,7 @@ void BasicThread::Join() if ( pthread_join(pthread, 0) != 0 ) reporter->FatalError("Failure joining thread %s", name.c_str()); - sem_destroy(&terminate); + pthread_mutex_destroy(&terminate); DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str()); @@ -120,9 +128,8 @@ void* BasicThread::launcher(void *arg) thread->Run(); // Wait until somebody actually wants us to terminate. - - if ( sem_wait(&thread->terminate) != 0 ) - reporter->FatalError("Failure flagging terminate condition for thread %s", thread->Name().c_str()); + if ( pthread_mutex_lock(&thread->terminate) != 0 ) + reporter->FatalError("Failure acquiring terminate mutex at end of thread %s", thread->Name().c_str()); return 0; } diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index 30a11b4505..df5665c464 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -14,50 +14,133 @@ namespace threading { class Manager; +/** + * Base class for all threads. + * + * This class encapsulates all the OS-level thread handling. All thread + * instances are automatically added to the threading::Manager for management. The + * manager also takes care of deleting them (which must not be done + * manually). + */ class BasicThread { public: + /** + * Creates a new thread object. Instantiating the object does however + * not yet start the actual OS thread, that requires calling Start(). + * + * Only Bro's main thread may create new thread instances. + * + * @param name A descriptive name for thread the thread. This may + * show up in messages to the user. + */ BasicThread(const string& name); // Managed by manager, must not delete otherwise. - virtual ~BasicThread(); + /** + * Returns a descriptive name for the thread. This is the name passed + * into the constructor. + * + * This method is safe to call from any thread. + */ const string& Name() const { return name; } - void Start(); // Spawns the thread and enters Run(). - void Stop(); // Signals the thread to terminate. + /** + * Starts the thread. Calling this methods will spawn a new OS thread + * executing Run(). Note that one can't restart a thread after a + * Stop(), doing so will be ignored. + * + * Only Bro's main thread must call this method. + */ + void Start(); + /** + * Signals the thread to stop. The method lets Terminating() now + * return true. It does however not force the thread to terminate. + * It's up to the Run() method to to query Terminating() and exit + * eventually. + * + * Calling this method has no effect if Start() hasn't been executed + * yet. + * + * Only Bro's main thread must call this method. + */ + void Stop(); + + /** + * Returns true if Terminate() has been called. + * + * This method is safe to call from any thread. + */ bool Terminating() const { return terminating; } - // A thread-safe version of fmt(). + /** + * A version of fmt() that the thread can safely use. + * + * This is safe to call from Run() but must not be used from any + * other thread than the current one. + */ const char* Fmt(const char* format, ...); protected: - virtual void Run() = 0; - - virtual void OnStart() {} - virtual void OnStop() {} - -private: friend class Manager; + /** + * Entry point for the thread. This must be overridden by derived + * classes and will execute in a separate thread once Start() is + * called. The thread will not terminate before this method finishes. + * An implementation should regularly check Terminating() to see if + * exiting has been requested. + */ + virtual void Run() = 0; + + /** + * Executed with Start(). This is a hook into starting the thread. It + * will be called from Bro's main thread after the OS thread has been + * started. + */ + virtual void OnStart() {} + + /** + * Executed with Stop(). This is a hook into stopping the thread. It + * will be called from Bro's main thread after the thread has been + * signaled to stop. + */ + virtual void OnStop() {} + + /** + * Destructor. This will be called by the manager. + * + * Only Bro's main thread may delete thread instances. + * + */ + virtual ~BasicThread(); + + /** + * Waits until the thread's Run() method has finished and then joins + * it. This is called from the threading::Manager. + */ + void Join(); + +private: + // pthread entry function. static void* launcher(void *arg); - // Used from the ThreadMgr. - void Join(); // Waits until the thread has terminated and then joins it. - + string name; + pthread_t pthread; bool started; // Set to to true once running. bool terminating; // Set to to true to signal termination. - string name; - pthread_t pthread; - sem_t terminate; + // Used as a semaphore to tell the pthread thread when it may + // terminate. + pthread_mutex_t terminate; // For implementing Fmt(). char* buf; unsigned int buf_len; + + static uint64_t thread_counter; }; } -extern threading::Manager* thread_mgr; - #endif diff --git a/src/threading/Manager.h b/src/threading/Manager.h index aa7292ee81..2c4f88fa1e 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -11,25 +11,78 @@ namespace threading { +/** + * The thread manager coordinates all child threads. Once a BasicThread is + * instantitated, it gets addedd to the manager, which will delete it later + * once it has terminated. + * + * In addition to basic threads, the manager also provides additional + * functionality specific to MsgThread instances. In particular, it polls + * their outgoing message queue on a regular basis and feeds data sent into + * the rest of Bro. It also triggers the regular heartbeats. + */ class Manager : public IOSource { public: + /** + * Constructor. Only a single instance of the manager must be + * created. + */ Manager(); + + /** + * Destructir. + */ ~Manager(); + /** + * Terminates the manager's processor. The method signals all threads + * to terminates and wait for them to do so. It then joins them and + * returns to the caller. Afterwards, no more thread instances may be + * created. + */ void Terminate(); protected: friend class BasicThread; friend class MsgThread; + /** + * Registers a new basic thread with the manager. This is + * automatically called by the thread's constructor. + * + * @param thread The thread. + */ void AddThread(BasicThread* thread); + + /** + * Registers a new message thread with the manager. This is + * automatically called by the thread's constructor. This must be + * called \a in \a addition to AddThread(BasicThread* thread). The + * MsgThread constructor makes sure to do so. + * + * @param thread The thread. + */ void AddMsgThread(MsgThread* thread); - // IOSource interface. + /** + * Part of the IOSource interface. + */ virtual void GetFds(int* read, int* write, int* except); + + /** + * Part of the IOSource interface. + */ virtual double NextTimestamp(double* network_time); + + /** + * Part of the IOSource interface. + */ virtual void Process(); + + /** + * Part of the IOSource interface. + */ virtual const char* Tag() { return "threading::Manager"; } private: @@ -41,12 +94,16 @@ private: typedef std::list msg_thread_list; msg_thread_list msg_threads; - bool did_process; - double next_beat; + bool did_process; // True if the last Process() found some work to do. + double next_beat; // Timestamp when the next heartbeat will be sent. }; } +/** + * A singleton instance of the thread manager. All methods must only be + * called from Bro's main thread. + */ extern threading::Manager* thread_mgr; #endif diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index d78c7533a3..455c177df6 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -6,25 +6,13 @@ using namespace threading; -static void strreplace(const string& s, const string& o, const string& n) - { - string r = s; - - while ( true ) - { - size_t i = r.find(o); - - if ( i == std::string::npos ) - break; - - r.replace(i, o.size(), n); - } - } - namespace threading { -// Standard messages. +////// Messages. +// Signals child thread to terminate. This is actually a no-op; its only +// purpose is unblock the current read operation so that the child's Run() +// methods can check the termination status. class TerminateMessage : public InputMessage { public: @@ -33,6 +21,22 @@ public: virtual bool Process() { return true; } }; +/// Sends a heartbeat to the child thread. +class HeartbeatMessage : public InputMessage +{ +public: + HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time) + : InputMessage("Heartbeat", thread) + { network_time = arg_network_time; current_time = arg_current_time; } + + virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); } + +private: + double network_time; + double current_time; +}; + +// A message from the child to be passed on to the Reporter. class ReporterMessage : public OutputMessage { public: @@ -52,21 +56,8 @@ private: Type type; }; -class HeartbeatMessage : public InputMessage -{ -public: - HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time) - : InputMessage("Heartbeat", thread) - { network_time = arg_network_time; current_time = arg_current_time; } - - virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); } - -private: - double network_time; - double current_time; -}; - #ifdef DEBUG +// A debug message from the child to be passed on to the DebugLogger. class DebugMessage : public OutputMessage { public: @@ -77,8 +68,7 @@ public: virtual bool Process() { string s = Object()->Name() + ": " + msg; - strreplace(s, "%", "%%"); - debug_logger.Log(stream, s.c_str()); + debug_logger.Log(stream, "%s", s.c_str()); return true; } private: @@ -89,7 +79,7 @@ private: } -// Methods. +////// Methods. Message::~Message() { diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 2e976c1773..8f37041bb6 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -15,121 +15,332 @@ class BasicInputMessage; class BasicOutputMessage; class HeartbeatMessage; +/** + * A specialized thread that provides bi-directional message passing between + * Bro's main thread and the child thread. Messages are instances of + * BasicInputMessage and BasicOutputMessage for message sent \a to the child + * thread and received \a from the child thread, respectively. + * + * The thread's Run() method implements main loop that processes incoming + * messages until Terminating() indicates that execution should stop. Once + * that happens, the thread stops accepting any new messages, finishes + * processes all remaining ones still in the queue, and then exits. + */ class MsgThread : public BasicThread { public: + /** + * Constructor. It automatically registers the thread with the + * threading::Manager. + * + * Only Bro's main thread may instantiate a new thread. + * + * @param name A descriptive name. This is passed on to BasicThread(). + */ MsgThread(const string& name); + /** + * Sends a message to the child thread. The message will be proceesed + * once the thread has retrieved it from its incoming queue. + * + * Only the main thread may call this method. + * + * @param msg The message. + */ void SendIn(BasicInputMessage* msg) { return SendIn(msg, false); } + + /** + * Sends a message from the child thread to the main thread. + * + * Only the child thread may call this method. + * + * @param msg The mesasge. + */ void SendOut(BasicOutputMessage* msg) { return SendOut(msg, false); } - BasicOutputMessage* RetrieveOut(); - - // Report an informational message, nothing that needs specific - // attention. + /** + * Reports an informational message from the child thread. The main + * thread will pass this to the Reporter once received. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void Info(const char* msg); - // Report a warning that may indicate a problem. + /** + * Reports a warning from the child thread that may indicate a + * problem. The main thread will pass this to the Reporter once + * received. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void Warning(const char* msg); - // Report a non-fatal error. Processing proceeds normally after the error - // has been reported. + /** + * Reports a non-fatal error from the child thread. The main thread + * will pass this to the Reporter once received. Processing proceeds + * normally after the error has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void Error(const char* msg); - // Report a fatal error. Bro will terminate after the message has been - // reported. + /** + * Reports a fatal error from the child thread. The main thread will + * pass this to the Reporter once received. Bro will terminate after + * the message has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void FatalError(const char* msg); - // Report a fatal error. Bro will terminate after the message has been - // reported and always generate a core dump. + /** + * Reports a fatal error from the child thread. The main thread will + * pass this to the Reporter once received. Bro will terminate with a + * core dump after the message has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void FatalErrorWithCore(const char* msg); - // Report about a potential internal problem. Bro will continue - // normally. + /** + * Reports a potential internal problem from the child thread. The + * main thread will pass this to the Reporter once received. Bro will + * continue normally. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void InternalWarning(const char* msg); - // Report an internal program error. Bro will terminate with a core - // dump after the message has been reported. + /** + * Reports an internal program error from the child thread. The main + * thread will pass this to the Reporter once received. Bro will + * terminate with a core dump after the message has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void InternalError(const char* msg); #ifdef DEBUG - // Records a debug message for the given stream. + /** + * Records a debug message for the given stream from the child + * thread. The main thread will pass this to the DebugLogger once + * received. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ void Debug(DebugStream stream, const char* msg); #endif - void Heartbeat(); - + /** + * Statistics about inter-thread communication. + */ struct Stats { - uint64_t sent_in; - uint64_t sent_out; - uint64_t pending_in; - uint64_t pending_out; + uint64_t sent_in; //! Number of messages sent to the child thread. + uint64_t sent_out; //! Number of messages sent from the child thread to the main thread + uint64_t pending_in; //! Number of messages sent to the child but not yet processed. + uint64_t pending_out; //! Number of messages sent from the child but not yet processed by the main thread. }; + /** + * Returns statistics about the inter-thread communication. + * + * @param stats A pointer to a structure that will be filled with + * current numbers. + */ void GetStats(Stats* stats); protected: + friend class Manager; friend class HeartbeatMessage; + /** + * Pops a message sent by the child from the child-to-main queue. + * + * This is method is called regularly by the threading::Manager. + * + * @return The message, wth ownership passed to caller. Returns null + * if the queue is empty. + */ + BasicOutputMessage* RetrieveOut(); + + /** + * Triggers a heartbeat message being sent to the client thread. + * + * This is method is called regularly by the threading::Manager. + */ + void Heartbeat(); + + /** + * Overriden from BasicThread. + * + */ virtual void Run(); virtual void OnStop(); virtual bool DoHeartbeat(double network_time, double current_time) { return true; } private: - friend class Manager; - + /** + * Pops a message sent by the main thread from the main-to-chold + * queue. + * + * Must only be called by the child thread. + * + * @return The message, wth ownership passed to caller. Returns null + * if the queue is empty. + */ BasicInputMessage* RetrieveIn(); + /** + * Queues a message for the child. + * + * Must only be called by the main thread. + * + * @param msg The message. + * + * @param force: If true, the message will be queued even when we're already + * Terminating(). Normally, the message would be discarded in that + * case. + */ void SendIn(BasicInputMessage* msg, bool force); + + /** + * Queues a message for the main thread. + * + * Must only be called by the child thread. + * + * @param msg The message. + * + * @param force: If true, the message will be queued even when we're already + * Terminating(). Normally, the message would be discarded in that + * case. + */ void SendOut(BasicOutputMessage* msg, bool force); + /** + * Returns true if there's at least one message pending for the child + * thread. + */ bool HasIn() { return queue_in.Ready(); } + + /** + * Returns true if there's at least one message pending for the main + * thread. + */ bool HasOut() { return queue_out.Ready(); } - Queue_ queue_in; - Queue_ queue_out; + Queue queue_in; + Queue queue_out; - uint64_t cnt_sent_in; - uint64_t cnt_sent_out; + uint64_t cnt_sent_in; // Counts message sent to child. + uint64_t cnt_sent_out; // Counts message sent by child. }; +/** + * Base class for all message between Bro's main process and a MsgThread. + */ class Message { public: + /** + * Destructor. + */ virtual ~Message(); + /** + * Returns a descriptive name for the message's general type. This is + * what's passed into the constructor and used mainly for debugging + * purposes. + */ const string& Name() const { return name; } + /** + * Callback that must be overriden for processing a message. + */ virtual bool Process() = 0; // Thread will be terminated if returngin false. protected: + /** + * Constructor. + * + * @param arg_name A descriptive name for the type of message. Used + * mainly for debugging purposes. + */ Message(const string& arg_name) { name = arg_name; } private: string name; }; +/** + * Base class for messages sent from Bro's main thread to a child MsgThread. + */ class BasicInputMessage : public Message { protected: + /** + * Constructor. + * + * @param name A descriptive name for the type of message. Used + * mainly for debugging purposes. + */ BasicInputMessage(const string& name) : Message(name) {} }; +/** + * Base class for messages sent from a child MsgThread to Bro's main thread. + */ class BasicOutputMessage : public Message { protected: + /** + * Constructor. + * + * @param name A descriptive name for the type of message. Used + * mainly for debugging purposes. + */ BasicOutputMessage(const string& name) : Message(name) {} }; +/** + * A paremeterized InputMessage that stores a pointer to an argument object. + * Normally, the objects will be used from the Process() callback. + */ template class InputMessage : public BasicInputMessage { public: + /** + * Returns the objects passed to the constructor. + */ O* Object() const { return object; } protected: + /** + * Constructor. + * + * @param name: A descriptive name for the type of message. Used + * mainly for debugging purposes. + * + * @param arg_object: An object to store with the message. + */ InputMessage(const string& name, O* arg_object) : BasicInputMessage(name) { object = arg_object; } @@ -137,13 +348,28 @@ private: O* object; }; +/** + * A paremeterized OututMessage that stores a pointer to an argument object. + * Normally, the objects will be used from the Process() callback. + */ template class OutputMessage : public BasicOutputMessage { public: + /** + * Returns the objects passed to the constructor. + */ O* Object() const { return object; } protected: + /** + * Constructor. + * + * @param name A descriptive name for the type of message. Used + * mainly for debugging purposes. + * + * @param arg_object An object to store with the message. + */ OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name) { object = arg_object; } diff --git a/src/threading/Queue.h b/src/threading/Queue.h index 49859dc051..add7019f9c 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -9,23 +9,53 @@ #include "Reporter.h" +#undef Queue // Defined elsewhere unfortunately. + namespace threading { /** - * Just a simple threaded queue wrapper class. Uses multiple queues and reads / writes in rotary fashion in an attempt to limit contention. - * Due to locking granularity, bulk put / get is no faster than single put / get as long as FIFO guarantee is required. + * A thread-safe single-reader single-writer queue. + * + * The implementation uses multiple queues and reads/writes in rotary fashion + * in an attempt to limit contention. + * + * All Queue instances must be instantiated by Bro's main thread. + * + * TODO: Unclear how critical performance is for this qeueue. We could like;y + * optimize it further if helpful. */ - template -class Queue_ +class Queue { public: - Queue_(); - ~Queue_(); + /** + * Constructor. + */ + Queue(); + /** + * Destructor. + */ + ~Queue(); + + /** + * Retrieves one elment. + */ T Get(); + + /** + * Queues one element. + */ void Put(T data); + + /** + * Returns true if the next Get() operation will succeed. + */ bool Ready(); + + /** + * Returns the number of queued items not yet retrieved. + */ uint64_t Size(); private: @@ -37,7 +67,7 @@ private: int read_ptr; // Where the next operation will read from int write_ptr; // Where the next operation will write to - uint64_t size; + uint64_t size; // Current queue size. }; inline static void safe_lock(pthread_mutex_t* mutex) @@ -53,7 +83,7 @@ inline static void safe_unlock(pthread_mutex_t* mutex) } template -inline Queue_::Queue_() +inline Queue::Queue() { read_ptr = 0; write_ptr = 0; @@ -69,7 +99,7 @@ inline Queue_::Queue_() } template -inline Queue_::~Queue_() +inline Queue::~Queue() { for( int i = 0; i < NUM_QUEUES; ++i ) { @@ -79,7 +109,7 @@ inline Queue_::~Queue_() } template -inline T Queue_::Get() +inline T Queue::Get() { safe_lock(&mutex[read_ptr]); @@ -100,7 +130,7 @@ inline T Queue_::Get() } template -inline void Queue_::Put(T data) +inline void Queue::Put(T data) { safe_lock(&mutex[write_ptr]); @@ -121,7 +151,7 @@ inline void Queue_::Put(T data) template -inline bool Queue_::Ready() +inline bool Queue::Ready() { safe_lock(&mutex[read_ptr]); @@ -133,7 +163,7 @@ inline bool Queue_::Ready() } template -inline uint64_t Queue_::Size() +inline uint64_t Queue::Size() { safe_lock(&mutex[read_ptr]); From 4f0fc571eff7e6fc3db5c06f8097159c765ee69d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 1 Feb 2012 00:34:18 -0800 Subject: [PATCH 07/56] Doing bulkd writes instead of individual writes now. Also slight change to Writer API, going back to how the rotate methods were before. --- src/logging/WriterBackend.cc | 86 +++++++++++++++++++++++------------ src/logging/WriterBackend.h | 31 +++++++------ src/logging/WriterFrontend.cc | 60 ++++++++++++++++++++---- src/logging/WriterFrontend.h | 11 +++-- src/logging/writers/Ascii.cc | 5 +- src/logging/writers/Ascii.h | 4 +- src/logging/writers/None.cc | 5 +- src/logging/writers/None.h | 4 +- src/threading/MsgThread.h | 16 ++++++- 9 files changed, 155 insertions(+), 67 deletions(-) diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 095490edc4..7c9c1d10ca 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -31,13 +31,13 @@ private: bool terminating; }; -class DisableMessage : public threading::OutputMessage +class FlushWriteBufferMessage : public threading::OutputMessage { public: - DisableMessage(WriterFrontend* writer) - : threading::OutputMessage("Disable", writer) {} + FlushWriteBufferMessage(WriterFrontend* writer) + : threading::OutputMessage("FlushWriteBuffer", writer) {} - virtual bool Process() { Object()->SetDisable(); return true; } + virtual bool Process() { Object()->FlushWriteBuffer(); return true; } }; } @@ -65,25 +65,31 @@ WriterBackend::~WriterBackend() } } -void WriterBackend::DeleteVals(Value** vals) +void WriterBackend::DeleteVals(int num_writes, Value*** vals) { - // Note this code is duplicated in Manager::DeleteVals(). - for ( int i = 0; i < num_fields; i++ ) - delete vals[i]; + for ( int j = 0; j < num_writes; ++j ) + { + // Note this code is duplicated in Manager::DeleteVals(). + for ( int i = 0; i < num_fields; i++ ) + delete vals[j][i]; + + delete [] vals[j]; + } delete [] vals; } -bool WriterBackend::FinishedRotation(WriterFrontend* writer, string new_name, string old_name, +bool WriterBackend::FinishedRotation(string new_name, string old_name, double open, double close, bool terminating) { - SendOut(new RotationFinishedMessage(writer, new_name, old_name, open, close, terminating)); + SendOut(new RotationFinishedMessage(frontend, new_name, old_name, open, close, terminating)); return true; } -bool WriterBackend::Init(string arg_path, int arg_num_fields, - const Field* const * arg_fields) +bool WriterBackend::Init(WriterFrontend* arg_frontend, string arg_path, int arg_num_fields, + const Field* const * arg_fields) { + frontend = arg_frontend; path = arg_path; num_fields = arg_num_fields; fields = arg_fields; @@ -94,7 +100,7 @@ bool WriterBackend::Init(string arg_path, int arg_num_fields, return true; } -bool WriterBackend::Write(int arg_num_fields, Value** vals) +bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals) { // Double-check that the arguments match. If we get this from remote, // something might be mixed up. @@ -107,30 +113,42 @@ bool WriterBackend::Write(int arg_num_fields, Value** vals) Debug(DBG_LOGGING, msg); #endif - DeleteVals(vals); + DeleteVals(num_writes, vals); return false; } - for ( int i = 0; i < num_fields; ++i ) - { - if ( vals[i]->type != fields[i]->type ) - { #ifdef DEBUG - const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)", - vals[i]->type, fields[i]->type); - Debug(DBG_LOGGING, msg); -#endif + // Double-check all the types match. + for ( int j = 0; j < num_writes; j++ ) + { + for ( int i = 0; i < num_fields; ++i ) + { + if ( vals[j][i]->type != fields[i]->type ) + { + const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)", + vals[j][i]->type, fields[i]->type); + Debug(DBG_LOGGING, msg); - DeleteVals(vals); - return false; + DeleteVals(num_writes, vals); + return false; + } } } +#endif - bool result = DoWrite(num_fields, fields, vals); + bool success = true; - DeleteVals(vals); + for ( int j = 0; j < num_writes; j++ ) + { + success = DoWrite(num_fields, fields, vals[j]); - return result; + if ( ! success ) + break; + } + + DeleteVals(num_writes, vals); + + return success; } bool WriterBackend::SetBuf(bool enabled) @@ -144,10 +162,10 @@ bool WriterBackend::SetBuf(bool enabled) return DoSetBuf(enabled); } -bool WriterBackend::Rotate(WriterFrontend* writer, string rotated_path, - double open, double close, bool terminating) +bool WriterBackend::Rotate(string rotated_path, double open, + double close, bool terminating) { - return DoRotate(writer, rotated_path, open, close, terminating); + return DoRotate(rotated_path, open, close, terminating); } bool WriterBackend::Flush() @@ -159,3 +177,11 @@ bool WriterBackend::Finish() { return DoFinish(); } + +bool WriterBackend::DoHeartbeat(double network_time, double current_time) + { + SendOut(new FlushWriteBufferMessage(frontend)); + return true; + } + + diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index d1e4634e6d..27f4fe45a5 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -19,6 +19,12 @@ public: virtual ~WriterBackend(); // One-time initialization of the writer to define the logged fields. + // + // "frontend" is the frontend writer that created this backend. The + // *only* purpose of this value is to be passed back via messages as + // a argument to callbacks. One must not otherwise access the + // frontend, it's running in a different thread. + // // Interpretation of "path" is left to the writer, and will be // corresponding the value configured on the script-level. // @@ -27,7 +33,7 @@ public: // // The new instance takes ownership of "fields", and will delete them // when done. - bool Init(string path, int num_fields, const Field* const * fields); + bool Init(WriterFrontend* frontend, string path, int num_fields, const Field* const * fields); // Writes one log entry. The method takes ownership of "vals" and // will return immediately after queueing the write request, which is @@ -38,7 +44,7 @@ public: // // Returns false if an error occured, in which case the writer must // not be used any further. - bool Write(int num_fields, Value** vals); + bool Write(int num_fields, int num_writes, Value*** vals); // Sets the buffering status for the writer, if the writer supports // that. (If not, it will be ignored). @@ -50,7 +56,7 @@ public: // Triggers rotation, if the writer supports that. (If not, it will // be ignored). - bool Rotate(WriterFrontend* writer, string rotated_path, double open, double close, bool terminating); + bool Rotate(string rotated_path, double open, double close, bool terminating); // Finishes writing to this logger regularly. Must not be called if // an error has been indicated earlier. After calling this, no @@ -81,9 +87,10 @@ public: // // terminating: True if rotation request occured due to the main Bro // process shutting down. - bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, + bool FinishedRotation(string new_name, string old_name, double open, double close, bool terminating); + protected: // Methods for writers to override. If any of these returs false, it // will be assumed that a fatal error has occured that prevents the @@ -128,11 +135,6 @@ protected: // RotationDone() to signal the log manager that potential // postprocessors can now run. // - // "writer" is the frontend writer that triggered the rotation. The - // *only* purpose of this value is to be passed into - // FinishedRotation() once done. You must not otherwise access the - // frontend, it's running in a different thread. - // // "rotate_path" reflects the path to where the rotated output is to // be moved, with specifics depending on the writer. It should // generally be interpreted in a way consistent with that of "path" @@ -149,8 +151,8 @@ protected: // // A writer may ignore rotation requests if it doesn't fit with its // semantics (but must still return true in that case). - virtual bool DoRotate(WriterFrontend* writer, string rotated_path, - double open, double close, bool terminating) = 0; + virtual bool DoRotate(string rotated_path, double open, double close, + bool terminating) = 0; // Called once on termination. Not called when any of the other // methods has previously signaled an error, i.e., executing this @@ -158,7 +160,9 @@ protected: virtual bool DoFinish() = 0; // Triggered by regular heartbeat messages from the main process. - virtual bool DoHeartbeat(double network_time, double current_time) { return true; }; + // + // Note when overriding, you must call WriterBackend::DoHeartbeat(). + virtual bool DoHeartbeat(double network_time, double current_time); private: friend class Manager; @@ -169,8 +173,9 @@ private: bool Disabled() { return disabled; } // Deletes the values as passed into Write(). - void DeleteVals(Value** vals); + void DeleteVals(int num_writes, Value*** vals); + WriterFrontend* frontend; string path; int num_fields; const Field* const * fields; diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 92c93c1c56..2f7c1d6e7e 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -9,13 +9,14 @@ namespace logging { class InitMessage : public threading::InputMessage { public: - InitMessage(WriterBackend* backend, const string path, const int num_fields, const Field* const *fields) + InitMessage(WriterBackend* backend, WriterFrontend* frontend, const string path, const int num_fields, const Field* const *fields) : threading::InputMessage("Init", backend), path(path), num_fields(num_fields), fields(fields) { } - virtual bool Process() { return Object()->Init(path, num_fields, fields); } + virtual bool Process() { return Object()->Init(frontend, path, num_fields, fields); } private: + WriterFrontend* frontend; const string path; const int num_fields; const Field * const* fields; @@ -31,7 +32,7 @@ public: rotated_path(rotated_path), open(open), close(close), terminating(terminating) { } - virtual bool Process() { return Object()->Rotate(frontend, rotated_path, open, close, terminating); } + virtual bool Process() { return Object()->Rotate(rotated_path, open, close, terminating); } private: WriterFrontend* frontend; @@ -44,16 +45,16 @@ private: class WriteMessage : public threading::InputMessage { public: - WriteMessage(WriterBackend* backend, const int num_fields, Value **vals) + WriteMessage(WriterBackend* backend, int num_fields, int num_writes, Value*** vals) : threading::InputMessage("Write", backend), - num_fields(num_fields), fields(fields), vals(vals) {} + num_fields(num_fields), vals(vals) {} - virtual bool Process() { return Object()->Write(num_fields, vals); } + virtual bool Process() { return Object()->Write(num_fields, num_writes, vals); } private: int num_fields; - Field* const* fields; - Value **vals; + int num_writes; + Value ***vals; }; class SetBufMessage : public threading::InputMessage @@ -96,6 +97,8 @@ using namespace logging; WriterFrontend::WriterFrontend(bro_int_t type) { disabled = initialized = false; + buf = true; + write_buffer_pos = 0; backend = log_mgr->CreateBackend(type); assert(backend); @@ -108,6 +111,7 @@ WriterFrontend::~WriterFrontend() void WriterFrontend::Stop() { + FlushWriteBuffer(); SetDisable(); backend->Stop(); } @@ -125,7 +129,7 @@ void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* cons fields = arg_fields; initialized = true; - backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + backend->SendIn(new InitMessage(backend, this, arg_path, arg_num_fields, arg_fields)); } void WriterFrontend::Write(int num_fields, Value** vals) @@ -133,7 +137,34 @@ void WriterFrontend::Write(int num_fields, Value** vals) if ( disabled ) return; - backend->SendIn(new WriteMessage(backend, num_fields, vals)); + if ( ! write_buffer ) + { + // Need new buffer. + write_buffer = new Value**[WRITER_BUFFER_SIZE]; + write_buffer_pos = 0; + } + + if ( write_buffer_pos >= WRITER_BUFFER_SIZE ) + // Buffer full. + FlushWriteBuffer(); + + write_buffer[write_buffer_pos++] = vals; + + if ( ! buf ) + // Send out immediately if we don't want buffering. + FlushWriteBuffer(); + } + +void WriterFrontend::FlushWriteBuffer() + { + if ( ! write_buffer_pos ) + // Nothing to do. + return; + + backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer)); + + // Clear buffer (no delete, we pass ownership to child thread.) + write_buffer = 0; } void WriterFrontend::SetBuf(bool enabled) @@ -141,7 +172,13 @@ void WriterFrontend::SetBuf(bool enabled) if ( disabled ) return; + buf = enabled; + backend->SendIn(new SetBufMessage(backend, enabled)); + + if ( ! buf ) + // Make sure no longer buffer any still queued data. + FlushWriteBuffer(); } void WriterFrontend::Flush() @@ -149,6 +186,7 @@ void WriterFrontend::Flush() if ( disabled ) return; + FlushWriteBuffer(); backend->SendIn(new FlushMessage(backend)); } @@ -157,6 +195,7 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool if ( disabled ) return; + FlushWriteBuffer(); backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); } @@ -165,6 +204,7 @@ void WriterFrontend::Finish() if ( disabled ) return; + FlushWriteBuffer(); backend->SendIn(new FinishMessage(backend)); } diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index 1998429d38..ed1a674842 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -34,6 +34,7 @@ public: void Write(int num_fields, Value** vals); void SetBuf(bool enabled); void Flush(); + void FlushWriteBuffer(); void Rotate(string rotated_path, double open, double close, bool terminating); void Finish(); @@ -49,18 +50,22 @@ public: protected: friend class Manager; - WriterBackend* backend; bool disabled; bool initialized; + bool buf; string path; int num_fields; const Field* const * fields; + + // Buffer for bulk writes. + static const int WRITER_BUFFER_SIZE = 50; + + int write_buffer_pos; + Value*** write_buffer; }; } - - #endif diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 70f513be3b..a1ceb6e217 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -317,8 +317,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, return true; } -bool Ascii::DoRotate(WriterFrontend* writer, string rotated_path, double open, - double close, bool terminating) +bool Ascii::DoRotate(string rotated_path, double open, double close, bool terminating) { // Don't rotate special files or if there's not one currently open. if ( ! file || IsSpecial(Path()) ) @@ -330,7 +329,7 @@ bool Ascii::DoRotate(WriterFrontend* writer, string rotated_path, double open, string nname = rotated_path + "." + LogExt(); rename(fname.c_str(), nname.c_str()); - if ( ! FinishedRotation(writer, nname, fname, open, close, terminating) ) + if ( ! FinishedRotation(nname, fname, open, close, terminating) ) { Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str())); return false; diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 37fcfef267..0c627c68e9 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -23,8 +23,8 @@ protected: virtual bool DoWrite(int num_fields, const Field* const * fields, Value** vals); virtual bool DoSetBuf(bool enabled); - virtual bool DoRotate(WriterFrontend* writer, string rotated_path, - double open, double close, bool terminating); + virtual bool DoRotate(string rotated_path, double open, + double close, bool terminating); virtual bool DoFlush(); virtual bool DoFinish(); diff --git a/src/logging/writers/None.cc b/src/logging/writers/None.cc index e419d88a6b..a9a7872f85 100644 --- a/src/logging/writers/None.cc +++ b/src/logging/writers/None.cc @@ -4,10 +4,9 @@ using namespace logging; using namespace writer; -bool None::DoRotate(WriterFrontend* writer, string rotated_path, - double open, double close, bool terminating) +bool None::DoRotate(string rotated_path, double open, double close, bool terminating) { - if ( ! FinishedRotation(writer, string("/dev/null"), Path(), open, close, terminating)) + if ( ! FinishedRotation(string("/dev/null"), Path(), open, close, terminating)) { Error(Fmt("error rotating %s", Path().c_str())); return false; diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h index 9b2ab6c698..9360ef44f6 100644 --- a/src/logging/writers/None.h +++ b/src/logging/writers/None.h @@ -23,8 +23,8 @@ protected: virtual bool DoWrite(int num_fields, const Field* const * fields, Value** vals) { return true; } virtual bool DoSetBuf(bool enabled) { return true; } - virtual bool DoRotate(WriterFrontend* writer, string rotated_path, - double open, double close, bool terminating); + virtual bool DoRotate(string rotated_path, double open, + double close, bool terminating); virtual bool DoFlush() { return true; } virtual bool DoFinish() { return true; } }; diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 8f37041bb6..ec249e90ad 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -184,8 +184,11 @@ protected: * Triggers a heartbeat message being sent to the client thread. * * This is method is called regularly by the threading::Manager. + * + * Can be overriden in derived classed to hook into the heart beat, + * but must call the parent implementation. */ - void Heartbeat(); + virtual void Heartbeat(); /** * Overriden from BasicThread. @@ -194,6 +197,17 @@ protected: virtual void Run(); virtual void OnStop(); + /** + * Regulatly triggered for execution in the child thread. + * + * When overriding, one must call the parent class' implementation. + * + * network_time: The network_time when the heartbeat was trigger by + * the main thread. + * + * current_time: Wall clock when the heartbeat was trigger by the + * main thread. + */ virtual bool DoHeartbeat(double network_time, double current_time) { return true; } private: From 1058e11ffbe9c6e659f963a87a9c696965207e58 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 1 Feb 2012 04:40:01 -0800 Subject: [PATCH 08/56] Adding thread statistics to prof.log --- src/Stats.cc | 15 ++++++++++++++- src/threading/Manager.cc | 17 +++++++++++++++++ src/threading/Manager.h | 21 +++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/Stats.cc b/src/Stats.cc index 55835613e9..05ce33daed 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -9,7 +9,7 @@ #include "ConnCompressor.h" #include "DNS_Mgr.h" #include "Trigger.h" - +#include "threading/Manager.h" int killed_by_inactivity = 0; @@ -217,6 +217,19 @@ void ProfileLogger::Log() current_timers[i])); } + file->Write(fmt("%0.6f Threads: current=%d\n", network_time, thread_mgr->NumThreads())); + + const threading::Manager::msg_stats_list& thread_stats = thread_mgr->GetMsgThreadStats(); + for ( threading::Manager::msg_stats_list::const_iterator i = thread_stats.begin(); + i != thread_stats.end(); ++i ) + { + threading::MsgThread::Stats s = i->second; + file->Write(fmt(" %20s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", + i->first.c_str(), + s.sent_in, s.sent_out, + s.pending_in, s.pending_out)); + } + // Script-level state. unsigned int size, mem = 0; PDict(ID)* globals = global_scope()->Vars(); diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d963876755..2e8f6eb1fc 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -101,4 +101,21 @@ void Manager::Process() next_beat = network_time + HEART_BEAT_INTERVAL; } +const threading::Manager::msg_stats_list& threading::Manager::GetMsgThreadStats() + { + stats.clear(); + + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + { + MsgThread* t = *i; + + MsgThread::Stats s; + t->GetStats(&s); + + stats.push_back(std::make_pair(t->Name(),s)); + } + + return stats; + } + diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 2c4f88fa1e..d2f97209c9 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -43,6 +43,25 @@ public: */ void Terminate(); + typedef std::list > msg_stats_list; + + /** + * Returns statistics from all current MsgThread instances. + * + * @return A list of statistics, with one entry for each MsgThread. + * Each entry is a tuple of thread name and statistics. The list + * reference remains valid until the next call to this method (or + * termination of the manager). + */ + const msg_stats_list& GetMsgThreadStats(); + + /** + * Returns the number of currently active threads. This counts all + * threads that are not yet joined, includingt any potentially in + * Terminating() state. + */ + int NumThreads() const { return all_threads.size(); } + protected: friend class BasicThread; friend class MsgThread; @@ -96,6 +115,8 @@ private: bool did_process; // True if the last Process() found some work to do. double next_beat; // Timestamp when the next heartbeat will be sent. + + msg_stats_list stats; }; } From 29fc56105ddabcc3f29c9b3457c4b9d53f0bca5e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 1 Feb 2012 07:16:24 -0800 Subject: [PATCH 09/56] Documenting logging API. --- src/logging/Manager.cc | 2 +- src/logging/Manager.h | 204 ++++++++++++++++--- src/logging/WriterBackend.cc | 56 +++++- src/logging/WriterBackend.h | 357 ++++++++++++++++++++++------------ src/logging/WriterFrontend.cc | 2 +- src/logging/WriterFrontend.h | 178 ++++++++++++++--- src/logging/writers/Ascii.h | 4 +- src/logging/writers/None.h | 2 +- 8 files changed, 625 insertions(+), 180 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 09c5030fdc..0876b10eb2 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -1279,7 +1279,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, } WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, Field** fields) + int num_fields, const Field* const* fields) { Stream* stream = FindStream(id); diff --git a/src/logging/Manager.h b/src/logging/Manager.h index 7fa2c271db..1267a19ca7 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -15,32 +15,63 @@ class RotationTimer; namespace logging { -// Description of a log field. +/** + * Definition of a log file, i.e., one column of a log stream. + */ struct Field { - string name; - TypeTag type; - // inner type of sets - TypeTag subtype; + string name; //! Name of the field. + TypeTag type; //! Type of the field. + TypeTag subtype; //! Inner type for sets. + /** + * Constructor. + */ Field() { subtype = TYPE_VOID; } + + /** + * Copy constructor. + */ Field(const Field& other) : name(other.name), type(other.type), subtype(other.subtype) { } - // (Un-)serialize. + /** + * Unserializes a field. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ bool Read(SerializationFormat* fmt); + + /** + * Serializes a field. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ bool Write(SerializationFormat* fmt) const; }; -// Values as logged by a writer. +/** + * Definition of a log value, i.e., a entry logged by a stream. + * + * This struct essentialy represents a serialization of a Val instance (for + * those Vals supported). + */ struct Value { - TypeTag type; - bool present; // False for unset fields. + TypeTag type; //! The type of the value. + bool present; //! False for optional record fields that are not set. - // The following union is a subset of BroValUnion, including only the - // types we can log directly. struct set_t { bro_int_t size; Value** vals; }; typedef set_t vec_t; + /** + * This union is a subset of BroValUnion, including only the types we + * can log directly. See IsCompatibleType(). + */ union _val { bro_int_t int_val; bro_uint_t uint_val; @@ -52,42 +83,173 @@ struct Value { vec_t vector_val; } val; + /** + * Constructor. + * + * arg_type: The type of the value. + * + * arg_present: False if the value represents an optional record field + * that is not set. + */ Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) : type(arg_type), present(arg_present) {} + + /** + * Destructor. + */ ~Value(); - // (Un-)serialize. + /** + * Unserializes a value. + * + * @param fmt The serialization format to use. The format handles low-level I/O. + * + * @return False if an error occured. + */ bool Read(SerializationFormat* fmt); + + /** + * Serializes a value. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ bool Write(SerializationFormat* fmt) const; - // Returns true if the type can be logged the framework. If - // `atomic_only` is true, will not permit composite types. + /** + * Returns true if the type can be represented by a Value. If + * `atomic_only` is true, will not permit composite types. + */ static bool IsCompatibleType(BroType* t, bool atomic_only=false); private: - Value(const Value& other) { } + Value(const Value& other) { } // Disabled. }; class WriterBackend; class WriterFrontend; class RotationFinishedMessage; +/** + * Singleton class for managing log streams. + */ class Manager { public: + /** + * Constructor. + */ Manager(); + + /** + * Destructor. + */ ~Manager(); - // These correspond to the BiFs visible on the scripting layer. The - // actual BiFs just forward here. + /** + * Creates a new log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param stream A record of script type \c Log::Stream. + * + * This method corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool CreateStream(EnumVal* id, RecordVal* stream); + + /** + * Enables a log log stream. + * + * @param id The enum value corresponding the log stream. + * + * This method corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool EnableStream(EnumVal* id); + + /** + * Disables a log stream. + * + * @param id The enum value corresponding the log stream. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool DisableStream(EnumVal* id); + + /** + * Adds a filter to a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param filter A record of script type \c Log::Filter. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool AddFilter(EnumVal* id, RecordVal* filter); + + /** + * Removes a filter from a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param name The name of the filter to remove. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool RemoveFilter(EnumVal* id, StringVal* name); + + /** + * Removes a filter from a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param name The name of the filter to remove. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool RemoveFilter(EnumVal* id, string name); + + /** + * Write a record to a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param colums A record of the type defined for the stream's + * columns. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ bool Write(EnumVal* id, RecordVal* columns); - bool SetBuf(EnumVal* id, bool enabled); // Adjusts all writers. - bool Flush(EnumVal* id); // Flushes all writers.. + + /** + * Sets log streams buffering state. This adjusts all associated + * writers to the new state. + * + * @param id The enum value corresponding the log stream. + * + * @param enabled False to disable buffering (default is enabled). + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool SetBuf(EnumVal* id, bool enabled); + + /** + * Flushes a log stream. This flushed all associated writers. + * + * @param id The enum value corresponding the log stream. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool Flush(EnumVal* id); protected: friend class WriterFrontend; @@ -103,7 +265,7 @@ protected: // Takes ownership of fields. WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, Field** fields); + int num_fields, const Field* const* fields); // Takes ownership of values.. bool Write(EnumVal* id, EnumVal* writer, string path, @@ -112,8 +274,6 @@ protected: // Announces all instantiated writers to peer. void SendAllWritersTo(RemoteSerializer::PeerID peer); - //// Functions safe to use by writers. - // Signals that a file has been rotated. bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, double open, double close, bool terminating); diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 7c9c1d10ca..e361ca69d3 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -40,6 +40,15 @@ public: virtual bool Process() { Object()->FlushWriteBuffer(); return true; } }; +class DisableMessage : public threading::OutputMessage +{ +public: + DisableMessage(WriterFrontend* writer) + : threading::OutputMessage("Disable", writer) {} + + virtual bool Process() { Object()->SetDisable(); return true; } +}; + } // Backend methods. @@ -86,8 +95,13 @@ bool WriterBackend::FinishedRotation(string new_name, string old_name, return true; } +void WriterBackend::DisableFrontend() + { + SendOut(new DisableMessage(frontend)); + } + bool WriterBackend::Init(WriterFrontend* arg_frontend, string arg_path, int arg_num_fields, - const Field* const * arg_fields) + const Field* const* arg_fields) { frontend = arg_frontend; path = arg_path; @@ -95,7 +109,10 @@ bool WriterBackend::Init(WriterFrontend* arg_frontend, string arg_path, int arg_ fields = arg_fields; if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) + { + DisableFrontend(); return false; + } return true; } @@ -114,6 +131,7 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals) #endif DeleteVals(num_writes, vals); + DisableFrontend(); return false; } @@ -129,6 +147,7 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals) vals[j][i]->type, fields[i]->type); Debug(DBG_LOGGING, msg); + DisableFrontend(); DeleteVals(num_writes, vals); return false; } @@ -148,6 +167,9 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals) DeleteVals(num_writes, vals); + if ( ! success ) + DisableFrontend(); + return success; } @@ -159,23 +181,47 @@ bool WriterBackend::SetBuf(bool enabled) buffering = enabled; - return DoSetBuf(enabled); + if ( ! DoSetBuf(enabled) ) + { + DisableFrontend(); + return false; + } + + return true; } bool WriterBackend::Rotate(string rotated_path, double open, double close, bool terminating) { - return DoRotate(rotated_path, open, close, terminating); + if ( ! DoRotate(rotated_path, open, close, terminating) ) + { + DisableFrontend(); + return false; + } + + return true; } bool WriterBackend::Flush() { - return DoFlush(); + if ( ! DoFlush() ) + { + DisableFrontend(); + return false; + } + + return true; } bool WriterBackend::Finish() { - return DoFinish(); + if ( ! DoFlush() ) + { + DisableFrontend(); + return false; + } + + return true; } bool WriterBackend::DoHeartbeat(double network_time, double current_time) diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 27f4fe45a5..b5d313a480 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -11,180 +11,291 @@ namespace logging { -// The backend runs in its own thread, separate from the main process. +/** + * Base class for writer implementation. When the logging::Manager creates a + * new logging filter, it instantiates a WriterFrontend. That then in turn + * creates a WriterBackend of the right type. The frontend then forwards + * message over the backend as its methods are called. + * + * All of this methods must be called only from the corresponding child + * thread (the constructor is the one exception.) + */ class WriterBackend : public threading::MsgThread { public: + /** + * Constructor. + * + * @param name A descriptive name for writer's type (e.g., \c Ascii). + */ WriterBackend(const string& name); + + /** + * Destructor. + */ virtual ~WriterBackend(); - // One-time initialization of the writer to define the logged fields. - // - // "frontend" is the frontend writer that created this backend. The - // *only* purpose of this value is to be passed back via messages as - // a argument to callbacks. One must not otherwise access the - // frontend, it's running in a different thread. - // - // Interpretation of "path" is left to the writer, and will be - // corresponding the value configured on the script-level. - // - // Returns false if an error occured, in which case the writer must - // not be used further. - // - // The new instance takes ownership of "fields", and will delete them - // when done. - bool Init(WriterFrontend* frontend, string path, int num_fields, const Field* const * fields); + /** + * One-time initialization of the writer to define the logged fields. + * + * @param frontend The frontend writer that created this backend. The + * *only* purpose of this value is to be passed back via messages as + * a argument to callbacks. One must not otherwise access the + * frontend, it's running in a different thread. + * + * @param path A string left to the interpretation of the writer + * implementation; it corresponds to the value configured on the + * script-level for the logging filter. + * + * @param num_fields The number of log fields for the stream. + * + * @param fields An array of size \a num_fields with the log fields. + * The methods takes ownership of the array. + * + * @return False if an error occured. + */ + bool Init(WriterFrontend* frontend, string path, int num_fields, const Field* const* fields); - // Writes one log entry. The method takes ownership of "vals" and - // will return immediately after queueing the write request, which is - // potentially before output has actually been written out. - // - // num_fields and the types of the Values must match what was passed - // to Init(). - // - // Returns false if an error occured, in which case the writer must - // not be used any further. + /** + * Writes one log entry. + * + * @param num_fields: The number of log fields for this stream. The + * value must match what was passed to Init(). + * + * @param An array of size \a num_fields with the log values. Their + * types musst match with the field passed to Init(). The method + * takes ownership of \a vals.. + * + * Returns false if an error occured, in which case the writer must + * not be used any further. + * + * @return False if an error occured. + */ bool Write(int num_fields, int num_writes, Value*** vals); - // Sets the buffering status for the writer, if the writer supports - // that. (If not, it will be ignored). + /** + * Sets the buffering status for the writer, assuming the writer + * supports that. (If not, it will be ignored). + * + * @param enabled False if buffering is to be disabled (by default + * it's on). + * + * @return False if an error occured. + */ bool SetBuf(bool enabled); - // Flushes any currently buffered output, if the writer supports - // that. (If not, it will be ignored). + /** + * Flushes any currently buffered output, assuming the writer + * supports that. (If not, it will be ignored). + * + * @return False if an error occured. + */ bool Flush(); - // Triggers rotation, if the writer supports that. (If not, it will - // be ignored). + /** + * Triggers rotation, if the writer supports that. (If not, it will + * be ignored). + * + * @return False if an error occured. + */ bool Rotate(string rotated_path, double open, double close, bool terminating); - // Finishes writing to this logger regularly. Must not be called if - // an error has been indicated earlier. After calling this, no - // further writing must be performed. + /** + * Finishes writing to this logger in a regularl fashion. Must not be + * called if an error has been indicated earlier. After calling this, + * no further writing must be performed. + * + * @return False if an error occured. + */ bool Finish(); - //// Thread-safe methods that may be called from the writer - //// implementation. + /** + * Disables the frontend that has instantiated this backend. Once + * disabled,the frontend will not send any further message over. + */ + void DisableFrontend(); - // The following methods return the information as passed to Init(). + /** + * Returns the log path as passed into the constructor. + */ const string Path() const { return path; } + + /** + * Returns the number of log fields as passed into the constructor. + */ int NumFields() const { return num_fields; } + + /** + * Returns the log fields as passed into the constructor. + */ const Field* const * Fields() const { return fields; } - // Returns the current buffering state. + /** + * Returns the current buffering state. + * + * @return True if buffering is enabled. + */ bool IsBuf() { return buffering; } - // Signals to the log manager that a file has been rotated. - // - // writer: The frontend writer that triggered the rotation. This must - // be the value passed into DoRotate(). - // - // new_name: The filename of the rotated file. old_name: The filename - // of the origina file. - // - // open/close: The timestamps when the original file was opened and - // closed, respectively. - // - // terminating: True if rotation request occured due to the main Bro - // process shutting down. + /** + * Signals that a file has been rotated. This must be called by a + * writer's implementation of DoRotate() once rotation has finished. + * + * Most of the parameters should be passed through from DoRotate(). + * + * @param new_name The filename of the rotated file. + * + * @param old_name The filename of the original file. + * + * @param open: The timestamp when the original file was opened. + * + * @param close: The timestamp when the origina file was closed. + * + * @param terminating: True if the original rotation request occured + * due to the main Bro process shutting down. + */ bool FinishedRotation(string new_name, string old_name, double open, double close, bool terminating); protected: - // Methods for writers to override. If any of these returs false, it - // will be assumed that a fatal error has occured that prevents the - // writer from further operation. It will then be disabled and - // deleted. When returning false, the writer should also report the - // error via Error(). Note that even if a writer does not support the - // functionality for one these methods (like rotation), it must still - // return true if that is not to be considered a fatal error. - // - // Called once for initialization of the writer. + /** + * Writer-specific intialization method. + * + * A writer implementation must override this method. If it returns + * false, it will be assumed that a fatal error has occured that + * prevents the writer from further operation; it will then be + * disabled and eventually deleted. When returning false, an + * implementation should also call Error() to indicate what happened. + */ virtual bool DoInit(string path, int num_fields, - const Field* const * fields) = 0; + const Field* const* fields) = 0; - // Called once per log entry to record. - virtual bool DoWrite(int num_fields, const Field* const * fields, + /** + * Writer-specific output method implementing recording of fone log + * entry. + * + * A writer implementation must override this method. If it returns + * false, it will be assumed that a fatal error has occured that + * prevents the writer from further operation; it will then be + * disabled and eventually deleted. When returning false, an + * implementation should also call Error() to indicate what happened. + */ + virtual bool DoWrite(int num_fields, const Field* const* fields, Value** vals) = 0; - // Called when the buffering status for this writer is changed. If - // buffering is disabled, the writer should attempt to write out - // information as quickly as possible even if doing so may have a - // performance impact. If enabled (which is the default), it may - // buffer data as helpful and write it out later in a way optimized - // for performance. The current buffering state can be queried via - // IsBuf(). - // - // A writer may ignore buffering changes if it doesn't fit with its - // semantics (but must still return true in that case). + /** + * Writer-specific method implementing a change of fthe buffering + * state. If buffering is disabled, the writer should attempt to + * write out information as quickly as possible even if doing so may + * have a performance impact. If enabled (which is the default), it + * may buffer data as helpful and write it out later in a way + * optimized for performance. The current buffering state can be + * queried via IsBuf(). + * + * A writer implementation must override this method but it can just + * ignore calls if buffering doesn't align with its semantics. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + */ virtual bool DoSetBuf(bool enabled) = 0; - // Called to flush any currently buffered output. - // - // A writer may ignore flush requests if it doesn't fit with its - // semantics (but must still return true in that case). + /** + * Writer-specific method implementing flushing of its output. + * + * A writer implementation must override this method but it can just + * ignore calls if flushing doesn't align with its semantics. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + */ virtual bool DoFlush() = 0; - // Called when a log output is to be rotated. Most directly this only - // applies to writers writing into files, which should then close the - // current file and open a new one. However, a writer may also - // trigger other apppropiate actions if semantics are similar. - // - // Once rotation has finished, the implementation should call - // RotationDone() to signal the log manager that potential - // postprocessors can now run. - // - // "rotate_path" reflects the path to where the rotated output is to - // be moved, with specifics depending on the writer. It should - // generally be interpreted in a way consistent with that of "path" - // as passed into DoInit(). As an example, for file-based output, - // "rotate_path" could be the original filename extended with a - // timestamp indicating the time of the rotation. - // - // "open" and "close" are the network time's when the *current* file - // was opened and closed, respectively. - // - // "terminating" indicated whether the rotation request occurs due - // the main Bro prcoess terminating (and not because we've reach a - // regularly scheduled time for rotation). - // - // A writer may ignore rotation requests if it doesn't fit with its - // semantics (but must still return true in that case). + /** + * Writer-specific method implementing log rotation. Most directly + * this only applies to writers writing into files, which should then + * close the current file and open a new one. However, a writer may + * also trigger other apppropiate actions if semantics are similar. * + * Once rotation has finished, the implementation must call + * FinishedRotation() to signal the log manager that potential + * postprocessors can now run. + * + * A writer implementation must override this method but it can just + * ignore calls if flushing doesn't align with its semantics. It + * still needs to call FinishedRotation() though. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + * + * @param rotate_path Reflects the path to where the rotated output + * is to be moved, with specifics depending on the writer. It should + * generally be interpreted in a way consistent with that of \c path + * as passed into DoInit(). As an example, for file-based output, \c + * rotate_path could be the original filename extended with a + * timestamp indicating the time of the rotation. + * + * @param open The network time when the *current* file was opened. + * + * @param close The network time when the *current* file was closed. + * + * @param terminating Indicates whether the rotation request occurs + * due the main Bro prcoess terminating (and not because we've + * reached a regularly scheduled time for rotation). + */ virtual bool DoRotate(string rotated_path, double open, double close, bool terminating) = 0; - // Called once on termination. Not called when any of the other - // methods has previously signaled an error, i.e., executing this - // method signals a regular shutdown of the writer. + /** + * Writer-specific method implementing log output finalization at + * termination. Not called when any of the other methods has + * previously signaled an error, i.e., executing this method signals + * a regular shutdown of the writer. + * + * A writer implementation must override this method but it can just + * ignore calls if flushing doesn't align with its semantics. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + */ virtual bool DoFinish() = 0; - // Triggered by regular heartbeat messages from the main process. - // - // Note when overriding, you must call WriterBackend::DoHeartbeat(). + /** + * Triggered by regular heartbeat messages from the main thread. + * + * This method can be overridden but once must call + * WriterBackend::DoHeartbeat(). + */ virtual bool DoHeartbeat(double network_time, double current_time); private: friend class Manager; - // When an error occurs, we call this method to set a flag marking - // the writer as disabled. The Manager will check the flag later and - // remove the writer. - bool Disabled() { return disabled; } - - // Deletes the values as passed into Write(). + /** + * Deletes the values as passed into Write(). + */ void DeleteVals(int num_writes, Value*** vals); + // Frontend that instantiated us. This object must not be access from + // this class, it's running in a different thread! WriterFrontend* frontend; - string path; - int num_fields; - const Field* const * fields; - bool buffering; - bool disabled; - // For implementing Fmt(). - char* buf; - unsigned int buf_len; + string path; // Log path. + int num_fields; // Number of log fields. + const Field* const* fields; // Log fields. + bool buffering; // True if buffering is enabled. }; diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 2f7c1d6e7e..137cdf90ec 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -9,7 +9,7 @@ namespace logging { class InitMessage : public threading::InputMessage { public: - InitMessage(WriterBackend* backend, WriterFrontend* frontend, const string path, const int num_fields, const Field* const *fields) + InitMessage(WriterBackend* backend, WriterFrontend* frontend, const string path, const int num_fields, const Field* const* fields) : threading::InputMessage("Init", backend), path(path), num_fields(num_fields), fields(fields) { } diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index ed1a674842..6f1bb4ea1b 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -1,6 +1,4 @@ // See the file "COPYING" in the main distribution directory for copyright. -// -// Bridge class between main process and writer threads. #ifndef LOGGING_WRITERFRONTEND_H #define LOGGING_WRITERFRONTEND_H @@ -13,57 +11,187 @@ namespace logging { class WriterBackend; +/** + * Bridge class between the logging::Manager and backend writer threads. The + * Manager instantiates one \a WriterFrontend for each open logging filter. + * Each frontend in turns instantiates a WriterBackend-derived class + * internally that's specific to the particular output format. That backend + * spawns a new thread, and it receives messages from the frontend that + * correspond to method called by the manager. + * + */ class WriterFrontend { public: + /** + * Constructor. + * + * type: The backend writer type, with the value corresponding to the + * script-level \c Log::Writer enum (e.g., \a WRITER_ASCII). The + * frontend will internally instantiate a WriterBackend of the + * corresponding type. + * + * Frontends must only be instantiated by the main thread. + */ WriterFrontend(bro_int_t type); + + /** + * Destructor. + * + * Frontends must only be destroyed by the main thread. + */ virtual ~WriterFrontend(); - // Disables the writers and stop the backend thread. + /** + * Stops all output to this writer. Calling this methods disables all + * message forwarding to the backend and stops the backend thread. + * + * This method must only be called from the main thread. + */ void Stop(); - // Interface methods to interact with the writer from the main thread - // (and only from the main thread), typicalli from the log manager. - // All these methods forward (via inter-thread messaging) to the - // corresponding methods of an internally created WriterBackend. See - // there for documentation. - // - // If any of these operations fails, the writer will be automatically - // (but asynchronoulsy) disabled. + /** + * Initializes the writer. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See WriterBackend::Init() for arguments. The method takes + * ownership of \a fields. + * + * This method must only be called from the main thread. + */ + void Init(string path, int num_fields, const Field* const* fields); - void Init(string path, int num_fields, const Field* const * fields); + /** + * Write out a record. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * As an optimization, if buffering is enabled (which is the default) + * this method may buffer several writes and send them over to the + * backend in bulk with a single message. An explicit bulk write of + * all currently buffered data can be triggered with + * FlushWriteBuffer(). The backend writer triggers this with a + * message at every heartbeat. + * + * See WriterBackend::Writer() for arguments (except that this method + * takes only a single record, not an array). The method takes + * ownership of \a vals. + * + * This method must only be called from the main thread. + */ void Write(int num_fields, Value** vals); + + /** + * Sets the buffering state. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See WriterBackend::SetBuf() for arguments. + * + * This method must only be called from the main thread. + */ void SetBuf(bool enabled); + + /** + * Flushes the output. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. In addition, it also triggers + * FlushWriteBuffer(). If the backend method fails, it sends a + * message back that will asynchronously call Disable(). + * + * This method must only be called from the main thread. + */ void Flush(); - void FlushWriteBuffer(); + + /** + * Triggers log rotation. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See WriterBackend::Rotate() for arguments. + * + * This method must only be called from the main thread. + */ void Rotate(string rotated_path, double open, double close, bool terminating); + + /** + * Finalizes writing to this tream. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * This method must only be called from the main thread. + */ void Finish(); - // Calling this disable the writer. All methods calls will be no-ops - // from now on. The Manager will eventually remove disabled writers. + /** + * Explicitly triggers a transfer of all potentially buffered Write() + * operations over to the backend. + * + * This method must only be called from the main thread. + */ + void FlushWriteBuffer(); + + /** + * Disables the writer frontend. From now on, all method calls that + * would normally send message over to the backend, turn into no-ops. + * Note though that it does not stop the backend itself, use Stop() + * to do thast as well (this method is primarily for use as callback + * when the backend wants to disable the frontend). + * + * Disabled frontend will eventually be discarded by the + * logging::Manager. + * + * This method must only be called from the main thread. + */ void SetDisable() { disabled = true; } + + /** + * Returns true if the writer frontend has been disabled with SetDisable(). + */ bool Disabled() { return disabled; } + /** + * Returns the log path as passed into the constructor. + */ const string Path() const { return path; } + + /** + * Returns the number of log fields as passed into the constructor. + */ int NumFields() const { return num_fields; } + + /** + * Returns the log fields as passed into the constructor. + */ const Field* const * Fields() const { return fields; } protected: friend class Manager; - WriterBackend* backend; - bool disabled; - bool initialized; - bool buf; + WriterBackend* backend; // The backend we have instanatiated. + bool disabled; // True if disabled. + bool initialized; // True if initialized. + bool buf; // True if buffering is enabled (default). - string path; - int num_fields; - const Field* const * fields; + string path; // The log path. + int num_fields; // The number of log fields. + const Field* const* fields; // The log fields. // Buffer for bulk writes. static const int WRITER_BUFFER_SIZE = 50; - - int write_buffer_pos; - Value*** write_buffer; + int write_buffer_pos; // Position of next write in buffer. + Value*** write_buffer; // Buffer of size WRITER_BUFFER_SIZE. }; } diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 0c627c68e9..4a9dea4950 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -19,8 +19,8 @@ public: protected: virtual bool DoInit(string path, int num_fields, - const Field* const * fields); - virtual bool DoWrite(int num_fields, const Field* const * fields, + const Field* const* fields); + virtual bool DoWrite(int num_fields, const Field* const* fields, Value** vals); virtual bool DoSetBuf(bool enabled); virtual bool DoRotate(string rotated_path, double open, diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h index 9360ef44f6..b25bb09348 100644 --- a/src/logging/writers/None.h +++ b/src/logging/writers/None.h @@ -20,7 +20,7 @@ protected: virtual bool DoInit(string path, int num_fields, const Field* const * fields) { return true; } - virtual bool DoWrite(int num_fields, const Field* const * fields, + virtual bool DoWrite(int num_fields, const Field* const* fields, Value** vals) { return true; } virtual bool DoSetBuf(bool enabled) { return true; } virtual bool DoRotate(string rotated_path, double open, From ffb4094d365c61a9df8fea0d50ae375f06cc56b4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 3 Feb 2012 02:41:10 -0800 Subject: [PATCH 10/56] Bugfixes --- src/logging/Manager.cc | 8 ++++---- src/logging/Manager.h | 2 +- src/logging/WriterBackend.cc | 8 ++++---- src/logging/WriterBackend.h | 15 ++++++++------- src/logging/WriterFrontend.cc | 22 ++++++++++------------ src/logging/writers/Ascii.cc | 2 +- src/logging/writers/Ascii.h | 5 +++-- src/logging/writers/None.h | 5 +++-- src/threading/MsgThread.cc | 8 +------- 9 files changed, 35 insertions(+), 40 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 0876b10eb2..63d4c60a5c 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -21,7 +21,7 @@ struct WriterDefinition { bro_int_t type; // The type. const char *name; // Descriptive name for error messages. bool (*init)(); // An optional one-time initialization function. - WriterBackend* (*factory)(); // A factory function creating instances. + WriterBackend* (*factory)(WriterFrontend* frontend); // A factory function creating instances. }; // Static table defining all availabel log writers. @@ -30,7 +30,7 @@ WriterDefinition log_writers[] = { { BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate }, // End marker, don't touch. - { BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)())0 } + { BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)(WriterFrontend* frontend))0 } }; struct Manager::Filter { @@ -436,7 +436,7 @@ Manager::~Manager() delete *s; } -WriterBackend* Manager::CreateBackend(bro_int_t type) +WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) { WriterDefinition* ld = log_writers; @@ -478,7 +478,7 @@ WriterBackend* Manager::CreateBackend(bro_int_t type) assert(ld->factory); - WriterBackend* backend = (*ld->factory)(); + WriterBackend* backend = (*ld->factory)(frontend); assert(backend); return backend; } diff --git a/src/logging/Manager.h b/src/logging/Manager.h index 1267a19ca7..f6829b3554 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -259,7 +259,7 @@ protected: // Instantiates a new WriterBackend of the given type (note that // doing so creates a new thread!). - WriterBackend* CreateBackend(bro_int_t type); + WriterBackend* CreateBackend(WriterFrontend* frontend, bro_int_t type); //// Function also used by the RemoteSerializer. diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index e361ca69d3..fe3a6ef560 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -55,12 +55,13 @@ public: using namespace logging; -WriterBackend::WriterBackend(const string& name) : MsgThread(name) +WriterBackend::WriterBackend(WriterFrontend* arg_frontend, const string& name) : MsgThread(name) { path = ""; num_fields = 0; fields = 0; buffering = true; + frontend = arg_frontend; } WriterBackend::~WriterBackend() @@ -100,10 +101,8 @@ void WriterBackend::DisableFrontend() SendOut(new DisableMessage(frontend)); } -bool WriterBackend::Init(WriterFrontend* arg_frontend, string arg_path, int arg_num_fields, - const Field* const* arg_fields) +bool WriterBackend::Init(string arg_path, int arg_num_fields, const Field* const* arg_fields) { - frontend = arg_frontend; path = arg_path; num_fields = arg_num_fields; fields = arg_fields; @@ -227,6 +226,7 @@ bool WriterBackend::Finish() bool WriterBackend::DoHeartbeat(double network_time, double current_time) { SendOut(new FlushWriteBufferMessage(frontend)); + return true; } diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index b5d313a480..21dcd41ff7 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -26,9 +26,15 @@ public: /** * Constructor. * + * @param frontend The frontend writer that created this backend. The + * *only* purpose of this value is to be passed back via messages as + * a argument to callbacks. One must not otherwise access the + * frontend, it's running in a different thread. + * * @param name A descriptive name for writer's type (e.g., \c Ascii). + * */ - WriterBackend(const string& name); + WriterBackend(WriterFrontend* frontend, const string& name); /** * Destructor. @@ -38,11 +44,6 @@ public: /** * One-time initialization of the writer to define the logged fields. * - * @param frontend The frontend writer that created this backend. The - * *only* purpose of this value is to be passed back via messages as - * a argument to callbacks. One must not otherwise access the - * frontend, it's running in a different thread. - * * @param path A string left to the interpretation of the writer * implementation; it corresponds to the value configured on the * script-level for the logging filter. @@ -54,7 +55,7 @@ public: * * @return False if an error occured. */ - bool Init(WriterFrontend* frontend, string path, int num_fields, const Field* const* fields); + bool Init(string path, int num_fields, const Field* const* fields); /** * Writes one log entry. diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 137cdf90ec..79c180b749 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -9,14 +9,13 @@ namespace logging { class InitMessage : public threading::InputMessage { public: - InitMessage(WriterBackend* backend, WriterFrontend* frontend, const string path, const int num_fields, const Field* const* fields) + InitMessage(WriterBackend* backend, const string path, const int num_fields, const Field* const* fields) : threading::InputMessage("Init", backend), path(path), num_fields(num_fields), fields(fields) { } - virtual bool Process() { return Object()->Init(frontend, path, num_fields, fields); } + virtual bool Process() { return Object()->Init(path, num_fields, fields); } private: - WriterFrontend* frontend; const string path; const int num_fields; const Field * const* fields; @@ -47,7 +46,7 @@ class WriteMessage : public threading::InputMessage public: WriteMessage(WriterBackend* backend, int num_fields, int num_writes, Value*** vals) : threading::InputMessage("Write", backend), - num_fields(num_fields), vals(vals) {} + num_fields(num_fields), num_writes(num_writes), vals(vals) {} virtual bool Process() { return Object()->Write(num_fields, num_writes, vals); } @@ -98,8 +97,9 @@ WriterFrontend::WriterFrontend(bro_int_t type) { disabled = initialized = false; buf = true; + write_buffer = 0; write_buffer_pos = 0; - backend = log_mgr->CreateBackend(type); + backend = log_mgr->CreateBackend(this, type); assert(backend); backend->Start(); @@ -129,7 +129,7 @@ void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* cons fields = arg_fields; initialized = true; - backend->SendIn(new InitMessage(backend, this, arg_path, arg_num_fields, arg_fields)); + backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); } void WriterFrontend::Write(int num_fields, Value** vals) @@ -144,15 +144,12 @@ void WriterFrontend::Write(int num_fields, Value** vals) write_buffer_pos = 0; } - if ( write_buffer_pos >= WRITER_BUFFER_SIZE ) - // Buffer full. - FlushWriteBuffer(); - write_buffer[write_buffer_pos++] = vals; - if ( ! buf ) - // Send out immediately if we don't want buffering. + if ( write_buffer_pos >= WRITER_BUFFER_SIZE || ! buf ) + // Buffer full (or no bufferin desired). FlushWriteBuffer(); + } void WriterFrontend::FlushWriteBuffer() @@ -165,6 +162,7 @@ void WriterFrontend::FlushWriteBuffer() // Clear buffer (no delete, we pass ownership to child thread.) write_buffer = 0; + write_buffer_pos = 0; } void WriterFrontend::SetBuf(bool enabled) diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index a1ceb6e217..0e522dde1d 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -10,7 +10,7 @@ using namespace logging; using namespace writer; -Ascii::Ascii() : WriterBackend("Ascii") +Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend, "Ascii") { file = 0; diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 4a9dea4950..4a24aad9b7 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -11,10 +11,11 @@ namespace logging { namespace writer { class Ascii : public WriterBackend { public: - Ascii(); + Ascii(WriterFrontend* frontend); ~Ascii(); - static WriterBackend* Instantiate() { return new Ascii; } + static WriterBackend* Instantiate(WriterFrontend* frontend) + { return new Ascii(frontend); } static string LogExt(); protected: diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h index b25bb09348..19dc685ecb 100644 --- a/src/logging/writers/None.h +++ b/src/logging/writers/None.h @@ -11,10 +11,11 @@ namespace logging { namespace writer { class None : public WriterBackend { public: - None() : WriterBackend("None") {} + None(WriterFrontend* frontend) : WriterBackend(frontend, "None") {} ~None() {}; - static WriterBackend* Instantiate() { return new None; } + static WriterBackend* Instantiate(WriterFrontend* frontend) + { return new None(frontend); } protected: virtual bool DoInit(string path, int num_fields, diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 455c177df6..52da7c7400 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -214,13 +214,7 @@ BasicOutputMessage* MsgThread::RetrieveOut() BasicOutputMessage* msg = queue_out.Get(); assert(msg); -#ifdef DEBUG - if ( msg->Name() != "DebugMessage" ) // Avoid recursion. - { - string s = Fmt("Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str()); - Debug(DBG_THREADING, s.c_str()); - } -#endif + DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str()); return msg; } From 4879cb7b0ddcba2111deebb688d18a5c9cd9c8af Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 3 Feb 2012 03:03:38 -0800 Subject: [PATCH 11/56] Improved signal handling. Sending SIGTERM triggers a normal shutdown of all threads that waits until they have processed their remaining data. However, sending a 2nd SIGTERM while waiting for them to finish will immediately kill them all. --- src/main.cc | 7 +++++++ src/threading/BasicThread.cc | 11 +++++++++++ src/threading/BasicThread.h | 9 +++++++++ src/threading/Manager.cc | 13 +++++++++++++ src/threading/Manager.h | 16 ++++++++++++++++ 5 files changed, 56 insertions(+) diff --git a/src/main.cc b/src/main.cc index 58a23e6c80..e224910db4 100644 --- a/src/main.cc +++ b/src/main.cc @@ -333,6 +333,13 @@ RETSIGTYPE sig_handler(int signo) { set_processing_status("TERMINATING", "sig_handler"); signal_val = signo; + + if ( thread_mgr->Terminating() && (signal_val == SIGTERM || signal_val == SIGINT) ) + // If the thread manager is already terminating (i.e., + // waiting for child threads to exit), another term signal + // will send the threads a kill. + thread_mgr->KillThreads(); + return RETSIGVAL; } diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index f7bd2afbcd..73dc562b31 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -113,6 +113,17 @@ void BasicThread::Join() pthread = 0; } +void BasicThread::Kill() + { + if ( ! (started && pthread) ) + return; + + // I believe this is safe to call from a signal handler ... Not error + // checking so that killing doesn't bail out if we have already + // terminated. + pthread_kill(pthread, SIGKILL); + } + void* BasicThread::launcher(void *arg) { BasicThread* thread = (BasicThread *)arg; diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index df5665c464..aeafc61c52 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -121,6 +121,15 @@ protected: */ void Join(); + /** + * Kills the thread immediately. One still needs to call Join() + * afterwards. + * + * This is called from the threading::Manager and safe to execute + * during a signal handler. + */ + void Kill(); + private: // pthread entry function. static void* launcher(void *arg); diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 2e8f6eb1fc..d07311bbe8 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -9,6 +9,7 @@ Manager::Manager() did_process = false; next_beat = 0; + terminating = false; } Manager::~Manager() @@ -21,6 +22,8 @@ void Manager::Terminate() { DBG_LOG(DBG_THREADING, "Terminating thread manager ..."); + terminating = true; + // First process remaining thread output for the message threads. do Process(); while ( did_process ); @@ -37,6 +40,16 @@ void Manager::Terminate() all_threads.clear(); msg_threads.clear(); + + terminating = false; + } + +void Manager::KillThreads() + { + DBG_LOG(DBG_THREADING, "Killing threads ..."); + + for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ ) + (*i)->Kill(); } void Manager::AddThread(BasicThread* thread) diff --git a/src/threading/Manager.h b/src/threading/Manager.h index d2f97209c9..7d9ba766d4 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -43,6 +43,21 @@ public: */ void Terminate(); + /** + * Returns True if we are currently in Terminate() waiting for + * threads to exit. + */ + bool Terminating() const { return terminating; } + + /** + * Immediately kills all child threads. It does however not yet join + * them, one still needs to call Terminate() for that. + * + * This method is safe to call from a signal handler, and can in fact + * be called while Terminate() is already in progress. + */ + void KillThreads(); + typedef std::list > msg_stats_list; /** @@ -115,6 +130,7 @@ private: bool did_process; // True if the last Process() found some work to do. double next_beat; // Timestamp when the next heartbeat will be sent. + bool terminating; // True if we are in Terminate(). msg_stats_list stats; }; From cf6a346b865fc95275eee83d25656fe95f7bafe3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 3 Feb 2012 03:27:25 -0800 Subject: [PATCH 12/56] Fixing prof.log output. The queue Size() method was not yet atomic. --- src/Stats.cc | 3 ++- src/threading/MsgThread.cc | 1 + src/threading/Queue.h | 17 ++++++++++------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Stats.cc b/src/Stats.cc index 05ce33daed..27b433c9ee 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -224,7 +224,8 @@ void ProfileLogger::Log() i != thread_stats.end(); ++i ) { threading::MsgThread::Stats s = i->second; - file->Write(fmt(" %20s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", + file->Write(fmt("%0.6f %-15s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", + network_time, i->first.c_str(), s.sent_in, s.sent_out, s.pending_in, s.pending_out)); diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 52da7c7400..1bda8943da 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -206,6 +206,7 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force) return; queue_out.Put(msg); + ++cnt_sent_out; } diff --git a/src/threading/Queue.h b/src/threading/Queue.h index add7019f9c..a25f897d23 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -67,7 +67,6 @@ private: int read_ptr; // Where the next operation will read from int write_ptr; // Where the next operation will write to - uint64_t size; // Current queue size. }; inline static void safe_lock(pthread_mutex_t* mutex) @@ -120,7 +119,6 @@ inline T Queue::Get() T data = messages[read_ptr].front(); messages[read_ptr].pop(); - --size; read_ptr = (read_ptr + 1) % NUM_QUEUES; @@ -139,7 +137,6 @@ inline void Queue::Put(T data) bool need_signal = messages[write_ptr].empty(); messages[write_ptr].push(data); - ++size; if ( need_signal ) pthread_cond_signal(&has_data[write_ptr]); @@ -165,13 +162,19 @@ inline bool Queue::Ready() template inline uint64_t Queue::Size() { - safe_lock(&mutex[read_ptr]); + // Need to lock all queues. + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_lock(&mutex[i]); - uint64_t s = size; + uint64_t size = 0; - safe_unlock(&mutex[read_ptr]); + for ( int i = 0; i < NUM_QUEUES; i++ ) + size += messages[i].size(); - return s; + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_unlock(&mutex[i]); + + return size; } } From 70fe7876a176e15bd0c8046dab939ad0f7fbde90 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 3 Feb 2012 04:01:24 -0800 Subject: [PATCH 13/56] Updating thread naming. Also includes experimental code to adapt the thread name as shown by top, but it's untested. --- src/Stats.cc | 2 +- src/logging/Manager.cc | 2 ++ src/logging/WriterBackend.cc | 6 +++++- src/logging/WriterBackend.h | 2 +- src/logging/WriterFrontend.cc | 9 +++++++++ src/logging/WriterFrontend.h | 12 ++++++++++++ src/logging/writers/Ascii.cc | 2 +- src/logging/writers/None.h | 2 +- src/threading/BasicThread.cc | 25 +++++++++++++++++++++++-- src/threading/BasicThread.h | 21 ++++++++++++++++++--- src/threading/MsgThread.cc | 8 +++++++- src/threading/MsgThread.h | 4 +--- 12 files changed, 81 insertions(+), 14 deletions(-) diff --git a/src/Stats.cc b/src/Stats.cc index 27b433c9ee..f418e9cccc 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -224,7 +224,7 @@ void ProfileLogger::Log() i != thread_stats.end(); ++i ) { threading::MsgThread::Stats s = i->second; - file->Write(fmt("%0.6f %-15s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", + file->Write(fmt("%0.6f %-25s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", network_time, i->first.c_str(), s.sent_in, s.sent_out, diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 63d4c60a5c..6d53ea363f 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -480,6 +480,8 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) WriterBackend* backend = (*ld->factory)(frontend); assert(backend); + + frontend->ty_name = ld->name; return backend; } diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index fe3a6ef560..4d2e497b14 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -55,13 +55,15 @@ public: using namespace logging; -WriterBackend::WriterBackend(WriterFrontend* arg_frontend, const string& name) : MsgThread(name) +WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread() { path = ""; num_fields = 0; fields = 0; buffering = true; frontend = arg_frontend; + + SetName(frontend->Name()); } WriterBackend::~WriterBackend() @@ -107,6 +109,8 @@ bool WriterBackend::Init(string arg_path, int arg_num_fields, const Field* const num_fields = arg_num_fields; fields = arg_fields; + SetName(frontend->Name()); + if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) { DisableFrontend(); diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 21dcd41ff7..33271e43f9 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -34,7 +34,7 @@ public: * @param name A descriptive name for writer's type (e.g., \c Ascii). * */ - WriterBackend(WriterFrontend* frontend, const string& name); + WriterBackend(WriterFrontend* frontend); /** * Destructor. diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 79c180b749..79278870f9 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -99,6 +99,7 @@ WriterFrontend::WriterFrontend(bro_int_t type) buf = true; write_buffer = 0; write_buffer_pos = 0; + ty_name = ""; backend = log_mgr->CreateBackend(this, type); assert(backend); @@ -109,6 +110,14 @@ WriterFrontend::~WriterFrontend() { } +string WriterFrontend::Name() const + { + if ( path.size() ) + return ty_name; + + return ty_name + "/" + path; + } + void WriterFrontend::Stop() { FlushWriteBuffer(); diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index 6f1bb4ea1b..e0bc590dfc 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -30,6 +30,9 @@ public: * frontend will internally instantiate a WriterBackend of the * corresponding type. * + * name: A descriptive name for the backend wroter type (e.g., \c + * Ascii). + * * Frontends must only be instantiated by the main thread. */ WriterFrontend(bro_int_t type); @@ -171,6 +174,14 @@ public: */ int NumFields() const { return num_fields; } + /** + * Returns a descriptive name for the writer, including the type of + * the backend and the path used. + * + * This method is safe to call from any thread. + */ + string Name() const; + /** * Returns the log fields as passed into the constructor. */ @@ -184,6 +195,7 @@ protected: bool initialized; // True if initialized. bool buf; // True if buffering is enabled (default). + string ty_name; // Name of the backend type. Set by the manager. string path; // The log path. int num_fields; // The number of log fields. const Field* const* fields; // The log fields. diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 0e522dde1d..7cc8459e68 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -10,7 +10,7 @@ using namespace logging; using namespace writer; -Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend, "Ascii") +Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) { file = 0; diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h index 19dc685ecb..6a62161f49 100644 --- a/src/logging/writers/None.h +++ b/src/logging/writers/None.h @@ -11,7 +11,7 @@ namespace logging { namespace writer { class None : public WriterBackend { public: - None(WriterFrontend* frontend) : WriterBackend(frontend, "None") {} + None(WriterFrontend* frontend) : WriterBackend(frontend) {} ~None() {}; static WriterBackend* Instantiate(WriterFrontend* frontend) diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 73dc562b31..4d51c3c4e4 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -9,7 +9,7 @@ using namespace threading; uint64_t BasicThread::thread_counter = 0; -BasicThread::BasicThread(const string& arg_name) +BasicThread::BasicThread() { started = false; terminating = false; @@ -18,7 +18,7 @@ BasicThread::BasicThread(const string& arg_name) buf = 0; buf_len = 1024; - name = Fmt("%s@%d", arg_name.c_str(), ++thread_counter); + name = Fmt("thread-%d", ++thread_counter); thread_mgr->AddThread(this); } @@ -27,6 +27,27 @@ BasicThread::~BasicThread() { } +void BasicThread::SetName(const string& arg_name) + { + // Slight race condition here with reader threads, but shouldn't matter. + name = arg_name; + } + +void BasicThread::SetOSName(const string& name) + { +#ifdef LINUX + pthread_setname_np(pthread_self(), name.c_str()); +#endif + +#ifdef __APPLE__ + pthread_setname_np(name.c_str()); +#endif + +#ifdef FREEBSD + pthread_set_name_np(pthread_self(), name, name.c_str()); +#endif + } + const char* BasicThread::Fmt(const char* format, ...) { if ( ! buf ) diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index aeafc61c52..6d2f739620 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -34,16 +34,31 @@ public: * @param name A descriptive name for thread the thread. This may * show up in messages to the user. */ - BasicThread(const string& name); // Managed by manager, must not delete otherwise. + BasicThread(); /** - * Returns a descriptive name for the thread. This is the name passed - * into the constructor. + * Returns a descriptive name for the thread. If not set via + * SetName(). If not set, a default name is choosen automatically. * * This method is safe to call from any thread. */ const string& Name() const { return name; } + /** + * Sets a descriptive name for the thread. This should be a string + * that's useful in output presented to the user and uniquely + * identifies the thread. + * + * This method must be called only from the thread itself. + */ + void SetName(const string& name); + + /** + * Set the name shown by the OS as the thread's description. Not + * supported on all OSs. + */ + void SetOSName(const string& name); + /** * Starts the thread. Calling this methods will spawn a new OS thread * executing Run(). Note that one can't restart a thread after a diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 1bda8943da..f41b20ddf9 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -127,7 +127,7 @@ bool ReporterMessage::Process() return true; } -MsgThread::MsgThread(const string& name) : BasicThread(name) +MsgThread::MsgThread() : BasicThread() { cnt_sent_in = cnt_sent_out = 0; thread_mgr->AddMsgThread(this); @@ -142,6 +142,12 @@ void MsgThread::OnStop() void MsgThread::Heartbeat() { SendIn(new HeartbeatMessage(this, network_time, current_time())); + + string name = Fmt("%s (%d/%d)", name.c_str(), + cnt_sent_in - queue_in.Size(), + cnt_sent_out - queue_out.Size()); + + SetOSName(name.c_str()); } void MsgThread::Info(const char* msg) diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index ec249e90ad..459ac6c603 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -34,10 +34,8 @@ public: * threading::Manager. * * Only Bro's main thread may instantiate a new thread. - * - * @param name A descriptive name. This is passed on to BasicThread(). */ - MsgThread(const string& name); + MsgThread(); /** * Sends a message to the child thread. The message will be proceesed From a0487ecb30e365956525aa6eb7cd1703d680b138 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 3 Feb 2012 14:12:29 -0800 Subject: [PATCH 14/56] move Value and Field from the logging namespace to the threading namespace, because other modules using threading will need them. --- src/Attr.cc | 4 +- src/CMakeLists.txt | 1 + src/RemoteSerializer.cc | 21 +- src/RemoteSerializer.h | 10 +- src/logging/Manager.cc | 328 +--------------------------- src/logging/Manager.h | 124 +---------- src/logging/WriterBackend.cc | 4 + src/logging/WriterBackend.h | 16 +- src/logging/WriterFrontend.cc | 4 + src/logging/WriterFrontend.h | 10 +- src/logging/writers/Ascii.cc | 3 + src/logging/writers/Ascii.h | 8 +- src/logging/writers/None.h | 6 +- src/threading/SerializationTypes.cc | 319 +++++++++++++++++++++++++++ src/threading/SerializationTypes.h | 126 +++++++++++ 15 files changed, 512 insertions(+), 472 deletions(-) create mode 100644 src/threading/SerializationTypes.cc create mode 100644 src/threading/SerializationTypes.h diff --git a/src/Attr.cc b/src/Attr.cc index b877250f52..40c6c1a75c 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -5,7 +5,7 @@ #include "Attr.h" #include "Expr.h" #include "Serializer.h" -#include "logging/Manager.h" +#include "threading/SerializationTypes.h" const char* attr_name(attr_tag t) { @@ -416,7 +416,7 @@ void Attributes::CheckAttr(Attr* a) break; case ATTR_LOG: - if ( ! logging::Value::IsCompatibleType(type) ) + if ( ! threading::Value::IsCompatibleType(type) ) Error("&log applied to a type that cannot be logged"); break; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 61a4847b70..7a3cc4babf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -411,6 +411,7 @@ set(bro_SRCS threading/BasicThread.cc threading/Manager.cc threading/MsgThread.cc + threading/SerializationTypes.cc logging/Manager.cc logging/WriterBackend.cc diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index a75812b42b..ba2598c018 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -184,6 +184,7 @@ #include "File.h" #include "Conn.h" #include "Reporter.h" +#include "threading/SerializationTypes.h" #include "logging/Manager.h" extern "C" { @@ -2476,7 +2477,7 @@ bool RemoteSerializer::ProcessRemotePrint() return true; } -bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields) +bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields) { loop_over_list(peers, i) { @@ -2486,7 +2487,7 @@ bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string return true; } -bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields) +bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields) { SetErrorDescr("logging"); @@ -2540,7 +2541,7 @@ error: return false; } -bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals) +bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals) { loop_over_list(peers, i) { @@ -2550,7 +2551,7 @@ bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, i return true; } -bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals) +bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals) { if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) return false; @@ -2641,7 +2642,7 @@ bool RemoteSerializer::ProcessLogCreateWriter() EnumVal* id_val = 0; EnumVal* writer_val = 0; - logging::Field** fields = 0; + threading::Field** fields = 0; BinarySerializationFormat fmt; fmt.StartRead(current_args->data, current_args->len); @@ -2658,11 +2659,11 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( ! success ) goto error; - fields = new logging::Field* [num_fields]; + fields = new threading::Field* [num_fields]; for ( int i = 0; i < num_fields; i++ ) { - fields[i] = new logging::Field; + fields[i] = new threading::Field; if ( ! fields[i]->Read(&fmt) ) goto error; } @@ -2703,7 +2704,7 @@ bool RemoteSerializer::ProcessLogWrite() // Unserialize one entry. EnumVal* id_val = 0; EnumVal* writer_val = 0; - logging::Value** vals = 0; + threading::Value** vals = 0; int id, writer; string path; @@ -2717,11 +2718,11 @@ bool RemoteSerializer::ProcessLogWrite() if ( ! success ) goto error; - vals = new logging::Value* [num_fields]; + vals = new threading::Value* [num_fields]; for ( int i = 0; i < num_fields; i++ ) { - vals[i] = new logging::Value; + vals[i] = new threading::Value; if ( ! vals[i]->Read(&fmt) ) goto error; } diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index ba0bde7d41..571fa72d39 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -15,7 +15,7 @@ class IncrementalSendTimer; -namespace logging { +namespace threading { class Field; class Value; } @@ -102,13 +102,13 @@ public: bool SendPrintHookEvent(BroFile* f, const char* txt, size_t len); // Send a request to create a writer on a remote side. - bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields); + bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields); // Broadcasts a request to create a writer. - bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Field* const * fields); + bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields); // Broadcast a log entry to everybody interested. - bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals); + bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals); // Synchronzizes time with all connected peers. Returns number of // current sync-point, or -1 on error. @@ -303,7 +303,7 @@ protected: bool SendID(SerialInfo* info, Peer* peer, const ID& id); bool SendCapabilities(Peer* peer); bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p); - bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const logging::Value* const * vals); + bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals); void UnregisterHandlers(Peer* peer); void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0); diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 6d53ea363f..2333d6c612 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -14,7 +14,11 @@ #include "writers/Ascii.h" #include "writers/None.h" +#include "threading/SerializationTypes.h" + using namespace logging; +using threading::Value; +using threading::Field; // Structure describing a log writer type. struct WriterDefinition { @@ -83,316 +87,6 @@ struct Manager::Stream { ~Stream(); }; -bool Field::Read(SerializationFormat* fmt) - { - int t; - int st; - - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); - type = (TypeTag) t; - subtype = (TypeTag) st; - - return success; - } - -bool Field::Write(SerializationFormat* fmt) const - { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); - } - -Value::~Value() - { - if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC) - && present ) - delete val.string_val; - - if ( type == TYPE_TABLE && present ) - { - for ( int i = 0; i < val.set_val.size; i++ ) - delete val.set_val.vals[i]; - - delete [] val.set_val.vals; - } - - if ( type == TYPE_VECTOR && present ) - { - for ( int i = 0; i < val.vector_val.size; i++ ) - delete val.vector_val.vals[i]; - - delete [] val.vector_val.vals; - } - } - -bool Value::IsCompatibleType(BroType* t, bool atomic_only) - { - if ( ! t ) - return false; - - switch ( t->Tag() ) { - case TYPE_BOOL: - case TYPE_INT: - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - case TYPE_SUBNET: - case TYPE_ADDR: - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - return true; - - case TYPE_RECORD: - return ! atomic_only; - - case TYPE_TABLE: - { - if ( atomic_only ) - return false; - - if ( ! t->IsSet() ) - return false; - - return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); - } - - case TYPE_VECTOR: - { - if ( atomic_only ) - return false; - - return IsCompatibleType(t->AsVectorType()->YieldType(), true); - } - - default: - return false; - } - - return false; - } - -bool Value::Read(SerializationFormat* fmt) - { - int ty; - - if ( ! (fmt->Read(&ty, "type") && fmt->Read(&present, "present")) ) - return false; - - type = (TypeTag)(ty); - - if ( ! present ) - return true; - - switch ( type ) { - case TYPE_BOOL: - case TYPE_INT: - return fmt->Read(&val.int_val, "int"); - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - return fmt->Read(&val.uint_val, "uint"); - - case TYPE_SUBNET: - { - uint32 net[4]; - if ( ! (fmt->Read(&net[0], "net0") && - fmt->Read(&net[1], "net1") && - fmt->Read(&net[2], "net2") && - fmt->Read(&net[3], "net3") && - fmt->Read(&val.subnet_val.width, "width")) ) - return false; - -#ifdef BROv6 - val.subnet_val.net[0] = net[0]; - val.subnet_val.net[1] = net[1]; - val.subnet_val.net[2] = net[2]; - val.subnet_val.net[3] = net[3]; -#else - val.subnet_val.net = net[0]; -#endif - return true; - } - - case TYPE_ADDR: - { - uint32 addr[4]; - if ( ! (fmt->Read(&addr[0], "addr0") && - fmt->Read(&addr[1], "addr1") && - fmt->Read(&addr[2], "addr2") && - fmt->Read(&addr[3], "addr3")) ) - return false; - - val.addr_val[0] = addr[0]; -#ifdef BROv6 - val.addr_val[1] = addr[1]; - val.addr_val[2] = addr[2]; - val.addr_val[3] = addr[3]; -#endif - return true; - } - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - return fmt->Read(&val.double_val, "double"); - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - { - val.string_val = new string; - return fmt->Read(val.string_val, "string"); - } - - case TYPE_TABLE: - { - if ( ! fmt->Read(&val.set_val.size, "set_size") ) - return false; - - val.set_val.vals = new Value* [val.set_val.size]; - - for ( int i = 0; i < val.set_val.size; ++i ) - { - val.set_val.vals[i] = new Value; - - if ( ! val.set_val.vals[i]->Read(fmt) ) - return false; - } - - return true; - } - - case TYPE_VECTOR: - { - if ( ! fmt->Read(&val.vector_val.size, "vector_size") ) - return false; - - val.vector_val.vals = new Value* [val.vector_val.size]; - - for ( int i = 0; i < val.vector_val.size; ++i ) - { - val.vector_val.vals[i] = new Value; - - if ( ! val.vector_val.vals[i]->Read(fmt) ) - return false; - } - - return true; - } - - default: - reporter->InternalError("unsupported type %s in Value::Write", type_name(type)); - } - - return false; - } - -bool Value::Write(SerializationFormat* fmt) const - { - if ( ! (fmt->Write((int)type, "type") && - fmt->Write(present, "present")) ) - return false; - - if ( ! present ) - return true; - - switch ( type ) { - case TYPE_BOOL: - case TYPE_INT: - return fmt->Write(val.int_val, "int"); - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - return fmt->Write(val.uint_val, "uint"); - - case TYPE_SUBNET: - { - uint32 net[4]; -#ifdef BROv6 - net[0] = val.subnet_val.net[0]; - net[1] = val.subnet_val.net[1]; - net[2] = val.subnet_val.net[2]; - net[3] = val.subnet_val.net[3]; -#else - net[0] = val.subnet_val.net; - net[1] = net[2] = net[3] = 0; -#endif - return fmt->Write(net[0], "net0") && - fmt->Write(net[1], "net1") && - fmt->Write(net[2], "net2") && - fmt->Write(net[3], "net3") && - fmt->Write(val.subnet_val.width, "width"); - } - - case TYPE_ADDR: - { - uint32 addr[4]; - addr[0] = val.addr_val[0]; -#ifdef BROv6 - addr[1] = val.addr_val[1]; - addr[2] = val.addr_val[2]; - addr[3] = val.addr_val[3]; -#else - addr[1] = addr[2] = addr[3] = 0; -#endif - return fmt->Write(addr[0], "addr0") && - fmt->Write(addr[1], "addr1") && - fmt->Write(addr[2], "addr2") && - fmt->Write(addr[3], "addr3"); - } - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - return fmt->Write(val.double_val, "double"); - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - return fmt->Write(*val.string_val, "string"); - - case TYPE_TABLE: - { - if ( ! fmt->Write(val.set_val.size, "set_size") ) - return false; - - for ( int i = 0; i < val.set_val.size; ++i ) - { - if ( ! val.set_val.vals[i]->Write(fmt) ) - return false; - } - - return true; - } - - case TYPE_VECTOR: - { - if ( ! fmt->Write(val.vector_val.size, "vector_size") ) - return false; - - for ( int i = 0; i < val.vector_val.size; ++i ) - { - if ( ! val.vector_val.vals[i]->Write(fmt) ) - return false; - } - - return true; - } - - default: - reporter->InternalError("unsupported type %s in Value::REad", type_name(type)); - } - - return false; - } - Manager::Filter::~Filter() { for ( int i = 0; i < num_fields; ++i ) @@ -552,7 +246,7 @@ bool Manager::CreateStream(EnumVal* id, RecordVal* sval) if ( ! (columns->FieldDecl(i)->FindAttr(ATTR_LOG)) ) continue; - if ( ! Value::IsCompatibleType(columns->FieldType(i)) ) + if ( ! threading::Value::IsCompatibleType(columns->FieldType(i)) ) { reporter->Error("type of field '%s' is not support for logging output", columns->FieldName(i)); @@ -1089,7 +783,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) if ( filter->local || filter->remote ) { - Value** vals = RecordToFilterVals(stream, filter, columns); + threading::Value** vals = RecordToFilterVals(stream, filter, columns); if ( filter->remote ) remote_serializer->SendLogWrite(stream->id, @@ -1125,15 +819,15 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) return true; } -Value* Manager::ValToLogVal(Val* val, BroType* ty) +threading::Value* Manager::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) ty = val->Type(); if ( ! val ) - return new Value(ty->Tag(), false); + return new threading::Value(ty->Tag(), false); - Value* lval = new Value(ty->Tag()); + threading::Value* lval = new threading::Value(ty->Tag()); switch ( lval->type ) { case TYPE_BOOL: @@ -1213,7 +907,7 @@ Value* Manager::ValToLogVal(Val* val, BroType* ty) set = new ListVal(TYPE_INT); lval->val.set_val.size = set->Length(); - lval->val.set_val.vals = new Value* [lval->val.set_val.size]; + lval->val.set_val.vals = new threading::Value* [lval->val.set_val.size]; for ( int i = 0; i < lval->val.set_val.size; i++ ) lval->val.set_val.vals[i] = ValToLogVal(set->Index(i)); @@ -1227,7 +921,7 @@ Value* Manager::ValToLogVal(Val* val, BroType* ty) VectorVal* vec = val->AsVectorVal(); lval->val.vector_val.size = vec->Size(); lval->val.vector_val.vals = - new Value* [lval->val.vector_val.size]; + new threading::Value* [lval->val.vector_val.size]; for ( int i = 0; i < lval->val.vector_val.size; i++ ) { diff --git a/src/logging/Manager.h b/src/logging/Manager.h index f6829b3554..c5d1a9fc2d 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -15,118 +15,6 @@ class RotationTimer; namespace logging { -/** - * Definition of a log file, i.e., one column of a log stream. - */ -struct Field { - string name; //! Name of the field. - TypeTag type; //! Type of the field. - TypeTag subtype; //! Inner type for sets. - - /** - * Constructor. - */ - Field() { subtype = TYPE_VOID; } - - /** - * Copy constructor. - */ - Field(const Field& other) - : name(other.name), type(other.type), subtype(other.subtype) { } - - /** - * Unserializes a field. - * - * @param fmt The serialization format to use. The format handles - * low-level I/O. - * - * @return False if an error occured. - */ - bool Read(SerializationFormat* fmt); - - /** - * Serializes a field. - * - * @param fmt The serialization format to use. The format handles - * low-level I/O. - * - * @return False if an error occured. - */ - bool Write(SerializationFormat* fmt) const; -}; - -/** - * Definition of a log value, i.e., a entry logged by a stream. - * - * This struct essentialy represents a serialization of a Val instance (for - * those Vals supported). - */ -struct Value { - TypeTag type; //! The type of the value. - bool present; //! False for optional record fields that are not set. - - struct set_t { bro_int_t size; Value** vals; }; - typedef set_t vec_t; - - /** - * This union is a subset of BroValUnion, including only the types we - * can log directly. See IsCompatibleType(). - */ - union _val { - bro_int_t int_val; - bro_uint_t uint_val; - uint32 addr_val[NUM_ADDR_WORDS]; - subnet_type subnet_val; - double double_val; - string* string_val; - set_t set_val; - vec_t vector_val; - } val; - - /** - * Constructor. - * - * arg_type: The type of the value. - * - * arg_present: False if the value represents an optional record field - * that is not set. - */ - Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) - : type(arg_type), present(arg_present) {} - - /** - * Destructor. - */ - ~Value(); - - /** - * Unserializes a value. - * - * @param fmt The serialization format to use. The format handles low-level I/O. - * - * @return False if an error occured. - */ - bool Read(SerializationFormat* fmt); - - /** - * Serializes a value. - * - * @param fmt The serialization format to use. The format handles - * low-level I/O. - * - * @return False if an error occured. - */ - bool Write(SerializationFormat* fmt) const; - - /** - * Returns true if the type can be represented by a Value. If - * `atomic_only` is true, will not permit composite types. - */ - static bool IsCompatibleType(BroType* t, bool atomic_only=false); - -private: - Value(const Value& other) { } // Disabled. -}; class WriterBackend; class WriterFrontend; @@ -168,7 +56,7 @@ public: * logging.bif, which just forwards here. */ bool EnableStream(EnumVal* id); - + /** * Disables a log stream. * @@ -265,11 +153,11 @@ protected: // Takes ownership of fields. WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, const Field* const* fields); + int num_fields, const threading::Field* const* fields); // Takes ownership of values.. bool Write(EnumVal* id, EnumVal* writer, string path, - int num_fields, Value** vals); + int num_fields, threading::Value** vals); // Announces all instantiated writers to peer. void SendAllWritersTo(RemoteSerializer::PeerID peer); @@ -282,7 +170,7 @@ protected: void Error(WriterFrontend* writer, const char* msg); // Deletes the values as passed into Write(). - void DeleteVals(int num_fields, Value** vals); + void DeleteVals(int num_fields, threading::Value** vals); private: struct Filter; @@ -292,10 +180,10 @@ private: bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); - Value** RecordToFilterVals(Stream* stream, Filter* filter, + threading::Value** RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns); - Value* ValToLogVal(Val* val, BroType* ty = 0); + threading::Value* ValToLogVal(Val* val, BroType* ty = 0); Stream* FindStream(EnumVal* id); void RemoveDisabledWriters(Stream* stream); void InstallRotationTimer(WriterInfo* winfo); diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 4d2e497b14..3ecc54e240 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -4,9 +4,13 @@ #include "WriterBackend.h" #include "WriterFrontend.h" +#include "../threading/SerializationTypes.h" // Messages sent from backend to frontend (i.e., "OutputMessages"). +using threading::Value; +using threading::Field; + namespace logging { class RotationFinishedMessage : public threading::OutputMessage diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 33271e43f9..9ffa26d0c8 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -55,7 +55,7 @@ public: * * @return False if an error occured. */ - bool Init(string path, int num_fields, const Field* const* fields); + bool Init(string path, int num_fields, const threading::Field* const* fields); /** * Writes one log entry. @@ -72,7 +72,7 @@ public: * * @return False if an error occured. */ - bool Write(int num_fields, int num_writes, Value*** vals); + bool Write(int num_fields, int num_writes, threading::Value*** vals); /** * Sets the buffering status for the writer, assuming the writer @@ -129,7 +129,7 @@ public: /** * Returns the log fields as passed into the constructor. */ - const Field* const * Fields() const { return fields; } + const threading::Field* const * Fields() const { return fields; } /** * Returns the current buffering state. @@ -170,7 +170,7 @@ protected: * implementation should also call Error() to indicate what happened. */ virtual bool DoInit(string path, int num_fields, - const Field* const* fields) = 0; + const threading::Field* const* fields) = 0; /** * Writer-specific output method implementing recording of fone log @@ -182,8 +182,8 @@ protected: * disabled and eventually deleted. When returning false, an * implementation should also call Error() to indicate what happened. */ - virtual bool DoWrite(int num_fields, const Field* const* fields, - Value** vals) = 0; + virtual bool DoWrite(int num_fields, const threading::Field* const* fields, + threading::Value** vals) = 0; /** * Writer-specific method implementing a change of fthe buffering @@ -287,7 +287,7 @@ private: /** * Deletes the values as passed into Write(). */ - void DeleteVals(int num_writes, Value*** vals); + void DeleteVals(int num_writes, threading::Value*** vals); // Frontend that instantiated us. This object must not be access from // this class, it's running in a different thread! @@ -295,7 +295,7 @@ private: string path; // Log path. int num_fields; // Number of log fields. - const Field* const* fields; // Log fields. + const threading::Field* const* fields; // Log fields. bool buffering; // True if buffering is enabled. }; diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 79278870f9..1f7af5a53d 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -1,6 +1,10 @@ #include "WriterFrontend.h" #include "WriterBackend.h" +#include "../threading/SerializationTypes.h" + +using threading::Value; +using threading::Field; namespace logging { diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index e0bc590dfc..56c8885cf9 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -64,7 +64,7 @@ public: * * This method must only be called from the main thread. */ - void Init(string path, int num_fields, const Field* const* fields); + void Init(string path, int num_fields, const threading::Field* const* fields); /** * Write out a record. @@ -86,7 +86,7 @@ public: * * This method must only be called from the main thread. */ - void Write(int num_fields, Value** vals); + void Write(int num_fields, threading::Value** vals); /** * Sets the buffering state. @@ -185,7 +185,7 @@ public: /** * Returns the log fields as passed into the constructor. */ - const Field* const * Fields() const { return fields; } + const threading::Field* const * Fields() const { return fields; } protected: friend class Manager; @@ -198,12 +198,12 @@ protected: string ty_name; // Name of the backend type. Set by the manager. string path; // The log path. int num_fields; // The number of log fields. - const Field* const* fields; // The log fields. + const threading::Field* const* fields; // The log fields. // Buffer for bulk writes. static const int WRITER_BUFFER_SIZE = 50; int write_buffer_pos; // Position of next write in buffer. - Value*** write_buffer; // Buffer of size WRITER_BUFFER_SIZE. + threading::Value*** write_buffer; // Buffer of size WRITER_BUFFER_SIZE. }; } diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 7cc8459e68..fc6832afea 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -6,9 +6,12 @@ #include "../../NetVar.h" #include "Ascii.h" +#include "../../threading/SerializationTypes.h" using namespace logging; using namespace writer; +using threading::Value; +using threading::Field; Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) { diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 4a24aad9b7..6f507aff01 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -20,9 +20,9 @@ public: protected: virtual bool DoInit(string path, int num_fields, - const Field* const* fields); - virtual bool DoWrite(int num_fields, const Field* const* fields, - Value** vals); + const threading::Field* const* fields); + virtual bool DoWrite(int num_fields, const threading::Field* const* fields, + threading::Value** vals); virtual bool DoSetBuf(bool enabled); virtual bool DoRotate(string rotated_path, double open, double close, bool terminating); @@ -31,7 +31,7 @@ protected: private: bool IsSpecial(string path) { return path.find("/dev/") == 0; } - bool DoWriteOne(ODesc* desc, Value* val, const Field* field); + bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field); bool WriteHeaderField(const string& key, const string& value); FILE* file; diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h index 6a62161f49..cce48953d1 100644 --- a/src/logging/writers/None.h +++ b/src/logging/writers/None.h @@ -19,10 +19,10 @@ public: protected: virtual bool DoInit(string path, int num_fields, - const Field* const * fields) { return true; } + const threading::Field* const * fields) { return true; } - virtual bool DoWrite(int num_fields, const Field* const* fields, - Value** vals) { return true; } + virtual bool DoWrite(int num_fields, const threading::Field* const* fields, + threading::Value** vals) { return true; } virtual bool DoSetBuf(bool enabled) { return true; } virtual bool DoRotate(string rotated_path, double open, double close, bool terminating); diff --git a/src/threading/SerializationTypes.cc b/src/threading/SerializationTypes.cc new file mode 100644 index 0000000000..01f0ac84ce --- /dev/null +++ b/src/threading/SerializationTypes.cc @@ -0,0 +1,319 @@ +// See the file "COPYING" in the main distribution directory for copyright. + + +#include "SerializationTypes.h" +#include "../RemoteSerializer.h" + + +using namespace threading; + +bool Field::Read(SerializationFormat* fmt) + { + int t; + int st; + + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); + type = (TypeTag) t; + subtype = (TypeTag) st; + + return success; + } + +bool Field::Write(SerializationFormat* fmt) const + { + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); + } + +Value::~Value() + { + if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC) + && present ) + delete val.string_val; + + if ( type == TYPE_TABLE && present ) + { + for ( int i = 0; i < val.set_val.size; i++ ) + delete val.set_val.vals[i]; + + delete [] val.set_val.vals; + } + + if ( type == TYPE_VECTOR && present ) + { + for ( int i = 0; i < val.vector_val.size; i++ ) + delete val.vector_val.vals[i]; + + delete [] val.vector_val.vals; + } + } + +bool Value::IsCompatibleType(BroType* t, bool atomic_only) + { + if ( ! t ) + return false; + + switch ( t->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_SUBNET: + case TYPE_ADDR: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + return true; + + case TYPE_RECORD: + return ! atomic_only; + + case TYPE_TABLE: + { + if ( atomic_only ) + return false; + + if ( ! t->IsSet() ) + return false; + + return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); + } + + case TYPE_VECTOR: + { + if ( atomic_only ) + return false; + + return IsCompatibleType(t->AsVectorType()->YieldType(), true); + } + + default: + return false; + } + + return false; + } + +bool Value::Read(SerializationFormat* fmt) + { + int ty; + + if ( ! (fmt->Read(&ty, "type") && fmt->Read(&present, "present")) ) + return false; + + type = (TypeTag)(ty); + + if ( ! present ) + return true; + + switch ( type ) { + case TYPE_BOOL: + case TYPE_INT: + return fmt->Read(&val.int_val, "int"); + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + return fmt->Read(&val.uint_val, "uint"); + + case TYPE_SUBNET: + { + uint32 net[4]; + if ( ! (fmt->Read(&net[0], "net0") && + fmt->Read(&net[1], "net1") && + fmt->Read(&net[2], "net2") && + fmt->Read(&net[3], "net3") && + fmt->Read(&val.subnet_val.width, "width")) ) + return false; + +#ifdef BROv6 + val.subnet_val.net[0] = net[0]; + val.subnet_val.net[1] = net[1]; + val.subnet_val.net[2] = net[2]; + val.subnet_val.net[3] = net[3]; +#else + val.subnet_val.net = net[0]; +#endif + return true; + } + + case TYPE_ADDR: + { + uint32 addr[4]; + if ( ! (fmt->Read(&addr[0], "addr0") && + fmt->Read(&addr[1], "addr1") && + fmt->Read(&addr[2], "addr2") && + fmt->Read(&addr[3], "addr3")) ) + return false; + + val.addr_val[0] = addr[0]; +#ifdef BROv6 + val.addr_val[1] = addr[1]; + val.addr_val[2] = addr[2]; + val.addr_val[3] = addr[3]; +#endif + return true; + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return fmt->Read(&val.double_val, "double"); + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + { + val.string_val = new string; + return fmt->Read(val.string_val, "string"); + } + + case TYPE_TABLE: + { + if ( ! fmt->Read(&val.set_val.size, "set_size") ) + return false; + + val.set_val.vals = new Value* [val.set_val.size]; + + for ( int i = 0; i < val.set_val.size; ++i ) + { + val.set_val.vals[i] = new Value; + + if ( ! val.set_val.vals[i]->Read(fmt) ) + return false; + } + + return true; + } + + case TYPE_VECTOR: + { + if ( ! fmt->Read(&val.vector_val.size, "vector_size") ) + return false; + + val.vector_val.vals = new Value* [val.vector_val.size]; + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + val.vector_val.vals[i] = new Value; + + if ( ! val.vector_val.vals[i]->Read(fmt) ) + return false; + } + + return true; + } + + default: + reporter->InternalError("unsupported type %s in Value::Write", type_name(type)); + } + + return false; + } + +bool Value::Write(SerializationFormat* fmt) const + { + if ( ! (fmt->Write((int)type, "type") && + fmt->Write(present, "present")) ) + return false; + + if ( ! present ) + return true; + + switch ( type ) { + case TYPE_BOOL: + case TYPE_INT: + return fmt->Write(val.int_val, "int"); + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + return fmt->Write(val.uint_val, "uint"); + + case TYPE_SUBNET: + { + uint32 net[4]; +#ifdef BROv6 + net[0] = val.subnet_val.net[0]; + net[1] = val.subnet_val.net[1]; + net[2] = val.subnet_val.net[2]; + net[3] = val.subnet_val.net[3]; +#else + net[0] = val.subnet_val.net; + net[1] = net[2] = net[3] = 0; +#endif + return fmt->Write(net[0], "net0") && + fmt->Write(net[1], "net1") && + fmt->Write(net[2], "net2") && + fmt->Write(net[3], "net3") && + fmt->Write(val.subnet_val.width, "width"); + } + + case TYPE_ADDR: + { + uint32 addr[4]; + addr[0] = val.addr_val[0]; +#ifdef BROv6 + addr[1] = val.addr_val[1]; + addr[2] = val.addr_val[2]; + addr[3] = val.addr_val[3]; +#else + addr[1] = addr[2] = addr[3] = 0; +#endif + return fmt->Write(addr[0], "addr0") && + fmt->Write(addr[1], "addr1") && + fmt->Write(addr[2], "addr2") && + fmt->Write(addr[3], "addr3"); + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return fmt->Write(val.double_val, "double"); + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + return fmt->Write(*val.string_val, "string"); + + case TYPE_TABLE: + { + if ( ! fmt->Write(val.set_val.size, "set_size") ) + return false; + + for ( int i = 0; i < val.set_val.size; ++i ) + { + if ( ! val.set_val.vals[i]->Write(fmt) ) + return false; + } + + return true; + } + + case TYPE_VECTOR: + { + if ( ! fmt->Write(val.vector_val.size, "vector_size") ) + return false; + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + if ( ! val.vector_val.vals[i]->Write(fmt) ) + return false; + } + + return true; + } + + default: + reporter->InternalError("unsupported type %s in Value::REad", type_name(type)); + } + + return false; + } + diff --git a/src/threading/SerializationTypes.h b/src/threading/SerializationTypes.h new file mode 100644 index 0000000000..8cae99c117 --- /dev/null +++ b/src/threading/SerializationTypes.h @@ -0,0 +1,126 @@ + +#ifndef THREADING_SERIALIZATIONTYPES_H +#define THREADING_SERIALIZATIONTZPES_H + +#include "../RemoteSerializer.h" + +using namespace std; + +namespace threading { + +/** + * Definition of a log file, i.e., one column of a log stream. + */ +struct Field { + string name; //! Name of the field. + TypeTag type; //! Type of the field. + TypeTag subtype; //! Inner type for sets. + + /** + * Constructor. + */ + Field() { subtype = TYPE_VOID; } + + /** + * Copy constructor. + */ + Field(const Field& other) + : name(other.name), type(other.type), subtype(other.subtype) { } + + /** + * Unserializes a field. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ + bool Read(SerializationFormat* fmt); + + /** + * Serializes a field. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ + bool Write(SerializationFormat* fmt) const; +}; + +/** + * Definition of a log value, i.e., a entry logged by a stream. + * + * This struct essentialy represents a serialization of a Val instance (for + * those Vals supported). + */ +struct Value { + TypeTag type; //! The type of the value. + bool present; //! False for optional record fields that are not set. + + struct set_t { bro_int_t size; Value** vals; }; + typedef set_t vec_t; + + /** + * This union is a subset of BroValUnion, including only the types we + * can log directly. See IsCompatibleType(). + */ + union _val { + bro_int_t int_val; + bro_uint_t uint_val; + uint32 addr_val[NUM_ADDR_WORDS]; + subnet_type subnet_val; + double double_val; + string* string_val; + set_t set_val; + vec_t vector_val; + } val; + + /** + * Constructor. + * + * arg_type: The type of the value. + * + * arg_present: False if the value represents an optional record field + * that is not set. + */ + Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) + : type(arg_type), present(arg_present) {} + + /** + * Destructor. + */ + ~Value(); + + /** + * Unserializes a value. + * + * @param fmt The serialization format to use. The format handles low-level I/O. + * + * @return False if an error occured. + */ + bool Read(SerializationFormat* fmt); + + /** + * Serializes a value. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ + bool Write(SerializationFormat* fmt) const; + + /** + * Returns true if the type can be represented by a Value. If + * `atomic_only` is true, will not permit composite types. + */ + static bool IsCompatibleType(BroType* t, bool atomic_only=false); + +private: + Value(const Value& other) { } // Disabled. +}; + +} + +#endif /* THREADING_SERIALIZATIONTZPES_H */ From 23b2c95644f3a48a3a154843041ae1b016f1dfc1 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 6 Feb 2012 10:57:07 -0800 Subject: [PATCH 15/56] backend does not need friend access to manager --- src/logging/WriterBackend.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 9ffa26d0c8..aa4448f5e2 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -282,8 +282,6 @@ protected: virtual bool DoHeartbeat(double network_time, double current_time); private: - friend class Manager; - /** * Deletes the values as passed into Write(). */ From f76bbf01a4e02fabd55ed38fba28e3acb7ab9036 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 6 Feb 2012 13:15:01 -0800 Subject: [PATCH 16/56] fix CreateBackend function - the way that the right backend was chosen & backends were initialized did not make sense... --- src/logging/Manager.cc | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index add10b3f10..e8d732d84a 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -142,12 +142,12 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) return 0; } - if ( ld->type == type ) - break; - - if ( ! ld->factory ) - // Oops, we can't instantiate this guy. - return 0; + if ( ld->type != type ) + { + // no, didn't find the right one... + ++ld; + continue; + } // If the writer has an init function, call it. if ( ld->init ) @@ -157,17 +157,24 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) // call it again later. ld->init = 0; else + { // Init failed, disable by deleting factory // function. ld->factory = 0; - DBG_LOG(DBG_LOGGING, "failed to init writer class %s", - ld->name); + DBG_LOG(DBG_LOGGING, "failed to init writer class %s", + ld->name); - return false; + return false; + } } - ++ld; + if ( ! ld->factory ) + // Oops, we can't instantiate this guy. + return 0; + + // all done. break. + break; } assert(ld->factory); From e22d396229343b4956d93b14ad9629cde4690dbf Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 6 Feb 2012 13:53:33 -0800 Subject: [PATCH 17/56] typo --- src/threading/SerializationTypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/threading/SerializationTypes.h b/src/threading/SerializationTypes.h index 11ceda929c..8f36402637 100644 --- a/src/threading/SerializationTypes.h +++ b/src/threading/SerializationTypes.h @@ -1,6 +1,6 @@ #ifndef THREADING_SERIALIZATIONTYPES_H -#define THREADING_SERIALIZATIONTZPES_H +#define THREADING_SERIALIZATIONTYPES_H #include "../RemoteSerializer.h" From b8ec653ebf31ffc7c2dffa02614f93b91bb45367 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 12 Feb 2012 09:41:43 -0800 Subject: [PATCH 18/56] Bugfixes. - Data queued at termination wasn't written out completely. - Fixed some race conditions. - Fixing IOSource integration. - Fixing setting thread names on Linux. - Fixing minor leaks. All tests now pass for me on Linux in debug and non-debug compiles. Remaining TODOs: - Needs leak check. - Test on MacOS and FreeBSD. - More testing: - High volume traffic. - Different platforms. --- src/logging/Manager.cc | 9 +++++++++ src/logging/Manager.h | 8 +++++++- src/logging/WriterBackend.cc | 8 ++++++-- src/logging/WriterFrontend.cc | 6 ++++-- src/logging/WriterFrontend.h | 3 --- src/logging/writers/Ascii.cc | 24 ++++++++++++++++++++++-- src/main.cc | 1 + src/threading/BasicThread.cc | 11 +++++++++-- src/threading/BasicThread.h | 2 +- src/threading/Manager.cc | 30 +++++++++++++++++++----------- src/threading/MsgThread.cc | 21 +++++++++++++++++---- src/threading/MsgThread.h | 7 +++++-- 12 files changed, 100 insertions(+), 30 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 6d53ea363f..593766e52a 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -1470,6 +1470,15 @@ bool Manager::Flush(EnumVal* id) return true; } +void Manager::Terminate() + { + for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) + { + if ( *s ) + Flush((*s)->id); + } + } + void Manager::Error(WriterFrontend* writer, const char* msg) { reporter->Error("error with writer for %s: %s", diff --git a/src/logging/Manager.h b/src/logging/Manager.h index f6829b3554..d12fc7e8fe 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -251,6 +251,12 @@ public: */ bool Flush(EnumVal* id); + /** + * Prepares the log manager to terminate. This will flush all log + * stream. + */ + void Terminate(); + protected: friend class WriterFrontend; friend class RotationFinishedMessage; @@ -258,7 +264,7 @@ protected: friend class ::RotationTimer; // Instantiates a new WriterBackend of the given type (note that - // doing so creates a new thread!). + // doing so creates a new thread!). WriterBackend* CreateBackend(WriterFrontend* frontend, bro_int_t type); //// Function also used by the RemoteSerializer. diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 4d2e497b14..fa86fce324 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -57,7 +57,7 @@ using namespace logging; WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread() { - path = ""; + path = ""; num_fields = 0; fields = 0; buffering = true; @@ -109,7 +109,9 @@ bool WriterBackend::Init(string arg_path, int arg_num_fields, const Field* const num_fields = arg_num_fields; fields = arg_fields; - SetName(frontend->Name()); + string name = Fmt("%s/%s", path.c_str(), frontend->Name().c_str()); + + SetName(name); if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) { @@ -229,6 +231,8 @@ bool WriterBackend::Finish() bool WriterBackend::DoHeartbeat(double network_time, double current_time) { + MsgThread::DoHeartbeat(network_time, current_time); + SendOut(new FlushWriteBufferMessage(frontend)); return true; diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 79278870f9..0a8ff4b09d 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -1,4 +1,6 @@ +#include "Net.h" + #include "WriterFrontend.h" #include "WriterBackend.h" @@ -155,8 +157,8 @@ void WriterFrontend::Write(int num_fields, Value** vals) write_buffer[write_buffer_pos++] = vals; - if ( write_buffer_pos >= WRITER_BUFFER_SIZE || ! buf ) - // Buffer full (or no bufferin desired). + if ( write_buffer_pos >= WRITER_BUFFER_SIZE || ! buf || terminating ) + // Buffer full (or no bufferin desired or termiating). FlushWriteBuffer(); } diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index e0bc590dfc..a1a1e2b86a 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -30,9 +30,6 @@ public: * frontend will internally instantiate a WriterBackend of the * corresponding type. * - * name: A descriptive name for the backend wroter type (e.g., \c - * Ascii). - * * Frontends must only be instantiated by the main thread. */ WriterFrontend(bro_int_t type); diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 7cc8459e68..5429bf0b97 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -171,14 +171,34 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) break; case TYPE_SUBNET: - desc->Add(dotted_addr(val->val.subnet_val.net)); + { + // FIXME: This will be replaced with string(addr) once the + // IPV6 branch is merged in. + uint32_t addr = ntohl(val->val.subnet_val.net); + char buf[32]; + snprintf(buf, sizeof(buf), "%d.%d.%d.%d", + addr >> 24, (addr >> 16) & 0xff, + (addr >> 8) & 0xff, addr & 0xff); + + desc->Add(buf); desc->Add("/"); desc->Add(val->val.subnet_val.width); break; + } case TYPE_ADDR: - desc->Add(dotted_addr(val->val.addr_val)); + { + // FIXME: This will be replaced with string(addr) once the + // IPV6 branch is merged in. + uint32_t addr = ntohl(*val->val.addr_val); + char buf[32]; + snprintf(buf, sizeof(buf), "%d.%d.%d.%d", + addr >> 24, (addr >> 16) & 0xff, + (addr >> 8) & 0xff, addr & 0xff); + + desc->Add(buf); break; + } case TYPE_TIME: case TYPE_INTERVAL: diff --git a/src/main.cc b/src/main.cc index e224910db4..c101e54e74 100644 --- a/src/main.cc +++ b/src/main.cc @@ -290,6 +290,7 @@ void terminate_bro() if ( remote_serializer ) remote_serializer->LogStats(); + log_mgr->Terminate(); thread_mgr->Terminate(); delete timer_mgr; diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 4d51c3c4e4..51c4f7a3bc 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -2,9 +2,14 @@ #include #include +#include "config.h" #include "BasicThread.h" #include "Manager.h" +#ifdef HAVE_LINUX +#include +#endif + using namespace threading; uint64_t BasicThread::thread_counter = 0; @@ -25,6 +30,8 @@ BasicThread::BasicThread() BasicThread::~BasicThread() { + if ( buf ) + free(buf); } void BasicThread::SetName(const string& arg_name) @@ -35,8 +42,8 @@ void BasicThread::SetName(const string& arg_name) void BasicThread::SetOSName(const string& name) { -#ifdef LINUX - pthread_setname_np(pthread_self(), name.c_str()); +#ifdef HAVE_LINUX + prctl(PR_SET_NAME, name.c_str(), 0, 0, 0); #endif #ifdef __APPLE__ diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index 6d2f739620..cc87ae03bc 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -82,7 +82,7 @@ public: void Stop(); /** - * Returns true if Terminate() has been called. + * Returns true if Stop() has been called. * * This method is safe to call from any thread. */ diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d07311bbe8..d008d2e5e8 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -7,9 +7,10 @@ Manager::Manager() { DBG_LOG(DBG_THREADING, "Creating thread manager ..."); - did_process = false; + did_process = true; next_beat = 0; terminating = false; + idle = false; } Manager::~Manager() @@ -41,6 +42,7 @@ void Manager::Terminate() all_threads.clear(); msg_threads.clear(); + idle = true; terminating = false; } @@ -70,18 +72,22 @@ void Manager::GetFds(int* read, int* write, int* except) double Manager::NextTimestamp(double* network_time) { - if ( did_process || ! next_beat == 0 ) - // If we had something to process last time (or haven't had a - // chance to check yet), we want to check for more asap. + if ( ::network_time && ! next_beat ) + next_beat = ::network_time + HEART_BEAT_INTERVAL; + +// fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat); + + if ( did_process || ::network_time > next_beat ) + // If we had something to process last time (or out heartbeat + // is due), we want to check for more asap. return timer_mgr->Time(); - // Else we assume we don't have much to do at all and wait for the next heart beat. - return next_beat; + return -1.0; } void Manager::Process() { - bool do_beat = (next_beat == 0 || network_time >= next_beat); + bool do_beat = (next_beat && network_time > next_beat); did_process = false; @@ -90,14 +96,17 @@ void Manager::Process() MsgThread* t = *i; if ( do_beat ) + { t->Heartbeat(); + next_beat = 0; + } if ( ! t->HasOut() ) continue; Message* msg = t->RetrieveOut(); - if ( msg->Process() ) + if ( msg->Process() && network_time ) did_process = true; else @@ -110,15 +119,14 @@ void Manager::Process() delete msg; } - if ( do_beat ) - next_beat = network_time + HEART_BEAT_INTERVAL; +// fprintf(stderr, "P %.6f %.6f do_beat=%d did_process=%d next_next=%.6f\n", network_time, timer_mgr->Time(), do_beat, (int)did_process, next_beat); } const threading::Manager::msg_stats_list& threading::Manager::GetMsgThreadStats() { stats.clear(); - for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) { MsgThread* t = *i; diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index f41b20ddf9..b7782b9a05 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -142,12 +142,19 @@ void MsgThread::OnStop() void MsgThread::Heartbeat() { SendIn(new HeartbeatMessage(this, network_time, current_time())); + } - string name = Fmt("%s (%d/%d)", name.c_str(), - cnt_sent_in - queue_in.Size(), - cnt_sent_out - queue_out.Size()); +bool MsgThread::DoHeartbeat(double network_time, double current_time) + { + string n = Name(); - SetOSName(name.c_str()); + n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", n.c_str(), + cnt_sent_in - queue_in.Size(), + cnt_sent_out - queue_out.Size()); + + SetOSName(n.c_str()); + + return true; } void MsgThread::Info(const char* msg) @@ -197,7 +204,10 @@ void MsgThread::Debug(DebugStream stream, const char* msg) void MsgThread::SendIn(BasicInputMessage* msg, bool force) { if ( Terminating() && ! force ) + { + delete msg; return; + } DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str()); @@ -209,7 +219,10 @@ void MsgThread::SendIn(BasicInputMessage* msg, bool force) void MsgThread::SendOut(BasicOutputMessage* msg, bool force) { if ( Terminating() && ! force ) + { + delete msg; return; + } queue_out.Put(msg); diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 459ac6c603..28c7690dfa 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -184,7 +184,10 @@ protected: * This is method is called regularly by the threading::Manager. * * Can be overriden in derived classed to hook into the heart beat, - * but must call the parent implementation. + * but must call the parent implementation. Note that this method is + * always called by the main thread and must not access data of the + * child thread directly. See DoHeartbeat() if you want to do + * something on the child-side. */ virtual void Heartbeat(); @@ -206,7 +209,7 @@ protected: * current_time: Wall clock when the heartbeat was trigger by the * main thread. */ - virtual bool DoHeartbeat(double network_time, double current_time) { return true; } + virtual bool DoHeartbeat(double network_time, double current_time); private: /** From 7fcb7b5f17a966b6f384f7c34aa234fb60a45483 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 12 Feb 2012 13:04:47 -0800 Subject: [PATCH 19/56] Save CPU when idle. This needs a bit more testing. It may also with the general problem of high CPU usage with low traffic. --- src/Net.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Net.cc b/src/Net.cc index 2d8ee85353..d2b505544e 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -454,6 +454,7 @@ void net_run() // date on timers and events. network_time = ct; expire_timers(); + usleep(1); // Just yield. } } From 1f8b299aaf37c5d03994c8ed9f6f7acaaba9a98b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 14 Feb 2012 10:12:09 -0800 Subject: [PATCH 20/56] Shortening file names a bit. --- src/Attr.cc | 2 +- src/CMakeLists.txt | 2 +- src/RemoteSerializer.cc | 2 +- src/logging/Manager.cc | 4 ++-- src/logging/WriterBackend.cc | 2 +- src/logging/WriterFrontend.cc | 2 +- src/logging/writers/Ascii.cc | 4 ++-- src/threading/{SerializationTypes.cc => SerialTypes.cc} | 2 +- src/threading/{SerializationTypes.h => SerialTypes.h} | 0 9 files changed, 10 insertions(+), 10 deletions(-) rename src/threading/{SerializationTypes.cc => SerialTypes.cc} (99%) rename src/threading/{SerializationTypes.h => SerialTypes.h} (100%) diff --git a/src/Attr.cc b/src/Attr.cc index 40c6c1a75c..82d9c9ddc7 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -5,7 +5,7 @@ #include "Attr.h" #include "Expr.h" #include "Serializer.h" -#include "threading/SerializationTypes.h" +#include "threading/SerialTypes.h" const char* attr_name(attr_tag t) { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7a3cc4babf..67d82c577a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -411,7 +411,7 @@ set(bro_SRCS threading/BasicThread.cc threading/Manager.cc threading/MsgThread.cc - threading/SerializationTypes.cc + threading/SerialTypes.cc logging/Manager.cc logging/WriterBackend.cc diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index ba2598c018..948dfddaff 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -184,7 +184,7 @@ #include "File.h" #include "Conn.h" #include "Reporter.h" -#include "threading/SerializationTypes.h" +#include "threading/SerialTypes.h" #include "logging/Manager.h" extern "C" { diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 5f9f1c4222..6078a1e566 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -7,6 +7,8 @@ #include "../NetVar.h" #include "../Net.h" +#include "threading/SerialTypes.h" + #include "Manager.h" #include "WriterFrontend.h" #include "WriterBackend.h" @@ -14,8 +16,6 @@ #include "writers/Ascii.h" #include "writers/None.h" -#include "threading/SerializationTypes.h" - using namespace logging; using threading::Value; using threading::Field; diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 0ffedf073c..f4e48ebaef 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -1,10 +1,10 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "util.h" +#include "threading/SerialTypes.h" #include "WriterBackend.h" #include "WriterFrontend.h" -#include "../threading/SerializationTypes.h" // Messages sent from backend to frontend (i.e., "OutputMessages"). diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index b0e780f27d..02f1a188d8 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -1,9 +1,9 @@ #include "Net.h" +#include "threading/SerialTypes.h" #include "WriterFrontend.h" #include "WriterBackend.h" -#include "../threading/SerializationTypes.h" using threading::Value; using threading::Field; diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index c974877175..0a101feb79 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -3,10 +3,10 @@ #include #include -#include "../../NetVar.h" +#include "NetVar.h" +#include "threading/SerialTypes.h" #include "Ascii.h" -#include "../../threading/SerializationTypes.h" using namespace logging; using namespace writer; diff --git a/src/threading/SerializationTypes.cc b/src/threading/SerialTypes.cc similarity index 99% rename from src/threading/SerializationTypes.cc rename to src/threading/SerialTypes.cc index f74de6ce57..f35d1fc6b0 100644 --- a/src/threading/SerializationTypes.cc +++ b/src/threading/SerialTypes.cc @@ -1,7 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "SerializationTypes.h" +#include "SerialTypes.h" #include "../RemoteSerializer.h" diff --git a/src/threading/SerializationTypes.h b/src/threading/SerialTypes.h similarity index 100% rename from src/threading/SerializationTypes.h rename to src/threading/SerialTypes.h From bc6ebe53cc40f28aebff0c744e33e92fdc6328ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 16 Feb 2012 11:31:30 -0800 Subject: [PATCH 21/56] remove unnecessary error function from manger --- src/logging/Manager.cc | 6 ------ src/logging/Manager.h | 3 --- 2 files changed, 9 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 5ab2e5bc77..b47a2f9eff 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -1181,12 +1181,6 @@ void Manager::Terminate() } } -void Manager::Error(WriterFrontend* writer, const char* msg) - { - reporter->Error("error with writer for %s: %s", - writer->Path().c_str(), msg); - } - // Timer which on dispatching rotates the filter. class RotationTimer : public Timer { public: diff --git a/src/logging/Manager.h b/src/logging/Manager.h index b65d22e3c0..d931bfaef8 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -172,9 +172,6 @@ protected: bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, double open, double close, bool terminating); - // Reports an error for the given writer. - void Error(WriterFrontend* writer, const char* msg); - // Deletes the values as passed into Write(). void DeleteVals(int num_fields, threading::Value** vals); From 14916b43f634c9b0dc989c8b89ffcd272951b633 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 27 Feb 2012 21:39:56 -0800 Subject: [PATCH 22/56] Readding deleted functions. These are needed in debug mode in turns out. --- src/net_util.cc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/net_util.cc b/src/net_util.cc index a65afb1a25..f78246c634 100644 --- a/src/net_util.cc +++ b/src/net_util.cc @@ -147,6 +147,26 @@ char addr_to_class(uint32 addr) return 'A'; } +const char* fmt_conn_id(const IPAddr& src_addr, uint32 src_port, + const IPAddr& dst_addr, uint32 dst_port) + { + static char buffer[512]; + + safe_snprintf(buffer, sizeof(buffer), "%s:%d > %s:%d", + string(src_addr).c_str(), src_port, + string(dst_addr).c_str(), dst_port); + + return buffer; + } + +const char* fmt_conn_id(const uint32* src_addr, uint32 src_port, + const uint32* dst_addr, uint32 dst_port) + { + IPAddr src(IPAddr::IPv6, src_addr, IPAddr::Network); + IPAddr dst(IPAddr::IPv6, dst_addr, IPAddr::Network); + return fmt_conn_id(src, src_port, dst, dst_port); + } + uint32 extract_uint32(const u_char* data) { uint32 val; From edc9bb14af6b2fe56318e5dbb07847d4413cc408 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 28 Feb 2012 15:12:35 -0800 Subject: [PATCH 23/56] Making exchange of addresses between threads thread-safe. As we can't use the IPAddr class (because it's not thread-safe), this involved a bit manual address manipulation and also shuffling some things around a bit. Not fully working yet, the tests for remote logging still fail. --- src/Anon.cc | 2 +- src/CompHash.cc | 2 +- src/ConnCompressor.cc | 16 ++++---- src/DCE_RPC.cc | 4 +- src/DNS_Mgr.cc | 4 +- src/Desc.cc | 10 +++++ src/Desc.h | 7 ++-- src/IPAddr.cc | 4 +- src/IPAddr.h | 45 ++++++++++++++++++-- src/RemoteSerializer.cc | 4 +- src/RuleMatcher.cc | 2 +- src/SerializationFormat.cc | 80 +++++++++++++++++++++++++++++++++++- src/SerializationFormat.h | 15 +++++++ src/Val.cc | 26 ++++++++---- src/Val.h | 8 +--- src/bro.bif | 22 +++++----- src/logging/Manager.cc | 4 +- src/logging/WriterBackend.cc | 36 ++++++++++++++++ src/logging/WriterBackend.h | 15 +++++++ src/logging/writers/Ascii.cc | 4 +- src/net_util.cc | 4 +- src/net_util.h | 10 ++++- src/threading/SerialTypes.cc | 63 +++++++++++++++++++--------- src/threading/SerialTypes.h | 22 ++++++++-- 24 files changed, 325 insertions(+), 84 deletions(-) diff --git a/src/Anon.cc b/src/Anon.cc index 4c4146ac3e..d2a28a0e08 100644 --- a/src/Anon.cc +++ b/src/Anon.cc @@ -154,7 +154,7 @@ void AnonymizeIPAddr_A50::init() int AnonymizeIPAddr_A50::PreservePrefix(ipaddr32_t input, int num_bits) { DEBUG_MSG("%s/%d\n", - IPAddr(IPAddr::IPv4, &input, IPAddr::Network).AsString().c_str(), + IPAddr(IPv4, &input, IPAddr::Network).AsString().c_str(), num_bits); if ( ! before_anonymization ) diff --git a/src/CompHash.cc b/src/CompHash.cc index ae0e082216..86677f9719 100644 --- a/src/CompHash.cc +++ b/src/CompHash.cc @@ -709,7 +709,7 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, const uint32* const kp = AlignType(kp0); kp1 = reinterpret_cast(kp+4); - IPAddr addr(IPAddr::IPv6, kp, IPAddr::Network); + IPAddr addr(IPv6, kp, IPAddr::Network); switch ( tag ) { case TYPE_ADDR: diff --git a/src/ConnCompressor.cc b/src/ConnCompressor.cc index 29e24457f5..e2b297220a 100644 --- a/src/ConnCompressor.cc +++ b/src/ConnCompressor.cc @@ -236,7 +236,7 @@ Connection* ConnCompressor::NextPacket(double t, HashKey* key, const IP_Hdr* ip, } else if ( ip->SrcAddr() == - IPAddr(IPAddr::IPv6, SrcAddr(pending), IPAddr::Network) && + IPAddr(IPv6, SrcAddr(pending), IPAddr::Network) && tp->th_sport == SrcPort(pending) ) // Another packet from originator. tc = NextFromOrig(pending, t, key, ip, tp); @@ -508,8 +508,8 @@ Connection* ConnCompressor::Instantiate(HashKey* key, PendingConn* pending) { // Instantantiate a Connection. ConnID conn_id; - conn_id.src_addr = IPAddr(IPAddr::IPv6, SrcAddr(pending), IPAddr::Network); - conn_id.dst_addr = IPAddr(IPAddr::IPv6, DstAddr(pending), IPAddr::Network); + conn_id.src_addr = IPAddr(IPv6, SrcAddr(pending), IPAddr::Network); + conn_id.dst_addr = IPAddr(IPv6, DstAddr(pending), IPAddr::Network); conn_id.src_port = SrcPort(pending); conn_id.dst_port = DstPort(pending); @@ -608,7 +608,7 @@ void ConnCompressor::PktHdrToPendingConn(double time, const HashKey* key, memcpy(&c->key, key->Key(), key->Size()); c->hash = key->Hash(); - IPAddr ip1(IPAddr::IPv6, c->key.ip1, IPAddr::Network); + IPAddr ip1(IPv6, c->key.ip1, IPAddr::Network); c->ip1_is_src = ip1 == ip->SrcAddr() && c->key.port1 == tp->th_sport; c->time = time; @@ -658,10 +658,10 @@ const IP_Hdr* ConnCompressor::PendingConnToPacket(const PendingConn* c) tp->th_urp = 0; } - IPAddr ip1(IPAddr::IPv6, c->key.ip1, IPAddr::Network); - IPAddr ip2(IPAddr::IPv6, c->key.ip2, IPAddr::Network); - if ( ip1.GetFamily() == IPAddr::IPv6 || - ip2.GetFamily() == IPAddr::IPv6 ) + IPAddr ip1(IPv6, c->key.ip1, IPAddr::Network); + IPAddr ip2(IPv6, c->key.ip2, IPAddr::Network); + if ( ip1.GetFamily() == IPv6 || + ip2.GetFamily() == IPv6 ) reporter->InternalError("IPv6 snuck into connection compressor"); else { diff --git a/src/DCE_RPC.cc b/src/DCE_RPC.cc index 88cdb48e80..21cb3be9a0 100644 --- a/src/DCE_RPC.cc +++ b/src/DCE_RPC.cc @@ -137,7 +137,7 @@ static bool is_mapped_dce_rpc_endpoint(const dce_rpc_endpoint_addr& addr) bool is_mapped_dce_rpc_endpoint(const ConnID* id, TransportProto proto) { - if ( id->dst_addr.GetFamily() == IPAddr::IPv6 ) + if ( id->dst_addr.GetFamily() == IPv6 ) // TODO: Does the protocol support v6 addresses? #773 return false; @@ -414,7 +414,7 @@ void DCE_RPC_Session::DeliverEpmapperMapResponse( case binpac::DCE_RPC_Simple::EPM_PROTOCOL_IP: uint32 hostip = floor->rhs()->data()->ip(); - mapped.addr.addr = IPAddr(IPAddr::IPv4, &hostip, IPAddr::Host); + mapped.addr.addr = IPAddr(IPv4, &hostip, IPAddr::Host); break; } } diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index a80de42704..9e65d3c9a9 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -321,10 +321,10 @@ void DNS_Mapping::Init(struct hostent* h) addrs = new IPAddr[num_addrs]; for ( int i = 0; i < num_addrs; ++i ) if ( h->h_addrtype == AF_INET ) - addrs[i] = IPAddr(IPAddr::IPv4, (uint32*)h->h_addr_list[i], + addrs[i] = IPAddr(IPv4, (uint32*)h->h_addr_list[i], IPAddr::Network); else if ( h->h_addrtype == AF_INET6 ) - addrs[i] = IPAddr(IPAddr::IPv6, (uint32*)h->h_addr_list[i], + addrs[i] = IPAddr(IPv6, (uint32*)h->h_addr_list[i], IPAddr::Network); } else diff --git a/src/Desc.cc b/src/Desc.cc index 12b4a524eb..9d94321427 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -157,6 +157,16 @@ void ODesc::Add(double d) } } +void ODesc::Add(const IPAddr& addr) + { + Add(addr.AsString()); + } + +void ODesc::Add(const IPPrefix& prefix) + { + Add(prefix.AsString()); + } + void ODesc::AddCS(const char* s) { int n = strlen(s); diff --git a/src/Desc.h b/src/Desc.h index 9f8d7ab109..9c60c68106 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -8,7 +8,6 @@ #include #include "BroString.h" -#include "IPAddr.h" typedef enum { DESC_READABLE, @@ -23,6 +22,8 @@ typedef enum { } desc_style; class BroFile; +class IPAddr; +class IPPrefix; class ODesc { public: @@ -76,8 +77,8 @@ public: void Add(int64 i); void Add(uint64 u); void Add(double d); - void Add(const IPAddr& addr) { Add(addr.AsString()); } - void Add(const IPPrefix& prefix) { Add(prefix.AsString()); } + void Add(const IPAddr& addr); + void Add(const IPPrefix& prefix); // Add s as a counted string. void AddCS(const char* s); diff --git a/src/IPAddr.cc b/src/IPAddr.cc index ff124025f9..29c8f1b8cf 100644 --- a/src/IPAddr.cc +++ b/src/IPAddr.cc @@ -250,7 +250,7 @@ IPPrefix::IPPrefix(const in6_addr& in6, uint8_t length) IPPrefix::IPPrefix(const IPAddr& addr, uint8_t length) : prefix(addr) { - if ( prefix.GetFamily() == IPAddr::IPv4 ) + if ( prefix.GetFamily() == IPv4 ) { if ( length > 32 ) reporter->InternalError("Bad IPAddr(v4) IPPrefix length : %d", @@ -275,7 +275,7 @@ string IPPrefix::AsString() const { char l[16]; - if ( prefix.GetFamily() == IPAddr::IPv4 ) + if ( prefix.GetFamily() == IPv4 ) modp_uitoa10(length - 96, l); else modp_uitoa10(length, l); diff --git a/src/IPAddr.h b/src/IPAddr.h index f0c0ac12c8..67d6f2112e 100644 --- a/src/IPAddr.h +++ b/src/IPAddr.h @@ -10,6 +10,8 @@ #include "BroString.h" #include "Hash.h" #include "util.h" +#include "Type.h" +#include "threading/SerialTypes.h" struct ConnID; class ExpectedConn; @@ -25,7 +27,7 @@ public: /** * Address family. */ - enum Family { IPv4, IPv6 }; + typedef IPFamily Family; /** * Byte order. @@ -318,14 +320,19 @@ public: return memcmp(&addr1.in6, &addr2.in6, sizeof(in6_addr)) < 0; } + /** Converts the address into the type used internally by the + * inter-thread communication. + */ + void ConvertToThreadingValue(threading::Value::addr_t* v) const; + friend HashKey* BuildConnIDHashKey(const ConnID& id); friend HashKey* BuildExpectedConnHashKey(const ExpectedConn& c); - friend class IPPrefix; - unsigned int MemoryAllocation() const { return padded_sizeof(*this); } private: + friend class IPPrefix; + /** * Initializes an address instance from a string representation. * @@ -384,6 +391,25 @@ inline bool IPAddr::IsLoopback() const && (in6.s6_addr[14] == 0) && (in6.s6_addr[15] == 1)); } +inline void IPAddr::ConvertToThreadingValue(threading::Value::addr_t* v) const + { + v->family = GetFamily(); + + switch ( v->family ) { + + case IPv4: + CopyIPv4(&v->in.in4); + return; + + case IPv6: + CopyIPv6(&v->in.in6); + return; + + // Can't be reached. + abort(); + } + } + /** * Returns a hash key for a given ConnID. Passes ownership to caller. */ @@ -459,7 +485,7 @@ public: */ uint8_t Length() const { - return prefix.GetFamily() == IPAddr::IPv4 ? length - 96 : length; + return prefix.GetFamily() == IPv4 ? length - 96 : length; } /** @@ -497,6 +523,8 @@ public: */ string AsString() const; + /** Converts the address into the type used internally by the inter-thread communicastion. + */ operator std::string() const { return AsString(); } /** @@ -516,6 +544,15 @@ public: return new HashKey(&key, sizeof(key)); } + /** Converts the prefix into the type used internally by the + * inter-thread communication. + */ + void ConvertToThreadingValue(threading::Value::subnet_t* v) const + { + v->length = length; + prefix.ConvertToThreadingValue(&v->prefix); + } + unsigned int MemoryAllocation() const { return padded_sizeof(*this); } /** diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 0d89e5ce99..4b8f527f2b 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -681,7 +681,7 @@ RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip, if ( ! initialized ) reporter->InternalError("remote serializer not initialized"); - if ( ip.GetFamily() == IPAddr::IPv6 ) + if ( ip.GetFamily() == IPv6 ) Error("inter-Bro communication not supported over IPv6"); const uint32* bytes; @@ -1238,7 +1238,7 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl) if ( ! initialized ) reporter->InternalError("remote serializer not initialized"); - if ( ip.GetFamily() == IPAddr::IPv6 ) + if ( ip.GetFamily() == IPv6 ) Error("inter-Bro communication not supported over IPv6"); const uint32* bytes; diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index a90bc83293..da12b1b679 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -1082,7 +1082,7 @@ static bool val_to_maskedval(Val* v, maskedvalue_list* append_to) bool is_v4_mask = m[0] == 0xffffffff && m[1] == m[0] && m[2] == m[0]; - if ( v->AsSubNet().Prefix().GetFamily() == IPAddr::IPv4 && + if ( v->AsSubNet().Prefix().GetFamily() == IPv4 && is_v4_mask ) { mval->val = ntohl(*n); diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index af3d9b44c2..ef2dc80cd7 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -250,9 +250,9 @@ bool BinarySerializationFormat::Read(IPAddr* addr, const char* tag) } if ( n == 1 ) - *addr = IPAddr(IPAddr::IPv4, raw, IPAddr::Network); + *addr = IPAddr(IPv4, raw, IPAddr::Network); else - *addr = IPAddr(IPAddr::IPv6, raw, IPAddr::Network); + *addr = IPAddr(IPv6, raw, IPAddr::Network); return true; } @@ -269,6 +269,33 @@ bool BinarySerializationFormat::Read(IPPrefix* prefix, const char* tag) return true; } +bool BinarySerializationFormat::Read(struct in_addr* addr, const char* tag) + { + uint32_t* bytes = (uint32_t*) &addr->s_addr; + + if ( ! Read(&bytes[0], "addr4") ) + return false; + + bytes[0] = htonl(bytes[0]); + return true; + } + +bool BinarySerializationFormat::Read(struct in6_addr* addr, const char* tag) + { + uint32_t* bytes = (uint32_t*) &addr->s6_addr; + + for ( int i = 0; i < 4; ++i ) + { + if ( ! Read(&bytes[i], "addr6-part") ) + return false; + + bytes[i] = htonl(bytes[i]); + } + + return true; + } + + bool BinarySerializationFormat::Write(char v, const char* tag) { DBG_LOG(DBG_SERIAL, "Write char %s [%s]", fmt_bytes(&v, 1), tag); @@ -362,6 +389,31 @@ bool BinarySerializationFormat::Write(const IPPrefix& prefix, const char* tag) return Write(prefix.Prefix(), "prefix") && Write(prefix.Length(), "width"); } +bool BinarySerializationFormat::Write(struct in_addr& addr, const char* tag) + { + const uint32_t* bytes; + bytes = (uint32_t*) &addr.s_addr; + + if ( ! Write(ntohl(bytes[0]), "addr4") ) + return false; + + return true; + } + +bool BinarySerializationFormat::Write(struct in6_addr& addr, const char* tag) + { + const uint32_t* bytes; + bytes = (uint32_t*) &addr.s6_addr; + + for ( int i = 0; i < 4; ++i ) + { + if ( ! Write(ntohl(bytes[i]), "addr6-part") ) + return false; + } + + return true; + } + bool BinarySerializationFormat::WriteOpenTag(const char* tag) { return true; @@ -464,6 +516,18 @@ bool XMLSerializationFormat::Read(IPPrefix* prefix, const char* tag) return false; } +bool XMLSerializationFormat::Read(struct in_addr* addr, const char* tag) + { + reporter->InternalError("no reading of xml"); + return false; + } + +bool XMLSerializationFormat::Read(struct in6_addr* addr, const char* tag) + { + reporter->InternalError("no reading of xml"); + return false; + } + bool XMLSerializationFormat::Write(char v, const char* tag) { return WriteElem(tag, "char", &v, 1); @@ -556,6 +620,18 @@ bool XMLSerializationFormat::Write(const IPPrefix& prefix, const char* tag) return false; } +bool XMLSerializationFormat::Write(struct in_addr& addr, const char* tag) + { + reporter->InternalError("XML output of in_addr not implemented"); + return false; + } + +bool XMLSerializationFormat::Write(struct in6_addr& addr, const char* tag) + { + reporter->InternalError("XML output of in6_addr not implemented"); + return false; + } + bool XMLSerializationFormat::WriteEncodedString(const char* s, int len) { while ( len-- ) diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index f5eb77c608..ba5ad195a2 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -9,6 +9,9 @@ using namespace std; #include "util.h" +class IPAddr; +class IPPrefix; + // Abstract base class. class SerializationFormat { public: @@ -30,6 +33,8 @@ public: virtual bool Read(string* s, const char* tag) = 0; virtual bool Read(IPAddr* addr, const char* tag) = 0; virtual bool Read(IPPrefix* prefix, const char* tag) = 0; + virtual bool Read(struct in_addr* addr, const char* tag) = 0; + virtual bool Read(struct in6_addr* addr, const char* tag) = 0; // Returns number of raw bytes read since last call to StartRead(). int BytesRead() const { return bytes_read; } @@ -54,6 +59,8 @@ public: virtual bool Write(const string& s, const char* tag) = 0; virtual bool Write(const IPAddr& addr, const char* tag) = 0; virtual bool Write(const IPPrefix& prefix, const char* tag) = 0; + virtual bool Write(struct in_addr& addr, const char* tag) = 0; + virtual bool Write(struct in6_addr& addr, const char* tag) = 0; virtual bool WriteOpenTag(const char* tag) = 0; virtual bool WriteCloseTag(const char* tag) = 0; @@ -96,6 +103,8 @@ public: virtual bool Read(string* s, const char* tag); virtual bool Read(IPAddr* addr, const char* tag); virtual bool Read(IPPrefix* prefix, const char* tag); + virtual bool Read(struct in_addr* addr, const char* tag); + virtual bool Read(struct in6_addr* addr, const char* tag); virtual bool Write(int v, const char* tag); virtual bool Write(uint16 v, const char* tag); virtual bool Write(uint32 v, const char* tag); @@ -109,6 +118,8 @@ public: virtual bool Write(const string& s, const char* tag); virtual bool Write(const IPAddr& addr, const char* tag); virtual bool Write(const IPPrefix& prefix, const char* tag); + virtual bool Write(struct in_addr& addr, const char* tag); + virtual bool Write(struct in6_addr& addr, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -133,6 +144,8 @@ public: virtual bool Write(const string& s, const char* tag); virtual bool Write(const IPAddr& addr, const char* tag); virtual bool Write(const IPPrefix& prefix, const char* tag); + virtual bool Write(struct in_addr& addr, const char* tag); + virtual bool Write(struct in6_addr& addr, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -150,6 +163,8 @@ public: virtual bool Read(string* s, const char* tag); virtual bool Read(IPAddr* addr, const char* tag); virtual bool Read(IPPrefix* prefix, const char* tag); + virtual bool Read(struct in_addr* addr, const char* tag); + virtual bool Read(struct in6_addr* addr, const char* tag); private: // Encodes non-printable characters. diff --git a/src/Val.cc b/src/Val.cc index db6e9eb23a..83bbc59b9d 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -606,7 +606,7 @@ ID* MutableVal::Bind() const ip = htonl(0x7f000001); // 127.0.0.1 safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#", - IPAddr(IPAddr::IPv4, &ip, IPAddr::Network)->AsString().c_str(), + IPAddr(IPv4, &ip, IPAddr::Network)->AsString().c_str(), getpid()); #else safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#", host, getpid()); @@ -864,12 +864,12 @@ AddrVal::AddrVal(const char* text) : Val(TYPE_ADDR) AddrVal::AddrVal(uint32 addr) : Val(TYPE_ADDR) { // ### perhaps do gethostbyaddr here? - val.addr_val = new IPAddr(IPAddr::IPv4, &addr, IPAddr::Network); + val.addr_val = new IPAddr(IPv4, &addr, IPAddr::Network); } AddrVal::AddrVal(const uint32 addr[4]) : Val(TYPE_ADDR) { - val.addr_val = new IPAddr(IPAddr::IPv6, addr, IPAddr::Network); + val.addr_val = new IPAddr(IPv6, addr, IPAddr::Network); } AddrVal::AddrVal(const IPAddr& addr) : Val(TYPE_ADDR) @@ -889,7 +889,7 @@ unsigned int AddrVal::MemoryAllocation() const Val* AddrVal::SizeVal() const { - if ( val.addr_val->GetFamily() == IPAddr::IPv4 ) + if ( val.addr_val->GetFamily() == IPv4 ) return new Val(32, TYPE_COUNT); else return new Val(128, TYPE_COUNT); @@ -933,13 +933,13 @@ SubNetVal::SubNetVal(const char* text, int width) : Val(TYPE_SUBNET) SubNetVal::SubNetVal(uint32 addr, int width) : Val(TYPE_SUBNET) { - IPAddr a(IPAddr::IPv4, &addr, IPAddr::Network); + IPAddr a(IPv4, &addr, IPAddr::Network); val.subnet_val = new IPPrefix(a, width); } SubNetVal::SubNetVal(const uint32* addr, int width) : Val(TYPE_SUBNET) { - IPAddr a(IPAddr::IPv6, addr, IPAddr::Network); + IPAddr a(IPv6, addr, IPAddr::Network); val.subnet_val = new IPPrefix(a, width); } @@ -953,6 +953,16 @@ SubNetVal::~SubNetVal() delete val.subnet_val; } +const IPAddr& SubNetVal::Prefix() const + { + return val.subnet_val->Prefix(); + } + +int SubNetVal::Width() const + { + return val.subnet_val->Length(); + } + unsigned int SubNetVal::MemoryAllocation() const { return padded_sizeof(*this) + val.subnet_val->MemoryAllocation(); @@ -978,7 +988,7 @@ IPAddr SubNetVal::Mask() const uint32 m[4]; for ( unsigned int i = 0; i < 4; ++i ) m[i] = 0; - IPAddr rval(IPAddr::IPv6, m, IPAddr::Host); + IPAddr rval(IPv6, m, IPAddr::Host); return rval; } @@ -994,7 +1004,7 @@ IPAddr SubNetVal::Mask() const while ( ++mp < m + 4 ) *mp = 0; - IPAddr rval(IPAddr::IPv6, m, IPAddr::Host); + IPAddr rval(IPv6, m, IPAddr::Host); return rval; } diff --git a/src/Val.h b/src/Val.h index 415996d97a..e939e51cdc 100644 --- a/src/Val.h +++ b/src/Val.h @@ -513,10 +513,6 @@ protected: #define UDP_PORT_MASK 0x20000 #define ICMP_PORT_MASK 0x30000 -typedef enum { - TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP, -} TransportProto; - class PortVal : public Val { public: // Constructors - both take the port number in host order. @@ -588,8 +584,8 @@ public: Val* SizeVal() const; - const IPAddr& Prefix() const { return val.subnet_val->Prefix(); } - int Width() const { return val.subnet_val->Length(); } + const IPAddr& Prefix() const; + int Width() const; IPAddr Mask() const; bool Contains(const IPAddr& addr) const; diff --git a/src/bro.bif b/src/bro.bif index ff06288940..684b888202 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1949,7 +1949,7 @@ function is_local_interface%(ip: addr%) : bool if ( ent ) { for ( unsigned int len = 0; ent->h_addr_list[len]; ++len ) - addrs.push_back(IPAddr(IPAddr::IPv4, (uint32*)ent->h_addr_list[len], + addrs.push_back(IPAddr(IPv4, (uint32*)ent->h_addr_list[len], IPAddr::Network)); } @@ -1958,7 +1958,7 @@ function is_local_interface%(ip: addr%) : bool if ( ent ) { for ( unsigned int len = 0; ent->h_addr_list[len]; ++len ) - addrs.push_back(IPAddr(IPAddr::IPv6, (uint32*)ent->h_addr_list[len], + addrs.push_back(IPAddr(IPv6, (uint32*)ent->h_addr_list[len], IPAddr::Network)); } @@ -2024,7 +2024,7 @@ function gethostname%(%) : string ## Returns: true if *a* is an IPv4 address, else false. function is_v4_addr%(a: addr%): bool %{ - if ( a->AsAddr().GetFamily() == IPAddr::IPv4 ) + if ( a->AsAddr().GetFamily() == IPv4 ) return new Val(1, TYPE_BOOL); else return new Val(0, TYPE_BOOL); @@ -2037,7 +2037,7 @@ function is_v4_addr%(a: addr%): bool ## Returns: true if *a* is an IPv6 address, else false. function is_v6_addr%(a: addr%): bool %{ - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) return new Val(1, TYPE_BOOL); else return new Val(0, TYPE_BOOL); @@ -3522,7 +3522,7 @@ function lookup_location%(a: addr%) : geo_location } #ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( geoip_v6 && a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( geoip_v6 && a->AsAddr().GetFamily() == IPv6 ) { geoipv6_t ga; a->AsAddr().CopyIPv6(&ga); @@ -3534,7 +3534,7 @@ function lookup_location%(a: addr%) : geo_location else #endif - if ( geoip && a->AsAddr().GetFamily() == IPAddr::IPv4 ) + if ( geoip && a->AsAddr().GetFamily() == IPv4 ) { const uint32* bytes; a->AsAddr().GetBytes(&bytes); @@ -3617,7 +3617,7 @@ function lookup_asn%(a: addr%) : count { // IPv6 support showed up in 1.4.5. #ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) { geoipv6_t ga; a->AsAddr().CopyIPv6(&ga); @@ -3626,7 +3626,7 @@ function lookup_asn%(a: addr%) : count else #endif - if ( a->AsAddr().GetFamily() == IPAddr::IPv4 ) + if ( a->AsAddr().GetFamily() == IPv4 ) { const uint32* bytes; a->AsAddr().GetBytes(&bytes); @@ -5353,7 +5353,7 @@ function preserve_prefix%(a: addr, width: count%): any AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; if ( ip_anon ) { - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) builtin_error("preserve_prefix() not supported for IPv6 addresses"); else { @@ -5382,7 +5382,7 @@ function preserve_subnet%(a: subnet%): any AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; if ( ip_anon ) { - if ( a->AsSubNet().Prefix().GetFamily() == IPAddr::IPv6 ) + if ( a->AsSubNet().Prefix().GetFamily() == IPv6 ) builtin_error("preserve_subnet() not supported for IPv6 addresses"); else { @@ -5418,7 +5418,7 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr if ( anon_class < 0 || anon_class >= NUM_ADDR_ANONYMIZATION_CLASSES ) builtin_error("anonymize_addr(): invalid ip addr anonymization class"); - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) { builtin_error("anonymize_addr() not supported for IPv6 addresses"); return 0; diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index ca9ec1c3c4..0753296cb4 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -862,11 +862,11 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty) break; case TYPE_SUBNET: - lval->val.subnet_val = new IPPrefix(val->AsSubNet()); + val->AsSubNet().ConvertToThreadingValue(&lval->val.subnet_val); break; case TYPE_ADDR: - lval->val.addr_val = new IPAddr(val->AsAddr()); + val->AsAddr().ConvertToThreadingValue(&lval->val.addr_val); break; case TYPE_DOUBLE: diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index f4e48ebaef..7c71c09604 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -242,4 +242,40 @@ bool WriterBackend::DoHeartbeat(double network_time, double current_time) return true; } +string WriterBackend::Render(const threading::Value::addr_t& addr) const + { + if ( addr.family == IPv4 ) + { + char s[INET_ADDRSTRLEN]; + + if ( inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) == NULL ) + return ""; + else + return s; + } + else + { + char s[INET6_ADDRSTRLEN]; + + if ( inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) == NULL ) + return ""; + else + return s; + } + } + +string WriterBackend::Render(const threading::Value::subnet_t& subnet) const + { + char l[16]; + + if ( subnet.prefix.family == IPv4 ) + modp_uitoa10(subnet.length - 96, l); + else + modp_uitoa10(subnet.length, l); + + string s = Render(subnet.prefix) + "/" + l; + + return s; + } + diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index f9653d6b69..efb3b5d95e 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -158,6 +158,21 @@ public: bool FinishedRotation(string new_name, string old_name, double open, double close, bool terminating); + /** Helper method to render an IP address as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + string Render(const threading::Value::addr_t& addr) const; + + /** Helper method to render an subnet value as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + string Render(const threading::Value::subnet_t& subnet) const; protected: /** diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index e5bfc205be..0759e60a82 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -177,11 +177,11 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) break; case TYPE_SUBNET: - desc->Add(*val->val.subnet_val); + desc->Add(Render(val->val.subnet_val)); break; case TYPE_ADDR: - desc->Add(*val->val.addr_val); + desc->Add(Render(val->val.addr_val)); break; case TYPE_TIME: diff --git a/src/net_util.cc b/src/net_util.cc index f78246c634..5e403a349f 100644 --- a/src/net_util.cc +++ b/src/net_util.cc @@ -162,8 +162,8 @@ const char* fmt_conn_id(const IPAddr& src_addr, uint32 src_port, const char* fmt_conn_id(const uint32* src_addr, uint32 src_port, const uint32* dst_addr, uint32 dst_port) { - IPAddr src(IPAddr::IPv6, src_addr, IPAddr::Network); - IPAddr dst(IPAddr::IPv6, dst_addr, IPAddr::Network); + IPAddr src(IPv6, src_addr, IPAddr::Network); + IPAddr dst(IPv6, dst_addr, IPAddr::Network); return fmt_conn_id(src, src_port, dst, dst_port); } diff --git a/src/net_util.h b/src/net_util.h index 8787340328..f61340869a 100644 --- a/src/net_util.h +++ b/src/net_util.h @@ -5,6 +5,13 @@ #include "config.h" +// Define first. +typedef enum { + TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP, +} TransportProto; + +typedef enum { IPv4, IPv6 } IPFamily; + #include #include @@ -21,7 +28,6 @@ #include #include "util.h" -#include "IPAddr.h" #ifdef HAVE_NETINET_IP6_H #include @@ -58,6 +64,8 @@ inline int seq_delta(uint32 a, uint32 b) return int(a-b); } +class IPAddr; + // Returns the ones-complement checksum of a chunk of b short-aligned bytes. extern int ones_complement_checksum(const void* p, int b, uint32 sum); extern int ones_complement_checksum(const IPAddr& a, uint32 sum); diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index d3735e34f3..32569a5442 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -30,12 +30,6 @@ Value::~Value() && present ) delete val.string_val; - if ( type == TYPE_ADDR && present ) - delete val.addr_val; - - if ( type == TYPE_SUBNET && present ) - delete val.subnet_val; - if ( type == TYPE_TABLE && present ) { for ( int i = 0; i < val.set_val.size; i++ ) @@ -130,8 +124,8 @@ bool Value::Read(SerializationFormat* fmt) if ( ! (fmt->Read(&val.port_val.port, "port") && fmt->Read(&proto, "proto") ) ) { return false; } - - switch (proto) { + + switch ( proto ) { case 0: val.port_val.proto = TRANSPORT_UNKNOWN; break; @@ -147,20 +141,35 @@ bool Value::Read(SerializationFormat* fmt) default: return false; } + return true; } - - case TYPE_SUBNET: - { - val.subnet_val = new IPPrefix; - return fmt->Read(val.subnet_val, "subnet"); - } - case TYPE_ADDR: { - val.addr_val = new IPAddr; - return fmt->Read(val.addr_val, "addr"); + int family; + + if ( ! fmt->Read(&family, "addr-family") ) + return false; + + switch ( family ) { + case 4: + val.addr_val.family = IPv4; + return fmt->Read(&val.addr_val.in.in4, "addr-in4"); + + case 6: + val.addr_val.family = IPv6; + return fmt->Read(&val.addr_val.in.in6, "addr-in6"); + + } + + // Can't be reached. + abort(); + } + + case TYPE_SUBNET: + { + // FIXME. } case TYPE_DOUBLE: @@ -239,13 +248,27 @@ bool Value::Write(SerializationFormat* fmt) const return fmt->Write(val.uint_val, "uint"); case TYPE_PORT: - return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); + return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); case TYPE_SUBNET: - return fmt->Write(*val.subnet_val, "subnet"); + return false; // FIXME. case TYPE_ADDR: - return fmt->Write(*val.addr_val, "addr"); + { + switch ( val.addr_val.family ) { + case IPv4: + return fmt->Write((int)4, "addr-family") + && fmt->Write(val.addr_val.in.in4, "addr-in4"); + + case IPv6: + return fmt->Write((int)6, "addr-family") + && fmt->Write(val.addr_val.in.in6, "addr-in6"); + break; + } + + // Can't be reached. + abort(); + } case TYPE_DOUBLE: case TYPE_TIME: diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index c53ca37dc0..adff2035d7 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -2,10 +2,13 @@ #ifndef THREADING_SERIALIZATIONTYPES_H #define THREADING_SERIALIZATIONTYPES_H -#include "../RemoteSerializer.h" - using namespace std; +#include "Type.h" +#include "net_util.h" + +class SerializationFormat; + namespace threading { /** @@ -62,6 +65,16 @@ struct Value { typedef set_t vec_t; struct port_t { bro_uint_t port; TransportProto proto; }; + struct addr_t { + IPFamily family; + union { + struct in_addr in4; + struct in6_addr in6; + } in; + }; + + struct subnet_t { addr_t prefix; uint8_t length; }; + /** * This union is a subset of BroValUnion, including only the types we * can log directly. See IsCompatibleType(). @@ -73,8 +86,8 @@ struct Value { double double_val; set_t set_val; vec_t vector_val; - IPAddr* addr_val; - IPPrefix* subnet_val; + addr_t addr_val; + subnet_t subnet_val; string* string_val; } val; @@ -120,6 +133,7 @@ struct Value { static bool IsCompatibleType(BroType* t, bool atomic_only=false); private: +friend class ::IPAddr; Value(const Value& other) { } // Disabled. }; From df874f0f6284ea8c916633754430a0e37f45c69b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 29 Feb 2012 14:32:29 -0800 Subject: [PATCH 24/56] A number of bugfixes for the recent threading updates. All tests pass now except one: scripts.base.frameworks.metrics.cluster-intermediate-update Couldn't figure out yet why that still fails. --- src/IPAddr.h | 6 ++--- src/SerializationFormat.cc | 15 +++++------ src/SerializationFormat.h | 12 ++++----- src/threading/Manager.cc | 7 +++-- src/threading/SerialTypes.cc | 51 +++++++++++++++++++++++++++++++----- 5 files changed, 63 insertions(+), 28 deletions(-) diff --git a/src/IPAddr.h b/src/IPAddr.h index 67d6f2112e..8e1921e07b 100644 --- a/src/IPAddr.h +++ b/src/IPAddr.h @@ -47,7 +47,7 @@ public: * * @param in6 The IPv6 address. */ - IPAddr(const in4_addr& in4) + explicit IPAddr(const in4_addr& in4) { memcpy(in6.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix)); memcpy(&in6.s6_addr[12], &in4.s_addr, sizeof(in4.s_addr)); @@ -58,7 +58,7 @@ public: * * @param in6 The IPv6 address. */ - IPAddr(const in6_addr& arg_in6) : in6(arg_in6) { } + explicit IPAddr(const in6_addr& arg_in6) : in6(arg_in6) { } /** * Constructs an address instance from a string representation. @@ -523,8 +523,6 @@ public: */ string AsString() const; - /** Converts the address into the type used internally by the inter-thread communicastion. - */ operator std::string() const { return AsString(); } /** diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index ef2dc80cd7..10dd4f29ea 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -295,7 +295,6 @@ bool BinarySerializationFormat::Read(struct in6_addr* addr, const char* tag) return true; } - bool BinarySerializationFormat::Write(char v, const char* tag) { DBG_LOG(DBG_SERIAL, "Write char %s [%s]", fmt_bytes(&v, 1), tag); @@ -389,10 +388,9 @@ bool BinarySerializationFormat::Write(const IPPrefix& prefix, const char* tag) return Write(prefix.Prefix(), "prefix") && Write(prefix.Length(), "width"); } -bool BinarySerializationFormat::Write(struct in_addr& addr, const char* tag) +bool BinarySerializationFormat::Write(const struct in_addr& addr, const char* tag) { - const uint32_t* bytes; - bytes = (uint32_t*) &addr.s_addr; + const uint32_t* bytes = (uint32_t*) &addr.s_addr; if ( ! Write(ntohl(bytes[0]), "addr4") ) return false; @@ -400,10 +398,9 @@ bool BinarySerializationFormat::Write(struct in_addr& addr, const char* tag) return true; } -bool BinarySerializationFormat::Write(struct in6_addr& addr, const char* tag) +bool BinarySerializationFormat::Write(const struct in6_addr& addr, const char* tag) { - const uint32_t* bytes; - bytes = (uint32_t*) &addr.s6_addr; + const uint32_t* bytes = (uint32_t*) &addr.s6_addr; for ( int i = 0; i < 4; ++i ) { @@ -620,13 +617,13 @@ bool XMLSerializationFormat::Write(const IPPrefix& prefix, const char* tag) return false; } -bool XMLSerializationFormat::Write(struct in_addr& addr, const char* tag) +bool XMLSerializationFormat::Write(const struct in_addr& addr, const char* tag) { reporter->InternalError("XML output of in_addr not implemented"); return false; } -bool XMLSerializationFormat::Write(struct in6_addr& addr, const char* tag) +bool XMLSerializationFormat::Write(const struct in6_addr& addr, const char* tag) { reporter->InternalError("XML output of in6_addr not implemented"); return false; diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index ba5ad195a2..f270b61bae 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -59,8 +59,8 @@ public: virtual bool Write(const string& s, const char* tag) = 0; virtual bool Write(const IPAddr& addr, const char* tag) = 0; virtual bool Write(const IPPrefix& prefix, const char* tag) = 0; - virtual bool Write(struct in_addr& addr, const char* tag) = 0; - virtual bool Write(struct in6_addr& addr, const char* tag) = 0; + virtual bool Write(const struct in_addr& addr, const char* tag) = 0; + virtual bool Write(const struct in6_addr& addr, const char* tag) = 0; virtual bool WriteOpenTag(const char* tag) = 0; virtual bool WriteCloseTag(const char* tag) = 0; @@ -118,8 +118,8 @@ public: virtual bool Write(const string& s, const char* tag); virtual bool Write(const IPAddr& addr, const char* tag); virtual bool Write(const IPPrefix& prefix, const char* tag); - virtual bool Write(struct in_addr& addr, const char* tag); - virtual bool Write(struct in6_addr& addr, const char* tag); + virtual bool Write(const struct in_addr& addr, const char* tag); + virtual bool Write(const struct in6_addr& addr, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -144,8 +144,8 @@ public: virtual bool Write(const string& s, const char* tag); virtual bool Write(const IPAddr& addr, const char* tag); virtual bool Write(const IPPrefix& prefix, const char* tag); - virtual bool Write(struct in_addr& addr, const char* tag); - virtual bool Write(struct in6_addr& addr, const char* tag); + virtual bool Write(const struct in_addr& addr, const char* tag); + virtual bool Write(const struct in6_addr& addr, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d008d2e5e8..24e100fe37 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -106,8 +106,11 @@ void Manager::Process() Message* msg = t->RetrieveOut(); - if ( msg->Process() && network_time ) - did_process = true; + if ( msg->Process() ) + { + if ( network_time ) + did_process = true; + } else { diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index 32569a5442..a5692b2ffd 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -147,7 +147,7 @@ bool Value::Read(SerializationFormat* fmt) case TYPE_ADDR: { - int family; + char family; if ( ! fmt->Read(&family, "addr-family") ) return false; @@ -169,7 +169,27 @@ bool Value::Read(SerializationFormat* fmt) case TYPE_SUBNET: { - // FIXME. + char length; + char family; + + if ( ! (fmt->Read(&length, "subnet-len") && fmt->Read(&family, "subnet-family")) ) + return false; + + switch ( family ) { + case 4: + val.subnet_val.length = (uint8_t)length; + val.subnet_val.prefix.family = IPv4; + return fmt->Read(&val.subnet_val.prefix.in.in4, "subnet-in4"); + + case 6: + val.subnet_val.length = (uint8_t)length; + val.subnet_val.prefix.family = IPv6; + return fmt->Read(&val.subnet_val.prefix.in.in6, "subnet-in6"); + + } + + // Can't be reached. + abort(); } case TYPE_DOUBLE: @@ -250,18 +270,15 @@ bool Value::Write(SerializationFormat* fmt) const case TYPE_PORT: return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); - case TYPE_SUBNET: - return false; // FIXME. - case TYPE_ADDR: { switch ( val.addr_val.family ) { case IPv4: - return fmt->Write((int)4, "addr-family") + return fmt->Write((char)4, "addr-family") && fmt->Write(val.addr_val.in.in4, "addr-in4"); case IPv6: - return fmt->Write((int)6, "addr-family") + return fmt->Write((char)6, "addr-family") && fmt->Write(val.addr_val.in.in6, "addr-in6"); break; } @@ -270,6 +287,26 @@ bool Value::Write(SerializationFormat* fmt) const abort(); } + case TYPE_SUBNET: + { + if ( ! fmt->Write((char)val.subnet_val.length, "subnet-length") ) + return false; + + switch ( val.subnet_val.prefix.family ) { + case IPv4: + return fmt->Write((char)4, "subnet-family") + && fmt->Write(val.subnet_val.prefix.in.in4, "subnet-in4"); + + case IPv6: + return fmt->Write((char)6, "subnet-family") + && fmt->Write(val.subnet_val.prefix.in.in6, "subnet-in6"); + break; + } + + // Can't be reached. + abort(); + } + case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: From 6df9004423dc0ea06aebe8201f4ca4bb937eda09 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 29 Feb 2012 16:06:54 -0800 Subject: [PATCH 25/56] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index 3b63c3f1e7..4d2dde5573 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 3b63c3f1e7d915b1bda16862bfa4a8593ffc38f6 +Subproject commit 4d2dde55733ed86ea3f2db8df5b78b0bcfbb54c4 From 355c447698dd86fc6a7e78694318a911bb7ee3e4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 29 Feb 2012 16:08:37 -0800 Subject: [PATCH 26/56] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index 4d2dde5573..d6e36c95e0 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 4d2dde55733ed86ea3f2db8df5b78b0bcfbb54c4 +Subproject commit d6e36c95e0335f7cc081191c8612085bd12706f9 From 56dd7918d0475df364b47b6133745bbb598536fc Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 29 Feb 2012 16:46:21 -0800 Subject: [PATCH 27/56] Adding missing includes needed on FreeBSD. --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- src/threading/SerialTypes.h | 4 ++++ 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/aux/binpac b/aux/binpac index 43308aab47..3034da8f08 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 43308aab47a3357ca1885e1b6954154a2744d821 +Subproject commit 3034da8f082b61157e234237993ffd7a95be6e62 diff --git a/aux/bro-aux b/aux/bro-aux index 139cc2e1e0..f53bcb2b49 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 139cc2e1e049c4e1cc7e95f20866102be1d3d599 +Subproject commit f53bcb2b492cb0db3dd288384040abc2ab711767 diff --git a/aux/broccoli b/aux/broccoli index d6e36c95e0..2602eb53e7 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit d6e36c95e0335f7cc081191c8612085bd12706f9 +Subproject commit 2602eb53e70d7f0afae8fac58d7636b9291974a4 diff --git a/aux/broctl b/aux/broctl index e908ba686d..954538514d 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit e908ba686dceb56065bdf569c18dd0f67f662f6b +Subproject commit 954538514d71983e7ef3f0e109960466096e1c1d diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index adff2035d7..db7dc837bd 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -4,6 +4,10 @@ using namespace std; +#include +#include +#include + #include "Type.h" #include "net_util.h" From 6429d1248adf0caa275187d8e204b52638ab3332 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 1 Mar 2012 16:00:30 -0800 Subject: [PATCH 28/56] Prevent manager from busy looping. I saw this with the new threading code but I'm wondering if it also helps with the "high CPU usage with low traffiv volume" problem. --- src/Net.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Net.cc b/src/Net.cc index d93f1e1a85..2ebbfb20ed 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -486,6 +486,8 @@ void net_run() // since Bro timers are not high-precision anyway.) if ( ! using_communication ) usleep(100000); + else + usleep(1000); // Flawfinder says about usleep: // From 554a29b3edd5e1504c5e611c7b7ff9287abae330 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 1 Mar 2012 16:04:34 -0800 Subject: [PATCH 29/56] Preventing busy looping when no threads have been spawned. --- src/threading/Manager.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 24e100fe37..f5770e2dd8 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -10,7 +10,7 @@ Manager::Manager() did_process = true; next_beat = 0; terminating = false; - idle = false; + idle = true; } Manager::~Manager() @@ -58,6 +58,7 @@ void Manager::AddThread(BasicThread* thread) { DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str()); all_threads.push_back(thread); + idle = false; } void Manager::AddMsgThread(MsgThread* thread) From c0678e7e1f649d4c69701eb7d03fd28893e12dd4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 17:14:58 -0800 Subject: [PATCH 30/56] Fixing problem logging remotely when local logging was turned off. For that, moved the remote logging from the Manager to the WriterFrontend. That also simplifies the Manager a bit. --- src/RemoteSerializer.cc | 13 +++++- src/logging/Manager.cc | 63 ++++++---------------------- src/logging/Manager.h | 3 +- src/logging/WriterFrontend.cc | 79 ++++++++++++++++++++++++++++++----- src/logging/WriterFrontend.h | 18 +++++++- src/threading/BasicThread.cc | 15 ++++--- 6 files changed, 116 insertions(+), 75 deletions(-) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 4b8f527f2b..c6b9623096 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -234,7 +234,7 @@ static const int PRINT_BUFFER_SIZE = 10 * 1024; static const int SOCKBUF_SIZE = 1024 * 1024; // Buffer size for remote-log data. -static const int LOG_BUFFER_SIZE = 50 * 1024; +static const int LOG_BUFFER_SIZE = 512; struct ping_args { uint32 seq; @@ -2587,7 +2587,10 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) || (network_time - last_flush > 1.0) ) { if ( ! FlushLogBuffer(peer) ) + { + delete [] data; return false; + } } // If the data is actually larger than our complete buffer, just send it out. @@ -2631,6 +2634,12 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( current_peer->state == Peer::CLOSING ) return false; +#ifdef USE_PERFTOOLS + // Don't track allocations here, they'll be released only after the + // main loop exists. And it's just a tiny amount anyway. + HeapLeakChecker::Disabler disabler; +#endif + assert(current_args); EnumVal* id_val = 0; @@ -2666,7 +2675,7 @@ bool RemoteSerializer::ProcessLogCreateWriter() id_val = new EnumVal(id, BifType::Enum::Log::ID); writer_val = new EnumVal(writer, BifType::Enum::Log::Writer); - if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields) ) + if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields, true, false) ) goto error; Unref(id_val); diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 0753296cb4..14fb3428fe 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -753,64 +753,25 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) for ( int j = 0; j < filter->num_fields; ++j ) arg_fields[j] = new Field(*filter->fields[j]); - if ( filter->remote ) - remote_serializer->SendLogCreateWriter(stream->id, - filter->writer, - path, - filter->num_fields, - arg_fields); + writer = CreateWriter(stream->id, filter->writer, + path, filter->num_fields, + arg_fields, filter->local, filter->remote); - if ( filter->local ) + if ( ! writer ) { - writer = CreateWriter(stream->id, filter->writer, - path, filter->num_fields, - arg_fields); - - if ( ! writer ) - { - Unref(columns); - return false; - } + Unref(columns); + return false; } - else - { - // Insert a null pointer into the map to make - // sure we don't try creating it again. - stream->writers.insert(Stream::WriterMap::value_type( - Stream::WriterPathPair(filter->writer->AsEnum(), path), 0)); - for( int i = 0; i < filter->num_fields; ++i) - delete arg_fields[i]; - - delete [] arg_fields; - } } // Alright, can do the write now. - if ( filter->local || filter->remote ) - { - threading::Value** vals = RecordToFilterVals(stream, filter, columns); - - if ( filter->remote ) - remote_serializer->SendLogWrite(stream->id, - filter->writer, - path, - filter->num_fields, - vals); - - if ( filter->local ) - { - // Write takes ownership of vals. - assert(writer); - writer->Write(filter->num_fields, vals); - } - - else - DeleteVals(filter->num_fields, vals); - - } + threading::Value** vals = RecordToFilterVals(stream, filter, columns); + // Write takes ownership of vals. + assert(writer); + writer->Write(filter->num_fields, vals); #ifdef DEBUG DBG_LOG(DBG_LOGGING, "Wrote record to filter '%s' on stream '%s'", @@ -976,7 +937,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, } WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, const Field* const* fields) + int num_fields, const Field* const* fields, bool local, bool remote) { Stream* stream = FindStream(id); @@ -992,7 +953,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, // return it. return w->second->writer; - WriterFrontend* writer_obj = new WriterFrontend(writer->AsEnum()); + WriterFrontend* writer_obj = new WriterFrontend(id, writer, local, remote); assert(writer_obj); writer_obj->Init(path, num_fields, fields); diff --git a/src/logging/Manager.h b/src/logging/Manager.h index d931bfaef8..bf097c5e1a 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -159,7 +159,8 @@ protected: // Takes ownership of fields. WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, const threading::Field* const* fields); + int num_fields, const threading::Field* const* fields, + bool local, bool remote); // Takes ownership of values.. bool Write(EnumVal* id, EnumVal* writer, string path, diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 02f1a188d8..26e8eaf22e 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -99,21 +99,36 @@ public: using namespace logging; -WriterFrontend::WriterFrontend(bro_int_t type) +WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote) { + stream = arg_stream; + writer = arg_writer; + Ref(stream); + Ref(writer); + disabled = initialized = false; buf = true; + local = arg_local; + remote = arg_remote; write_buffer = 0; write_buffer_pos = 0; ty_name = ""; - backend = log_mgr->CreateBackend(this, type); - assert(backend); - backend->Start(); + if ( local ) + { + backend = log_mgr->CreateBackend(this, writer->AsEnum()); + assert(backend); + backend->Start(); + } + + else + backend = 0; } WriterFrontend::~WriterFrontend() { + Unref(stream); + Unref(writer); } string WriterFrontend::Name() const @@ -128,7 +143,9 @@ void WriterFrontend::Stop() { FlushWriteBuffer(); SetDisable(); - backend->Stop(); + + if ( backend ) + backend->Stop(); } void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* const * arg_fields) @@ -144,7 +161,17 @@ void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* cons fields = arg_fields; initialized = true; - backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + + if ( backend ) + backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + + if ( remote ) + remote_serializer->SendLogCreateWriter(stream, + writer, + arg_path, + arg_num_fields, + arg_fields); + } void WriterFrontend::Write(int num_fields, Value** vals) @@ -152,6 +179,19 @@ void WriterFrontend::Write(int num_fields, Value** vals) if ( disabled ) return; + if ( remote ) + remote_serializer->SendLogWrite(stream, + writer, + path, + num_fields, + vals); + + if ( ! backend ) + { + DeleteVals(vals); + return; + } + if ( ! write_buffer ) { // Need new buffer. @@ -173,7 +213,8 @@ void WriterFrontend::FlushWriteBuffer() // Nothing to do. return; - backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer)); + if ( backend ) + backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer)); // Clear buffer (no delete, we pass ownership to child thread.) write_buffer = 0; @@ -187,7 +228,8 @@ void WriterFrontend::SetBuf(bool enabled) buf = enabled; - backend->SendIn(new SetBufMessage(backend, enabled)); + if ( backend ) + backend->SendIn(new SetBufMessage(backend, enabled)); if ( ! buf ) // Make sure no longer buffer any still queued data. @@ -200,7 +242,9 @@ void WriterFrontend::Flush() return; FlushWriteBuffer(); - backend->SendIn(new FlushMessage(backend)); + + if ( backend ) + backend->SendIn(new FlushMessage(backend)); } void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating) @@ -209,7 +253,9 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool return; FlushWriteBuffer(); - backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); + + if ( backend ) + backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); } void WriterFrontend::Finish() @@ -218,7 +264,18 @@ void WriterFrontend::Finish() return; FlushWriteBuffer(); - backend->SendIn(new FinishMessage(backend)); + + if ( backend ) + backend->SendIn(new FinishMessage(backend)); + } + +void WriterFrontend::DeleteVals(Value** vals) + { + // Note this code is duplicated in Manager::DeleteVals(). + for ( int i = 0; i < num_fields; i++ ) + delete vals[i]; + + delete [] vals; } diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index 4386a15f64..3e05d17c9e 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -25,14 +25,21 @@ public: /** * Constructor. * - * type: The backend writer type, with the value corresponding to the + * stream: The logging stream. + * + * writer: The backend writer type, with the value corresponding to the * script-level \c Log::Writer enum (e.g., \a WRITER_ASCII). The * frontend will internally instantiate a WriterBackend of the * corresponding type. + * + * local: If true, the writer will instantiate a local backend. + * + * remote: If true, the writer will forward all data to remote + * clients. * * Frontends must only be instantiated by the main thread. */ - WriterFrontend(bro_int_t type); + WriterFrontend(EnumVal* stream, EnumVal* writer, bool local, bool remote); /** * Destructor. @@ -187,10 +194,17 @@ public: protected: friend class Manager; + void DeleteVals(threading::Value** vals); + + EnumVal* stream; + EnumVal* writer; + WriterBackend* backend; // The backend we have instanatiated. bool disabled; // True if disabled. bool initialized; // True if initialized. bool buf; // True if buffering is enabled (default). + bool local; // True if logging locally. + bool remote; // True if loggin remotely. string ty_name; // Name of the backend type. Set by the manager. string path; // The log path. diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 51c4f7a3bc..e590b13434 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -20,8 +20,8 @@ BasicThread::BasicThread() terminating = false; pthread = 0; - buf = 0; - buf_len = 1024; + buf_len = 2048; + buf = (char*) malloc(buf_len); name = Fmt("thread-%d", ++thread_counter); @@ -57,9 +57,6 @@ void BasicThread::SetOSName(const string& name) const char* BasicThread::Fmt(const char* format, ...) { - if ( ! buf ) - buf = (char*) malloc(buf_len); - va_list al; va_start(al, format); int n = safe_vsnprintf(buf, buf_len, format, al); @@ -67,13 +64,15 @@ const char* BasicThread::Fmt(const char* format, ...) if ( (unsigned int) n >= buf_len ) { // Not enough room, grow the buffer. - buf_len = n + 32; - buf = (char*) realloc(buf, buf_len); + int tmp_len = n + 32; + char* tmp = (char*) malloc(tmp_len); // Is it portable to restart? va_start(al, format); - n = safe_vsnprintf(buf, buf_len, format, al); + n = safe_vsnprintf(tmp, tmp_len, format, al); va_end(al); + + free(tmp); } return buf; From bf14bd91d7332384931ebd1b42401b8dd0c14754 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 17:16:51 -0800 Subject: [PATCH 31/56] Removing some no longer needed checks. --- src/logging/Manager.cc | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 14fb3428fe..4e97351e57 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -105,9 +105,6 @@ Manager::Stream::~Stream() { WriterInfo* winfo = i->second; - if ( ! winfo ) - continue; - if ( winfo->rotation_timer ) timer_mgr->Cancel(winfo->rotation_timer); @@ -207,7 +204,7 @@ Manager::WriterInfo* Manager::FindWriter(WriterFrontend* writer) { WriterInfo* winfo = i->second; - if ( winfo && winfo->writer == writer ) + if ( winfo->writer == writer ) return winfo; } } @@ -221,7 +218,7 @@ void Manager::RemoveDisabledWriters(Stream* stream) for ( Stream::WriterMap::iterator j = stream->writers.begin(); j != stream->writers.end(); j++ ) { - if ( j->second && j->second->writer->Disabled() ) + if ( j->second->writer->Disabled() ) { j->second->writer->Stop(); delete j->second; @@ -740,7 +737,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) if ( w != stream->writers.end() ) // We know this writer already. - writer = w->second ? w->second->writer : 0; + writer = w->second->writer; else { @@ -948,7 +945,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path)); - if ( w != stream->writers.end() && w->second ) + if ( w != stream->writers.end() ) // If we already have a writer for this. That's fine, we just // return it. return w->second->writer; @@ -1050,8 +1047,7 @@ bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, return false; } - if ( w->second ) - w->second->writer->Write(num_fields, vals); + w->second->writer->Write(num_fields, vals); DBG_LOG(DBG_LOGGING, "Wrote pre-filtered record to path '%s' on stream '%s'", @@ -1072,9 +1068,6 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) { - if ( ! i->second ) - continue; - WriterFrontend* writer = i->second->writer; EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer); @@ -1095,10 +1088,7 @@ bool Manager::SetBuf(EnumVal* id, bool enabled) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - { - if ( i->second ) - i->second->writer->SetBuf(enabled); - } + i->second->writer->SetBuf(enabled); RemoveDisabledWriters(stream); @@ -1116,10 +1106,7 @@ bool Manager::Flush(EnumVal* id) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - { - if ( i->second ) - i->second->writer->Flush(); - } + i->second->writer->Flush(); RemoveDisabledWriters(stream); From 83038d78e0511a0fbed649e514a27904f5071c2f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 17:29:23 -0800 Subject: [PATCH 32/56] Adding new leak tests involving remote logging. --- .../manager-1.metrics.log | 10 +++ .../core.leaks.remote/sender.test.failure.log | 10 +++ .../core.leaks.remote/sender.test.log | 12 +++ .../core.leaks.remote/sender.test.success.log | 9 +++ testing/btest/core/leaks/basic-cluster.bro | 39 +++++++++ testing/btest/core/leaks/remote.bro | 79 +++++++++++++++++++ .../external/scripts/perftools-adapt-paths | 2 +- 7 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log create mode 100644 testing/btest/Baseline/core.leaks.remote/sender.test.failure.log create mode 100644 testing/btest/Baseline/core.leaks.remote/sender.test.log create mode 100644 testing/btest/Baseline/core.leaks.remote/sender.test.success.log create mode 100644 testing/btest/core/leaks/basic-cluster.bro create mode 100644 testing/btest/core/leaks/remote.bro diff --git a/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log new file mode 100644 index 0000000000..42fcd6a526 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path metrics +#fields ts metric_id filter_name index.host index.str index.network value +#types time enum string addr string subnet count +1331256494.591966 TEST_METRIC foo-bar 6.5.4.3 - - 4 +1331256494.591966 TEST_METRIC foo-bar 7.2.1.5 - - 2 +1331256494.591966 TEST_METRIC foo-bar 1.2.3.4 - - 6 diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log new file mode 100644 index 0000000000..5a26f322f4 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test.failure +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.log b/testing/btest/Baseline/core.leaks.remote/sender.test.log new file mode 100644 index 0000000000..9d2ba26f48 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.success.log b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log new file mode 100644 index 0000000000..1b2ed452a0 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log @@ -0,0 +1,9 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test.success +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro new file mode 100644 index 0000000000..a82f52c8b2 --- /dev/null +++ b/testing/btest/core/leaks/basic-cluster.bro @@ -0,0 +1,39 @@ +# Needs perftools support. +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT +# @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run worker-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT +# @TEST-EXEC: btest-bg-run worker-2 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT +# @TEST-EXEC: btest-bg-wait -k 30 +# @TEST-EXEC: btest-diff manager-1/metrics.log + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")], + ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"], +}; +@TEST-END-FILE + +redef Log::default_rotation_interval = 0secs; + +redef enum Metrics::ID += { + TEST_METRIC, +}; + +event bro_init() &priority=5 + { + Metrics::add_filter(TEST_METRIC, + [$name="foo-bar", + $break_interval=3secs]); + + if ( Cluster::local_node_type() == Cluster::WORKER ) + { + Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); + Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); + Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); + } + } diff --git a/testing/btest/core/leaks/remote.bro b/testing/btest/core/leaks/remote.bro new file mode 100644 index 0000000000..fa72ce6024 --- /dev/null +++ b/testing/btest/core/leaks/remote.bro @@ -0,0 +1,79 @@ +# Needs perftools support. +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-wait -k 10 +# @TEST-EXEC: btest-diff sender/test.log +# @TEST-EXEC: btest-diff sender/test.failure.log +# @TEST-EXEC: btest-diff sender/test.success.log +# @TEST-EXEC: cmp receiver/test.log sender/test.log +# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log +# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log + +# This is the common part loaded by both sender and receiver. +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { LOG }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); + Log::add_filter(Test::LOG, [$name="f1", $path="test.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); +} + +##### + +@TEST-START-FILE sender.bro + +module Test; + +@load frameworks/communication/listen + +function fail(rec: Log): bool + { + return rec$status != "success"; + } + +event remote_connection_handshake_done(p: event_peer) + { + Log::add_filter(Test::LOG, [$name="f2", $path="test.failure", $pred=fail]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + + # Log something. + Log::write(Test::LOG, r); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + disconnect(p); + } +@TEST-END-FILE + +@TEST-START-FILE receiver.bro + +##### + +redef Communication::nodes += { + ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T] +}; + +@TEST-END-FILE diff --git a/testing/external/scripts/perftools-adapt-paths b/testing/external/scripts/perftools-adapt-paths index 2eda2477c7..cfecd39993 100755 --- a/testing/external/scripts/perftools-adapt-paths +++ b/testing/external/scripts/perftools-adapt-paths @@ -7,4 +7,4 @@ cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1 -grep -q "No leaks found" $1 +grep -qv "detected leaks of" $1 From 51009b73bcd23c812f8244f55ee5c6e5172598fd Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 18:13:17 -0800 Subject: [PATCH 33/56] Finetuning communication CPU usage. --- .../base/frameworks/cluster/setup-connections.bro | 2 +- src/RemoteSerializer.cc | 14 ++++++++++++++ src/RemoteSerializer.h | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index b5a0d25e1f..20646525be 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -44,7 +44,7 @@ event bro_init() &priority=9 { if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events]; + [$host=n$ip, $connect=F, $class=i, $sync=F, $auth=T, $events=worker2proxy_events]; # accepts connections from the previous one. # (This is not ideal for setups with many proxies) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index c6b9623096..56e27c2104 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -532,6 +532,7 @@ RemoteSerializer::RemoteSerializer() terminating = false; in_sync = 0; last_flush = 0; + received_logs = 0; } RemoteSerializer::~RemoteSerializer() @@ -1353,6 +1354,14 @@ double RemoteSerializer::NextTimestamp(double* local_network_time) { Poll(false); + if ( received_logs > 0 ) + { + // If we processed logs last time, assume there's more. + idle = false; + received_logs = 0; + return timer_mgr->Time(); + } + double et = events.length() ? events[0]->time : -1; double pt = packets.length() ? packets[0]->time : -1; @@ -2744,6 +2753,8 @@ bool RemoteSerializer::ProcessLogWrite() fmt.EndRead(); + ++received_logs; + return true; error: @@ -3385,6 +3396,9 @@ void SocketComm::Run() small_timeout.tv_usec = io->CanWrite() || io->CanRead() ? 1 : 10; + if ( ! io->CanWrite() ) + usleep(10); + int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except, &small_timeout); diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index eabcb18a38..05d25ca525 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -338,6 +338,7 @@ private: int propagate_accesses; bool ignore_accesses; bool terminating; + int received_logs; Peer* source_peer; PeerID id_counter; // Keeps track of assigned IDs. uint32 current_sync_point; From 8eaf40ec18222d54ae4a76f535008951a73d7fca Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 20:24:12 -0800 Subject: [PATCH 34/56] Reverting accidental commit. Thanks, Seth! --- scripts/base/frameworks/cluster/setup-connections.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index 20646525be..b5a0d25e1f 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -44,7 +44,7 @@ event bro_init() &priority=9 { if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, $class=i, $sync=F, $auth=T, $events=worker2proxy_events]; + [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events]; # accepts connections from the previous one. # (This is not ideal for setups with many proxies) From 89a3bb33c8003bd0be1369e756eef31af23cb18e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 8 Mar 2012 20:22:39 -0800 Subject: [PATCH 35/56] Don't assert during shutdown. --- src/RemoteSerializer.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 56e27c2104..f29e907790 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -2561,7 +2561,9 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st if ( ! peer->logs_requested ) return false; - assert(peer->log_buffer); + if ( ! peer->log_buffer ) + // Peer shutting down. + return false; // Serialize the log record entry. From e3f5cbb670a0a59ab80ed58224f23cb053e4cc9f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 16 Mar 2012 09:11:31 -0700 Subject: [PATCH 36/56] Small fixes and tweaks. - Fixing tiny leak. - Fixing threads stat output. --- src/logging/Manager.cc | 5 ++--- src/threading/Manager.cc | 30 +++++++++++++++--------------- src/threading/MsgThread.cc | 4 ++-- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 4e97351e57..74220ecde4 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -677,11 +677,11 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) Val* path_arg; if ( filter->path_val ) - path_arg = filter->path_val; + path_arg = filter->path_val->Ref(); else path_arg = new StringVal(""); - vl.append(path_arg->Ref()); + vl.append(path_arg); Val* rec_arg; BroType* rt = filter->path_func->FType()->Args()->FieldType("rec"); @@ -715,7 +715,6 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) if ( ! filter->path_val ) { - Unref(path_arg); filter->path = v->AsString()->CheckString(); filter->path_val = v->Ref(); } diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index f5770e2dd8..43eb0313f4 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -102,25 +102,25 @@ void Manager::Process() next_beat = 0; } - if ( ! t->HasOut() ) - continue; - - Message* msg = t->RetrieveOut(); - - if ( msg->Process() ) + while ( t->HasOut() ) { - if ( network_time ) - did_process = true; + Message* msg = t->RetrieveOut(); + + if ( msg->Process() ) + { + if ( network_time ) + did_process = true; + } + + else + { + string s = msg->Name() + " failed, terminating thread"; + reporter->Error("%s", s.c_str()); + t->Stop(); } - else - { - string s = msg->Name() + " failed, terminating thread"; - reporter->Error("%s", s.c_str()); - t->Stop(); + delete msg; } - - delete msg; } // fprintf(stderr, "P %.6f %.6f do_beat=%d did_process=%d next_next=%.6f\n", network_time, timer_mgr->Time(), do_beat, (int)did_process, next_beat); diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index b7782b9a05..145e16c57b 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -281,7 +281,7 @@ void MsgThread::GetStats(Stats* stats) { stats->sent_in = cnt_sent_in; stats->sent_out = cnt_sent_out; - stats->pending_in = cnt_sent_in - queue_in.Size(); - stats->pending_out = cnt_sent_out - queue_out.Size(); + stats->pending_in = queue_in.Size(); + stats->pending_out = queue_out.Size(); } From d7c9471818ed60453fc319388277ebaf43939b27 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 23 Mar 2012 15:57:25 -0700 Subject: [PATCH 37/56] Extending queue statistics. --- .../frameworks/cluster/setup-connections.bro | 2 +- src/Stats.cc | 9 ++++- src/logging/WriterFrontend.h | 2 +- src/threading/MsgThread.cc | 2 + src/threading/MsgThread.h | 4 ++ src/threading/Queue.h | 39 +++++++++++++++++++ 6 files changed, 54 insertions(+), 4 deletions(-) diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index b5a0d25e1f..20646525be 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -44,7 +44,7 @@ event bro_init() &priority=9 { if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events]; + [$host=n$ip, $connect=F, $class=i, $sync=F, $auth=T, $events=worker2proxy_events]; # accepts connections from the previous one. # (This is not ideal for setups with many proxies) diff --git a/src/Stats.cc b/src/Stats.cc index a2e7496c5f..c3035231e9 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -210,11 +210,16 @@ void ProfileLogger::Log() i != thread_stats.end(); ++i ) { threading::MsgThread::Stats s = i->second; - file->Write(fmt("%0.6f %-25s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 "\n", + file->Write(fmt("%0.6f %-25s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 + " (#queue r/w: in=%" PRIu64 "/%" PRIu64 " out=%" PRIu64 "/%" PRIu64 ")" + "\n", network_time, i->first.c_str(), s.sent_in, s.sent_out, - s.pending_in, s.pending_out)); + s.pending_in, s.pending_out, + s.queue_in_stats.num_reads, s.queue_in_stats.num_writes, + s.queue_out_stats.num_reads, s.queue_out_stats.num_writes + )); } // Script-level state. diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index 3e05d17c9e..4d22bd9b1f 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -212,7 +212,7 @@ protected: const threading::Field* const* fields; // The log fields. // Buffer for bulk writes. - static const int WRITER_BUFFER_SIZE = 50; + static const int WRITER_BUFFER_SIZE = 1000; int write_buffer_pos; // Position of next write in buffer. threading::Value*** write_buffer; // Buffer of size WRITER_BUFFER_SIZE. }; diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 145e16c57b..ddcd3df1dd 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -283,5 +283,7 @@ void MsgThread::GetStats(Stats* stats) stats->sent_out = cnt_sent_out; stats->pending_in = queue_in.Size(); stats->pending_out = queue_out.Size(); + queue_in.GetStats(&stats->queue_in_stats); + queue_out.GetStats(&stats->queue_out_stats); } diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 28c7690dfa..5ac1c0f780 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -154,6 +154,10 @@ public: uint64_t sent_out; //! Number of messages sent from the child thread to the main thread uint64_t pending_in; //! Number of messages sent to the child but not yet processed. uint64_t pending_out; //! Number of messages sent from the child but not yet processed by the main thread. + + /// Statistics from our queues. + Queue::Stats queue_in_stats; + Queue::Stats queue_out_stats; }; /** diff --git a/src/threading/Queue.h b/src/threading/Queue.h index a25f897d23..985ba31714 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -58,6 +58,22 @@ public: */ uint64_t Size(); + /** + * Statistics about inter-thread communication. + */ + struct Stats + { + uint64_t num_reads; //! Number of messages read from the queue. + uint64_t num_writes; //! Number of messages written to the queue. + }; + + /** + * Returns statistics about the queue's usage. + * + * @param stats A pointer to a structure that will be filled with + * current numbers. */ + void GetStats(Stats* stats); + private: static const int NUM_QUEUES = 8; @@ -67,6 +83,10 @@ private: int read_ptr; // Where the next operation will read from int write_ptr; // Where the next operation will write to + + // Statistics. + uint64_t num_reads; + uint64_t num_writes; }; inline static void safe_lock(pthread_mutex_t* mutex) @@ -86,6 +106,7 @@ inline Queue::Queue() { read_ptr = 0; write_ptr = 0; + num_reads = num_writes = 0; for( int i = 0; i < NUM_QUEUES; ++i ) { @@ -121,6 +142,7 @@ inline T Queue::Get() messages[read_ptr].pop(); read_ptr = (read_ptr + 1) % NUM_QUEUES; + ++num_reads; safe_unlock(&mutex[old_read_ptr]); @@ -142,6 +164,7 @@ inline void Queue::Put(T data) pthread_cond_signal(&has_data[write_ptr]); write_ptr = (write_ptr + 1) % NUM_QUEUES; + ++num_writes; safe_unlock(&mutex[old_write_ptr]); } @@ -177,7 +200,23 @@ inline uint64_t Queue::Size() return size; } +template +inline void Queue::GetStats(Stats* stats) + { + // To be safe, we look all queues. That's probably unneccessary, but + // doesn't really hurt. + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_lock(&mutex[i]); + + stats->num_reads = num_reads; + stats->num_writes = num_writes; + + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_unlock(&mutex[i]); + } + } + #endif From 0ceca706f6d1a465bcb00b28164751e16e7ca0ff Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 26 Mar 2012 14:35:01 -0500 Subject: [PATCH 38/56] Change routing0_data_to_addrs BIF to return vector of addresses. Because the order of addresses in type 0 routing headers is interesting/important. --- scripts/base/init-bare.bro | 7 +++++++ src/bro.bif | 15 ++++++--------- .../Baseline/bifs.routing0_data_to_addrs/output | 5 +---- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index b9eca66d24..b2237d7af8 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -46,6 +46,13 @@ type index_vec: vector of count; ## then remove this alias. type string_vec: vector of string; +## A vector of addresses. +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. +type addr_vec: vector of addr; + ## A table of strings indexed by strings. ## ## .. todo:: We need this type definition only for declaring builtin functions via diff --git a/src/bro.bif b/src/bro.bif index 64ed7d1f2f..5ecc582a07 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2050,18 +2050,15 @@ function is_v6_addr%(a: addr%): bool # =========================================================================== ## Converts the *data* field of :bro:type:`ip6_routing` records that have -## *rtype* of 0 into a set of addresses. +## *rtype* of 0 into a vector of addresses. ## ## s: The *data* field of an :bro:type:`ip6_routing` record that has ## an *rtype* of 0. ## -## Returns: The set of addresses contained in the routing header data. -function routing0_data_to_addrs%(s: string%): addr_set +## Returns: The vector of addresses contained in the routing header data. +function routing0_data_to_addrs%(s: string%): addr_vec %{ - BroType* index_type = base_type(TYPE_ADDR); - TypeList* set_index = new TypeList(index_type); - set_index->Append(index_type); - TableVal* tv = new TableVal(new SetType(set_index, 0)); + VectorVal* rval = new VectorVal(new VectorType(base_type(TYPE_ADDR))); int len = s->Len(); const u_char* bytes = s->Bytes(); @@ -2074,12 +2071,12 @@ function routing0_data_to_addrs%(s: string%): addr_set while ( len > 0 ) { IPAddr a(IPAddr::IPv6, (const uint32*) bytes, IPAddr::Network); - tv->Assign(new AddrVal(a), 0); + rval->Assign(rval->Size(), new AddrVal(a), 0); bytes += 16; len -= 16; } - return tv; + return rval; %} ## Converts a :bro:type:`addr` to a :bro:type:`index_vec`. diff --git a/testing/btest/Baseline/bifs.routing0_data_to_addrs/output b/testing/btest/Baseline/bifs.routing0_data_to_addrs/output index 7179bf8564..c79aef89d0 100644 --- a/testing/btest/Baseline/bifs.routing0_data_to_addrs/output +++ b/testing/btest/Baseline/bifs.routing0_data_to_addrs/output @@ -1,4 +1 @@ -{ -2001:78:1:32::1, -2001:78:1:32::2 -} +[2001:78:1:32::1, 2001:78:1:32::2] From 256cd592a7d4c0bdbf43c3f2e9c4e1cdb0fe995a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 28 Mar 2012 13:49:28 -0500 Subject: [PATCH 39/56] Improve handling of IPv6 Routing Type 0 headers. - For RH0 headers with non-zero segments left, a "routing0_segleft" flow_weird event is raised (with a destination indicating the last address in the routing header), and an "rh0_segleft" event can also be handled if the other contents of the packet header are of interest. No further analysis is done as the complexity required to correctly identify destination endpoints of connections doesn't seem worth it as RH0 has been deprecated by RFC 5095. - For RH0 headers without any segments left, a "routing0_header" flow_weird event is raised, but further analysis still occurs as normal. --- src/IP.cc | 18 ++++++++++++++ src/IP.h | 21 +++++++++++++++++ src/Sessions.cc | 16 +++++++++++++ src/event.bif | 8 +++++++ .../Baseline/core.ipv6_ext_headers/output | 2 +- .../btest/Baseline/core.ipv6_rh0/segleft.out | 2 ++ .../btest/Baseline/core.ipv6_rh0/segleft0.out | 2 ++ ...uting.trace => ipv6-hbh-rh0-segleft.trace} | Bin .../btest/Traces/ipv6-hbh-rh0-segleft0.trace | Bin 0 -> 162 bytes .../btest/bifs/routing0_data_to_addrs.test | 4 ++-- testing/btest/core/ipv6_ext_headers.test | 2 +- testing/btest/core/ipv6_rh0.test | 22 ++++++++++++++++++ 12 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 testing/btest/Baseline/core.ipv6_rh0/segleft.out create mode 100644 testing/btest/Baseline/core.ipv6_rh0/segleft0.out rename testing/btest/Traces/{ext_hdr_hbh_routing.trace => ipv6-hbh-rh0-segleft.trace} (100%) create mode 100644 testing/btest/Traces/ipv6-hbh-rh0-segleft0.trace create mode 100644 testing/btest/core/ipv6_rh0.test diff --git a/src/IP.cc b/src/IP.cc index 4148c58a33..f82b7a0fd7 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -305,6 +305,24 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, bool set_next, uint16 next) chain.push_back(p); + // RFC 5095 deprecates routing type 0 headers, so raise weirds for that + if ( current_type == IPPROTO_ROUTING && + ((const struct ip6_rthdr*)hdrs)->ip6r_type == 0 ) + { + IPAddr src(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_src); + + if ( ((const struct ip6_rthdr*)hdrs)->ip6r_segleft > 0 ) + { + const in6_addr* a = (const in6_addr*)(hdrs+len-16); + reporter->Weird(src, *a, "routing0_segleft"); + } + else + { + IPAddr dst(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_dst); + reporter->Weird(src, dst, "routing0_header"); + } + } + hdrs += len; length += len; } while ( current_type != IPPROTO_FRAGMENT && diff --git a/src/IP.h b/src/IP.h index cb5bcf77c7..daa508db7f 100644 --- a/src/IP.h +++ b/src/IP.h @@ -171,6 +171,20 @@ public: { return IsFragment() ? (ntohs(GetFragHdr()->ip6f_offlg) & 0x0001) != 0 : 0; } + /** + * Returns whether the chain contains a routing type 0 extension header + * with nonzero segments left. + */ + bool RH0SegLeft() const + { + for ( size_t i = 0; i < chain.size(); ++i ) + if ( chain[i]->Type() == IPPROTO_ROUTING && + ((const struct ip6_rthdr*)chain[i]->Data())->ip6r_type == 0 && + ((const struct ip6_rthdr*)chain[i]->Data())->ip6r_segleft > 0 ) + return true; + return false; + } + /** * Returns a vector of ip6_ext_hdr RecordVals that includes script-layer * representation of all extension headers in the chain. @@ -343,6 +357,13 @@ public: size_t NumHeaders() const { return ip4 ? 1 : ip6_hdrs->Size(); } + /** + * Returns true if this is an IPv6 header containing a routing type 0 + * extension with nonzero segments left, else returns false. + */ + bool RH0SegLeft() const + { return ip4 ? false : ip6_hdrs->RH0SegLeft(); } + /** * Returns an ip_hdr or ip6_hdr_chain RecordVal. */ diff --git a/src/Sessions.cc b/src/Sessions.cc index 84b57bdc62..b5bb485d72 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -481,6 +481,22 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, return; } + // Stop analyzing IPv6 packets that use routing type 0 headers with segments + // left since RH0 headers are deprecated by RFC 5095 and we'd have to make + // extra effort to get the destination in the connection/flow endpoint right + if ( ip_hdr->RH0SegLeft() ) + { + dump_this_packet = 1; + if ( rh0_segleft ) + { + val_list* vl = new val_list(); + vl->append(ip_hdr->BuildPktHdrVal()); + mgr.QueueEvent(rh0_segleft, vl); + } + Remove(f); + return; + } + int proto = ip_hdr->NextProto(); if ( CheckHeaderTrunc(proto, len, caplen, hdr, pkt) ) diff --git a/src/event.bif b/src/event.bif index 113c003e37..20714c0931 100644 --- a/src/event.bif +++ b/src/event.bif @@ -478,6 +478,14 @@ event ipv6_ext_headers%(c: connection, p: pkt_hdr%); ## .. bro:see:: new_packet tcp_packet ipv6_ext_headers event esp_packet%(p: pkt_hdr%); +## Generated for any packets using an IPv6 Routing Type 0 extension header +## with non-zero segments left. +## +## p: Information from the header of the packet that triggered the event. +## +## .. bro:see:: new_packet tcp_packet ipv6_ext_headers +event rh0_segleft%(p: pkt_hdr%); + ## Generated for every packet that has non-empty transport-layer payload. This is a ## very low-level and expensive event that should be avoided when at all possible. ## It's usually infeasible to handle when processing even medium volumes of diff --git a/testing/btest/Baseline/core.ipv6_ext_headers/output b/testing/btest/Baseline/core.ipv6_ext_headers/output index a5a0caf7c6..58332ca900 100644 --- a/testing/btest/Baseline/core.ipv6_ext_headers/output +++ b/testing/btest/Baseline/core.ipv6_ext_headers/output @@ -1 +1 @@ -[ip=, ip6=[class=0, flow=0, len=59, nxt=0, hlim=64, src=2001:4f8:4:7:2e0:81ff:fe52:ffff, dst=2001:4f8:4:7:2e0:81ff:fe52:9a6b, exts=[[id=0, hopopts=[nxt=43, len=0, options=[[otype=1, len=4, data=\0\0\0\0]]], dstopts=, routing=, fragment=, ah=, esp=], [id=43, hopopts=, dstopts=, routing=[nxt=17, len=4, rtype=0, segleft=2, data=\0\0\0\0 ^A\0x\0^A\02\0\0\0\0\0\0\0^A ^A\0x\0^A\02\0\0\0\0\0\0\0^B], fragment=, ah=, esp=]]], tcp=, udp=[sport=53/udp, dport=53/udp, ulen=11], icmp=] +[ip=, ip6=[class=0, flow=0, len=68, nxt=0, hlim=64, src=2001:4f8:4:7:2e0:81ff:fe52:ffff, dst=2001:4f8:4:7:2e0:81ff:fe52:9a6b, exts=[[id=0, hopopts=[nxt=43, len=0, options=[[otype=1, len=4, data=\0\0\0\0]]], dstopts=, routing=, fragment=, ah=, esp=], [id=43, hopopts=, dstopts=, routing=[nxt=6, len=4, rtype=0, segleft=0, data=\0\0\0\0 ^A\0x\0^A\02\0\0\0\0\0\0\0^A ^A\0x\0^A\02\0\0\0\0\0\0\0^B], fragment=, ah=, esp=]]], tcp=[sport=30000/tcp, dport=80/tcp, seq=0, ack=0, hl=20, dl=0, flags=2, win=8192], udp=, icmp=] diff --git a/testing/btest/Baseline/core.ipv6_rh0/segleft.out b/testing/btest/Baseline/core.ipv6_rh0/segleft.out new file mode 100644 index 0000000000..3c722ee3b4 --- /dev/null +++ b/testing/btest/Baseline/core.ipv6_rh0/segleft.out @@ -0,0 +1,2 @@ +flow_weird routing0_segleft from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:78:1:32::2 +rh0 w/ segments left from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:4f8:4:7:2e0:81ff:fe52:9a6b diff --git a/testing/btest/Baseline/core.ipv6_rh0/segleft0.out b/testing/btest/Baseline/core.ipv6_rh0/segleft0.out new file mode 100644 index 0000000000..ae57c7cc8d --- /dev/null +++ b/testing/btest/Baseline/core.ipv6_rh0/segleft0.out @@ -0,0 +1,2 @@ +flow_weird routing0_header from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:4f8:4:7:2e0:81ff:fe52:9a6b +new_connection: [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/tcp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=80/tcp] diff --git a/testing/btest/Traces/ext_hdr_hbh_routing.trace b/testing/btest/Traces/ipv6-hbh-rh0-segleft.trace similarity index 100% rename from testing/btest/Traces/ext_hdr_hbh_routing.trace rename to testing/btest/Traces/ipv6-hbh-rh0-segleft.trace diff --git a/testing/btest/Traces/ipv6-hbh-rh0-segleft0.trace b/testing/btest/Traces/ipv6-hbh-rh0-segleft0.trace new file mode 100644 index 0000000000000000000000000000000000000000..35f5b3afe633cc81fc2444b10fa278a29d81783f GIT binary patch literal 162 zcmca|c+)~A1{MYwaA0F#U<7gw`4#(HHgGdk0ofq@9}FO>+U_QR7%mJB3XCj2fSTEv z9yI>{7xe!>Dt}hCHUlHrXf~(?3XBXDK;w-d<}fg#@tH~u7y_Vj3;|3E4EkLR3;-CU BAFlua literal 0 HcmV?d00001 diff --git a/testing/btest/bifs/routing0_data_to_addrs.test b/testing/btest/bifs/routing0_data_to_addrs.test index 4bf15cae87..de10dd80e0 100644 --- a/testing/btest/bifs/routing0_data_to_addrs.test +++ b/testing/btest/bifs/routing0_data_to_addrs.test @@ -1,7 +1,7 @@ -# @TEST-EXEC: bro -C -b -r $TRACES/ext_hdr_hbh_routing.trace %INPUT >output +# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft.trace %INPUT >output # @TEST-EXEC: btest-diff output -event ipv6_ext_headers(c: connection, p: pkt_hdr) +event rh0_segleft(p: pkt_hdr) { for ( h in p$ip6$exts ) if ( p$ip6$exts[h]$id == IPPROTO_ROUTING ) diff --git a/testing/btest/core/ipv6_ext_headers.test b/testing/btest/core/ipv6_ext_headers.test index 170a67bc72..0cf3f2f3fb 100644 --- a/testing/btest/core/ipv6_ext_headers.test +++ b/testing/btest/core/ipv6_ext_headers.test @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -C -b -r $TRACES/ext_hdr_hbh_routing.trace %INPUT >output +# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft0.trace %INPUT >output # @TEST-EXEC: btest-diff output # Just check that the event is raised correctly for a packet containing diff --git a/testing/btest/core/ipv6_rh0.test b/testing/btest/core/ipv6_rh0.test new file mode 100644 index 0000000000..18c23ed3b7 --- /dev/null +++ b/testing/btest/core/ipv6_rh0.test @@ -0,0 +1,22 @@ +# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft0.trace %INPUT >segleft0.out +# @TEST-EXEC: btest-diff segleft0.out +# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft.trace %INPUT >segleft.out +# @TEST-EXEC: btest-diff segleft.out + +# This will be raised only by the packet with RH0 and segments left. +event rh0_segleft(p: pkt_hdr) + { + print fmt("rh0 w/ segments left from %s to %s", p$ip6$src, p$ip6$dst); + } + +# This will be raised only by the packet with RH0 and no segments left. +event new_connection(c: connection) + { + print fmt("new_connection: %s", c$id); + } + +# This will be raised by any packet with RH0 regardless of segments left. +event flow_weird(name: string, src: addr, dst: addr) + { + print fmt("flow_weird %s from %s to %s", name, src, dst); + } From 8a1d71dc0864d33aff81d9ab5e6f5b4265ed7d21 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 28 Mar 2012 14:14:20 -0500 Subject: [PATCH 40/56] Remove dead tcp_checksum function from net_util --- src/net_util.cc | 27 --------------------------- src/net_util.h | 1 - 2 files changed, 28 deletions(-) diff --git a/src/net_util.cc b/src/net_util.cc index 1a4e9f1a7f..9e023a5fc1 100644 --- a/src/net_util.cc +++ b/src/net_util.cc @@ -38,33 +38,6 @@ int ones_complement_checksum(const IPAddr& a, uint32 sum) return ones_complement_checksum(bytes, len*4, sum); } -int tcp_checksum(const struct ip* ip, const struct tcphdr* tp, int len) - { - // ### Note, this is only correct for IPv4. This routine is only - // used by the connection compressor (which we turn off for IPv6 - // traffic). - - int tcp_len = tp->th_off * 4 + len; - uint32 sum; - - if ( len % 2 == 1 ) - // Add in pad byte. - sum = htons(((const u_char*) tp)[tcp_len - 1] << 8); - else - sum = 0; - - sum = ones_complement_checksum((void*) &ip->ip_src.s_addr, 4, sum); - sum = ones_complement_checksum((void*) &ip->ip_dst.s_addr, 4, sum); - - uint32 addl_pseudo = - (htons(IPPROTO_TCP) << 16) | htons((unsigned short) tcp_len); - - sum = ones_complement_checksum((void*) &addl_pseudo, 4, sum); - sum = ones_complement_checksum((void*) tp, tcp_len, sum); - - return sum; - } - int udp_checksum(const struct ip* ip, const struct udphdr* up, int len) { uint32 sum; diff --git a/src/net_util.h b/src/net_util.h index 8787340328..5e39a11714 100644 --- a/src/net_util.h +++ b/src/net_util.h @@ -62,7 +62,6 @@ inline int seq_delta(uint32 a, uint32 b) extern int ones_complement_checksum(const void* p, int b, uint32 sum); extern int ones_complement_checksum(const IPAddr& a, uint32 sum); -extern int tcp_checksum(const struct ip* ip, const struct tcphdr* tp, int len); extern int udp_checksum(const struct ip* ip, const struct udphdr* up, int len); extern int udp6_checksum(const struct ip6_hdr* ip, const struct udphdr* up, int len); From 42066cc1fd35ca7b63daeaf588271ec3c040385e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 28 Mar 2012 14:53:59 -0700 Subject: [PATCH 41/56] Teaching cmake to always link in tcmalloc if it finds it. Also renaming --enable-perftools to --enable-perftool-debug to indicate that the switch is only relevant for debugging the heap. It's not needed to pick up tcmalloc for better performance. --with-perftools can still (and always) be used to give a hint where to find the libraries. With the threading, using tcmalloc improves memory usage on FreeBSD significantly when running on a trace. If it fixes the live problems, remains to be seen ... --- CMakeLists.txt | 20 +++++++++++++++----- cmake | 2 +- configure | 9 ++++----- src/DPM.cc | 2 +- src/File.cc | 4 ++-- src/ID.cc | 2 +- src/Login.cc | 2 +- src/PersistenceSerializer.cc | 2 +- src/RemoteSerializer.cc | 4 ++-- src/RuleMatcher.cc | 2 +- src/StateAccess.cc | 2 +- src/main.cc | 18 +++++++++--------- src/util.h | 2 +- 13 files changed, 40 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de3138c20c..febc2d6ec1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,11 +89,20 @@ if (LIBGEOIP_FOUND) endif () set(USE_PERFTOOLS false) -if (ENABLE_PERFTOOLS) - find_package(GooglePerftools) - if (GOOGLEPERFTOOLS_FOUND) - set(USE_PERFTOOLS true) - include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR}) +set(USE_PERFTOOLS_DEBUG false) + +find_package(GooglePerftools) + +if (GOOGLEPERFTOOLS_FOUND) + include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR}) + set(USE_PERFTOOLS true) + + if (ENABLE_PERFTOOLS_DEBUG) + # Enable heap debugging with perftools. + set(USE_PERFTOOLS_DEBUG true) + list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG}) + else () + # Link in tcmalloc for better performance. list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES}) endif () endif () @@ -183,6 +192,7 @@ message( "\n" "\nGeoIP: ${USE_GEOIP}" "\nGoogle perftools: ${USE_PERFTOOLS}" + "\n debugging: ${USE_PERFTOOLS_DEBUG}" "\n" "\n================================================================\n" ) diff --git a/cmake b/cmake index 2cc1055770..4b573ed849 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 2cc105577044a2d214124568f3f2496ed2ccbb34 +Subproject commit 4b573ed849f131ebb8e34fa24786d56f9805e444 diff --git a/configure b/configure index 43afb4ae99..05aa12815b 100755 --- a/configure +++ b/configure @@ -27,7 +27,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... Optional Features: --enable-debug compile in debugging mode - --enable-perftools use Google's perftools + --enable-perftools-debug use Google's perftools for debugging --disable-broccoli don't build or install the Broccoli library --disable-broctl don't install Broctl --disable-auxtools don't build or install auxilliary tools @@ -91,7 +91,7 @@ append_cache_entry BRO_ROOT_DIR PATH /usr/local/bro append_cache_entry PY_MOD_INSTALL_DIR PATH /usr/local/bro/lib/broctl append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING /usr/local/bro/share/bro append_cache_entry ENABLE_DEBUG BOOL false -append_cache_entry ENABLE_PERFTOOLS BOOL false +append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false append_cache_entry BinPAC_SKIP_INSTALL BOOL true append_cache_entry BUILD_SHARED_LIBS BOOL true append_cache_entry INSTALL_AUX_TOOLS BOOL true @@ -132,8 +132,8 @@ while [ $# -ne 0 ]; do --enable-debug) append_cache_entry ENABLE_DEBUG BOOL true ;; - --enable-perftools) - append_cache_entry ENABLE_PERFTOOLS BOOL true + --enable-perftools-debug) + append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL true ;; --disable-broccoli) append_cache_entry INSTALL_BROCCOLI BOOL false @@ -178,7 +178,6 @@ while [ $# -ne 0 ]; do append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg ;; --with-perftools=*) - append_cache_entry ENABLE_PERFTOOLS BOOL true append_cache_entry GooglePerftools_ROOT_DIR PATH $optarg ;; --with-python=*) diff --git a/src/DPM.cc b/src/DPM.cc index 595ee42ec8..0902ae9a45 100644 --- a/src/DPM.cc +++ b/src/DPM.cc @@ -74,7 +74,7 @@ void DPM::PostScriptInit() void DPM::AddConfig(const Analyzer::Config& cfg) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif diff --git a/src/File.cc b/src/File.cc index 080923ad37..d4e31bcc16 100644 --- a/src/File.cc +++ b/src/File.cc @@ -232,7 +232,7 @@ BroFile::~BroFile() delete [] access; delete [] cipher_buffer; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->UnIgnoreObject(this); #endif } @@ -255,7 +255,7 @@ void BroFile::Init() cipher_ctx = 0; cipher_buffer = 0; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->IgnoreObject(this); #endif } diff --git a/src/ID.cc b/src/ID.cc index 3f5c76ca1d..a70aa3fd0e 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -372,7 +372,7 @@ ID* ID::Unserialize(UnserialInfo* info) Ref(id); global_scope()->Insert(id->Name(), id); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->IgnoreObject(id); #endif } diff --git a/src/Login.cc b/src/Login.cc index 56efd12f53..e626fb3a0a 100644 --- a/src/Login.cc +++ b/src/Login.cc @@ -38,7 +38,7 @@ Login_Analyzer::Login_Analyzer(AnalyzerTag::Tag tag, Connection* conn) if ( ! re_skip_authentication ) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif re_skip_authentication = init_RE(skip_authentication); diff --git a/src/PersistenceSerializer.cc b/src/PersistenceSerializer.cc index c757467f90..d9baad05bb 100644 --- a/src/PersistenceSerializer.cc +++ b/src/PersistenceSerializer.cc @@ -137,7 +137,7 @@ bool PersistenceSerializer::CheckForFile(UnserialInfo* info, const char* file, bool PersistenceSerializer::ReadAll(bool is_init, bool delete_files) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index f29e907790..e9fbe0aab8 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -2645,7 +2645,7 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( current_peer->state == Peer::CLOSING ) return false; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG // Don't track allocations here, they'll be released only after the // main loop exists. And it's just a tiny amount anyway. HeapLeakChecker::Disabler disabler; @@ -2866,7 +2866,7 @@ void RemoteSerializer::GotID(ID* id, Val* val) (desc && *desc) ? desc : "not set"), current_peer); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG // May still be cached, but we don't care. heap_checker->IgnoreObject(id); #endif diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index da12b1b679..c9cf1f5c11 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -191,7 +191,7 @@ void RuleMatcher::Delete(RuleHdrTest* node) bool RuleMatcher::ReadFiles(const name_list& files) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 7abef72c46..2d0a8dfc5a 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -678,7 +678,7 @@ bool StateAccess::DoUnserialize(UnserialInfo* info) target.id = new ID(name, SCOPE_GLOBAL, true); Ref(target.id); global_scope()->Insert(name, target.id); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->IgnoreObject(target.id); #endif } diff --git a/src/main.cc b/src/main.cc index e484b58fe2..17a798ea81 100644 --- a/src/main.cc +++ b/src/main.cc @@ -65,7 +65,7 @@ extern "C" { #include "setsignal.h" }; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker* heap_checker = 0; int perftools_leaks = 0; int perftools_profile = 0; @@ -177,7 +177,7 @@ void usage() fprintf(stderr, " -W|--watchdog | activate watchdog timer\n"); fprintf(stderr, " -Z|--doc-scripts | generate documentation for all loaded scripts\n"); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG fprintf(stderr, " -m|--mem-leaks | show leaks [perftools]\n"); fprintf(stderr, " -M|--mem-profile | record heap [perftools]\n"); #endif @@ -244,7 +244,7 @@ void done_with_network() net_finish(1); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG if ( perftools_profile ) { @@ -424,7 +424,7 @@ int main(int argc, char** argv) #ifdef USE_IDMEF {"idmef-dtd", required_argument, 0, 'n'}, #endif -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG {"mem-leaks", no_argument, 0, 'm'}, {"mem-profile", no_argument, 0, 'M'}, #endif @@ -466,7 +466,7 @@ int main(int argc, char** argv) safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLOPSWbdghvZ", sizeof(opts)); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG strncat(opts, "mM", 2); #endif @@ -622,7 +622,7 @@ int main(int argc, char** argv) exit(0); break; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG case 'm': perftools_leaks = 1; break; @@ -758,14 +758,14 @@ int main(int argc, char** argv) // nevertheless reported; see perftools docs), thus // we suppress some messages here. -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG { HeapLeakChecker::Disabler disabler; #endif yyparse(); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG } #endif @@ -1019,7 +1019,7 @@ int main(int argc, char** argv) if ( profiling_logger ) profiling_logger->Log(); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG if ( perftools_leaks ) heap_checker = new HeapLeakChecker("net_run"); diff --git a/src/util.h b/src/util.h index 498bdf00e4..a4e3aa71b8 100644 --- a/src/util.h +++ b/src/util.h @@ -37,7 +37,7 @@ #endif -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG #include #include extern HeapLeakChecker* heap_checker; From c382439079755f1ca613881b1699ba77bfe9f246 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 28 Mar 2012 15:39:56 -0700 Subject: [PATCH 42/56] Switching log buffer size back to normal --- src/RemoteSerializer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index e9fbe0aab8..017f260bdf 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -234,7 +234,7 @@ static const int PRINT_BUFFER_SIZE = 10 * 1024; static const int SOCKBUF_SIZE = 1024 * 1024; // Buffer size for remote-log data. -static const int LOG_BUFFER_SIZE = 512; +static const int LOG_BUFFER_SIZE = 50 * 1024; struct ping_args { uint32 seq; From 76af3cf825037353b2a8ae09e59ab0cf7333128f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 28 Mar 2012 15:52:13 -0700 Subject: [PATCH 43/56] Updating submodule(s). [nomail] --- aux/broctl | 2 +- aux/btest | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broctl b/aux/broctl index 66e9e87bee..c86b7e990b 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 66e9e87beebce983fa0f479b0284d5690b0290d4 +Subproject commit c86b7e990b4d39cd48c0cb692077aa081b418149 diff --git a/aux/btest b/aux/btest index dc78a3ebf5..120c978a12 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit dc78a3ebf5cd8fbd1b3034990e36fa21a51d1a19 +Subproject commit 120c978a1236db4f48c6f38a3a99199d85bb904e From 97652bc144df90b63f3ab075e4e3b4e7932ab3ac Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 28 Mar 2012 16:15:52 -0700 Subject: [PATCH 44/56] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index 120c978a12..c8e8fe477b 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 120c978a1236db4f48c6f38a3a99199d85bb904e +Subproject commit c8e8fe477b5dec635e5ce00f3f764fad069c549c From 6e7faafdb7bbf46306a186a033a85e3b34dd364d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 30 Mar 2012 12:40:31 -0500 Subject: [PATCH 45/56] Fix compile errors due to now-explicit IPAddr ctors and global IPFamily enum. --- src/IP.cc | 6 +++--- src/bro.bif | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/IP.cc b/src/IP.cc index 7f616fbbb0..bb60d17f15 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -74,8 +74,8 @@ RecordVal* IPv6_Hdr::BuildRecordVal(VectorVal* chain) const rv->Assign(2, new Val(ntohs(ip6->ip6_plen), TYPE_COUNT)); rv->Assign(3, new Val(ip6->ip6_nxt, TYPE_COUNT)); rv->Assign(4, new Val(ip6->ip6_hlim, TYPE_COUNT)); - rv->Assign(5, new AddrVal(ip6->ip6_src)); - rv->Assign(6, new AddrVal(ip6->ip6_dst)); + rv->Assign(5, new AddrVal(IPAddr(ip6->ip6_src))); + rv->Assign(6, new AddrVal(IPAddr(ip6->ip6_dst))); if ( ! chain ) chain = new VectorVal(new VectorType( hdrType(ip6_ext_hdr_type, "ip6_ext_hdr")->Ref())); @@ -314,7 +314,7 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, bool set_next, uint16 next) if ( ((const struct ip6_rthdr*)hdrs)->ip6r_segleft > 0 ) { const in6_addr* a = (const in6_addr*)(hdrs+len-16); - reporter->Weird(src, *a, "routing0_segleft"); + reporter->Weird(src, IPAddr(*a), "routing0_segleft"); } else { diff --git a/src/bro.bif b/src/bro.bif index 6766a89142..fa6766a7bf 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2070,7 +2070,7 @@ function routing0_data_to_addrs%(s: string%): addr_vec while ( len > 0 ) { - IPAddr a(IPAddr::IPv6, (const uint32*) bytes, IPAddr::Network); + IPAddr a(IPv6, (const uint32*) bytes, IPAddr::Network); rval->Assign(rval->Size(), new AddrVal(a), 0); bytes += 16; len -= 16; From c381da9ccfd956be506b17267e92077a0dc1868c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 3 Apr 2012 22:18:35 -0700 Subject: [PATCH 46/56] This could be fixing the memory problems finally. Keeping fingers crossed ... --- src/threading/Manager.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 43eb0313f4..7481e83192 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -73,14 +73,11 @@ void Manager::GetFds(int* read, int* write, int* except) double Manager::NextTimestamp(double* network_time) { - if ( ::network_time && ! next_beat ) - next_beat = ::network_time + HEART_BEAT_INTERVAL; - // fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat); - if ( did_process || ::network_time > next_beat ) + if ( did_process || ::network_time > next_beat || ! next_beat ) // If we had something to process last time (or out heartbeat - // is due), we want to check for more asap. + // is due or not set yet), we want to check for more asap. return timer_mgr->Time(); return -1.0; @@ -88,7 +85,13 @@ double Manager::NextTimestamp(double* network_time) void Manager::Process() { - bool do_beat = (next_beat && network_time > next_beat); + bool do_beat = false; + + if ( network_time && (network_time > next_beat || ! next_beat) ) + { + do_beat = true; + next_beat = ::network_time + HEART_BEAT_INTERVAL; + } did_process = false; @@ -97,10 +100,7 @@ void Manager::Process() MsgThread* t = *i; if ( do_beat ) - { t->Heartbeat(); - next_beat = 0; - } while ( t->HasOut() ) { From fe61d02a9f9d32620a9783f1c61a0fe56880235f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Apr 2012 10:42:50 -0700 Subject: [PATCH 47/56] Two more tweaks to reliably terminate when reading from trace. --- src/threading/Manager.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 7481e83192..abdbf849b5 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -43,6 +43,7 @@ void Manager::Terminate() msg_threads.clear(); idle = true; + closed = true; terminating = false; } @@ -75,7 +76,7 @@ double Manager::NextTimestamp(double* network_time) { // fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat); - if ( did_process || ::network_time > next_beat || ! next_beat ) + if ( ::network_time && (did_process || ::network_time > next_beat || ! next_beat) ) // If we had something to process last time (or out heartbeat // is due or not set yet), we want to check for more asap. return timer_mgr->Time(); From d8d83f590bb9836205f71a596b2868ffb6d486f4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 4 Apr 2012 15:27:43 -0500 Subject: [PATCH 48/56] Fix handling of IPv6 atomic fragments. The FragReassembler expire_timer was left uninitialized until after the first fragment is added, but since the atomic fragment is also the last, the reassembler thought expire_timer needed to be deleted. This fix just initializes expire_timer before the first fragment is added. --- src/Frag.cc | 4 ++-- .../btest/Baseline/core.ipv6-atomic-frag/output | 4 ++++ testing/btest/Traces/ipv6-http-atomic-frag.trace | Bin 0 -> 4040 bytes testing/btest/core/ipv6-atomic-frag.test | 7 +++++++ 4 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 testing/btest/Baseline/core.ipv6-atomic-frag/output create mode 100644 testing/btest/Traces/ipv6-http-atomic-frag.trace create mode 100644 testing/btest/core/ipv6-atomic-frag.test diff --git a/src/Frag.cc b/src/Frag.cc index 9bd16a71c9..04298e14ad 100644 --- a/src/Frag.cc +++ b/src/Frag.cc @@ -52,8 +52,6 @@ FragReassembler::FragReassembler(NetSessions* arg_s, frag_size = 0; // flag meaning "not known" next_proto = ip->NextProto(); - AddFragment(t, ip, pkt); - if ( frag_timeout != 0.0 ) { expire_timer = new FragTimer(this, t + frag_timeout); @@ -61,6 +59,8 @@ FragReassembler::FragReassembler(NetSessions* arg_s, } else expire_timer = 0; + + AddFragment(t, ip, pkt); } FragReassembler::~FragReassembler() diff --git a/testing/btest/Baseline/core.ipv6-atomic-frag/output b/testing/btest/Baseline/core.ipv6-atomic-frag/output new file mode 100644 index 0000000000..4a628a4bdc --- /dev/null +++ b/testing/btest/Baseline/core.ipv6-atomic-frag/output @@ -0,0 +1,4 @@ +[orig_h=2001:db8:1::2, orig_p=36951/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp] +[orig_h=2001:db8:1::2, orig_p=59694/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp] +[orig_h=2001:db8:1::2, orig_p=27393/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp] +[orig_h=2001:db8:1::2, orig_p=45805/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp] diff --git a/testing/btest/Traces/ipv6-http-atomic-frag.trace b/testing/btest/Traces/ipv6-http-atomic-frag.trace new file mode 100644 index 0000000000000000000000000000000000000000..d5d9db276cc668c51dc6289a25628db8fd3786cd GIT binary patch literal 4040 zcmchZUr19?9LIn6y3M&%wA>nKvZ4pYoSBrN!_>$iOmAu~(A3Fhn{Gz9C|_dzH3&kX z2#Ub$Ee&B$MT~+B!=3_PLL{>%B{S`%ddPmia}?HXjjemv;NZEtXTQ(ye1E_5^-RAE zJ`h0z8=ucw00AFbhy3;Dx{{$8=QJS0e7m^&5w0|7uFZDPHJFmsal*)v9+>24lu)t9)FYV!SiXM(&{xl*elohi=M$ErW$}k0* zCwb&ngC0gOD2)h_y>Ui1n*uEh05-SZ=dCcL*xW9M6aS{z{2ss8RZ;D5ZnqWjIle98 zuW{CS+S$+6xr^5q;e=N37r2rIBrV*ig#(kN5uu(1ao3ZpZMJ}ZH^AXTO_righto>H zGy?(eqC~2Q?_=U;qWHaB{F^m;bz8Fy8^=eDQ@hLwm#!v&&?{vkVTlulkf4e>y{Jw* zYfT@(Nz`3?N*h(3T`?S7>S}-mD@dM;{b-*6ba35WoM1$PQsY^${;b)v38U-Cc=Bhs z@lgJ+Z>gNW4oRt0-Ti1sMA$K*jd~Q~R)`buD6G!@aTEx+4}nUH%>+wD!O-DMfvU57 zD~lLesSkxkPOuM=7P+rf(!$BKKwGHF;ac2C(y80{Mr<4>HBRY$PROfF0KrAJPONp5|r9PkMO<2-GI^O z$rf7BEa^T%eG|$|S2zE8BsEfXyG=a*7uD%eZ6R)jI00>8=a-1K@D_nei_HXUM8Q_5 z*nI0Vtpe^)SxfbUysN6U!*f^uqIG%A%2!#?r7$aNo zJm!QcB&ed!7^<_%I;9Yx4Ug6MCr#92RS<(MkP!d`mq5y%?0!#kx-I;`UL?YEBq+6o z;L_Lp<7F6qj%?xKBDV#KUYi&Fwh(vzb4V(u>IS#){Dd dl@^-`R*r%#Q?Z|2;ak~uLiueWEOLTv;TKT4;Youtput +# @TEST-EXEC: btest-diff output + +event new_connection(c: connection) + { + print c$id; + } From fb0614b5c64a544e44ba30441f498c8a36b62406 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Apr 2012 17:46:13 -0700 Subject: [PATCH 49/56] Adding notice_policy.log canonification for external tests. These was still producing false positives. --- testing/scripts/diff-canon-notice-policy | 10 ++++++++++ testing/scripts/diff-canonifier-external | 2 ++ 2 files changed, 12 insertions(+) create mode 100755 testing/scripts/diff-canon-notice-policy diff --git a/testing/scripts/diff-canon-notice-policy b/testing/scripts/diff-canon-notice-policy new file mode 100755 index 0000000000..f05abaa103 --- /dev/null +++ b/testing/scripts/diff-canon-notice-policy @@ -0,0 +1,10 @@ +#! /usr/bin/awk -f +# +# A diff canonifier that removes the priorities in notice_policy.log. + +/^#/ && $2 == "notice_policy" { filter = 1; } + +filter == 1 && /^[^#]/ { sub("^[0-9]*", "X"); } + +{ print; } + diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external index 6796614362..e788a4a1bb 100755 --- a/testing/scripts/diff-canonifier-external +++ b/testing/scripts/diff-canonifier-external @@ -6,4 +6,6 @@ | `dirname $0`/diff-remove-uids \ | `dirname $0`/diff-remove-mime-types \ | `dirname $0`/diff-remove-x509-names \ + | `dirname $0`/diff-canon-notice-policy \ | `dirname $0`/diff-sort + From 4e49b98bbabcd7acdeeb4fcfa14c4961fdc0b565 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Apr 2012 17:57:38 -0700 Subject: [PATCH 50/56] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index c8e8fe477b..8da6c55697 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit c8e8fe477b5dec635e5ce00f3f764fad069c549c +Subproject commit 8da6c55697ff580600cfff474f4ccba2a592f911 From c372eaf7b59b30638c3a34e53114259c638c691e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Apr 2012 18:36:40 -0700 Subject: [PATCH 51/56] Updating submodule(s). [nomail] --- cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake b/cmake index 550ab2c8d9..5ddec45563 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 550ab2c8d95b1d3e18e40a903152650e6c7a3c45 +Subproject commit 5ddec4556338339fc4d1da27bce766a827990543 From d0b68771ef78227315724cdf99b296ed723200bb Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Apr 2012 18:37:10 -0700 Subject: [PATCH 52/56] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aux/binpac b/aux/binpac index dd1a3a95f0..56ae73ab99 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit dd1a3a95f07082efcd5274b21104a038d523d132 +Subproject commit 56ae73ab995dda665d8918d1a6b3af39b15991e3 diff --git a/aux/bro-aux b/aux/bro-aux index a59b35bdad..12d32194c1 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit a59b35bdada8f70fb1a59bf7bb2976534c86d378 +Subproject commit 12d32194c19d2dce06818588a2aeccf234de1889 diff --git a/aux/broccoli b/aux/broccoli index a4046c2f79..60898666ba 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit a4046c2f79b6ab0ac19ae8be94b79c6ce578bea7 +Subproject commit 60898666ba1df1913c08ad5045b1e56f974060cc diff --git a/aux/broctl b/aux/broctl index c86b7e990b..4d1a0692a7 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit c86b7e990b4d39cd48c0cb692077aa081b418149 +Subproject commit 4d1a0692a7d7b5229230856a4041f70fd3a6eaa5 From 017622427bc9ac791860e31d7c64b20d820373c1 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Apr 2012 18:42:15 -0700 Subject: [PATCH 53/56] Fixing perftools-debug support. --- config.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.h.in b/config.h.in index e744cb7dbd..0047344c51 100644 --- a/config.h.in +++ b/config.h.in @@ -109,7 +109,7 @@ #cmakedefine HAVE_GEOIP_CITY_EDITION_REV0_V6 /* Use Google's perftools */ -#cmakedefine USE_PERFTOOLS +#cmakedefine USE_PERFTOOLS_DEBUG /* Version number of package */ #define VERSION "@VERSION@" From c92dc7e6afd9906a25f2c503ad1aacff05c2f08f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Apr 2012 19:27:43 -0700 Subject: [PATCH 54/56] Reverting SocketComm change tuning I/O behaviour. Not sure that's right. --- src/RemoteSerializer.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 110a25e66f..bf195e9d3a 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -3399,8 +3399,10 @@ void SocketComm::Run() small_timeout.tv_usec = io->CanWrite() || io->CanRead() ? 1 : 10; +#if 0 if ( ! io->CanWrite() ) usleep(10); +#endif int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except, &small_timeout); From 8e2ce1d350510ca7ab52f0d5a0e4654780eb1125 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 5 Apr 2012 11:41:14 -0500 Subject: [PATCH 55/56] Fix CMake from warning about unused ENABLE_PERFTOOLS_DEBUG variable. The variable was never "used" in the case google perftools was not found, so CMake warned about it being unused since it was manually specified as a cache variable in the configure script. There might be a better way to organize that, but this is a quick/easy hack. --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index febc2d6ec1..d27fa2d40b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,6 +107,11 @@ if (GOOGLEPERFTOOLS_FOUND) endif () endif () +if (ENABLE_PERFTOOLS_DEBUG) + # Just a no op to prevent CMake from complaining about manually-specified + # ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found +endif () + set(brodeps ${BinPAC_LIBRARY} ${PCAP_LIBRARY} From c63d23f3af359731cbd79c1519415d832a95f994 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 5 Apr 2012 15:30:40 -0700 Subject: [PATCH 56/56] Preventing Bro processes that do neither local logging nor request remote logs from spawning threads. This applies to the proxy, which was still opening all the log files with then idle threads. --- src/RemoteSerializer.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index bf195e9d3a..61be8a9e8f 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -2503,6 +2503,9 @@ bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) return false; + if ( ! peer->logs_requested ) + return false; + BinarySerializationFormat fmt; fmt.StartWrite(); @@ -2625,6 +2628,9 @@ error: bool RemoteSerializer::FlushLogBuffer(Peer* p) { + if ( ! p->logs_requested ) + return false; + last_flush = network_time; if ( p->state == Peer::CLOSING )