Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-03 07:08:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge branch 'topic/robin/log-threads'

Browse source 
* topic/robin/log-threads: (42 commits)
  Two more tweaks to reliably terminate when reading from trace.
  This could be fixing the memory problems finally.
  Fix compile errors due to now-explicit IPAddr ctors and global IPFamily enum.
  Switching log buffer size back to normal
  Teaching cmake to always link in tcmalloc if it finds it.
  Extending queue statistics.
  Small fixes and tweaks.
  Don't assert during shutdown.
  Reverting accidental commit.
  Finetuning communication CPU usage.
  Adding new leak tests involving remote logging.
  Removing some no longer needed checks.
  Fixing problem logging remotely when local logging was turned off.
  Preventing busy looping when no threads have been spawned.
  Prevent manager from busy looping.
  Adding missing includes needed on FreeBSD.
  Updating submodule(s).
  Updating submodule(s).
  A number of bugfixes for the recent threading updates.
  Making exchange of addresses between threads thread-safe.
  ...
This commit is contained in:
Robin Sommer 2012-04-04 17:25:20 -07:00
commit 077089a047
69 changed files with 4171 additions and 1157 deletions
Download patch file Download diff file Expand all files Collapse all files

16
CHANGES
View file

@ -1,4 +1,20 @@
2.0-245 | 2012-04-04 17:25:20 -0700
* Internal restructuring of the logging framework: we now spawn
threads doing the I/O. From a user's perspective not much should
change, except that the OS may now show a bunch of Bro threads.
(Gilbert Clark and Robin Sommer).
* When building Bro, we now always link in tcmalloc if it's found at
configure time. If it's installed but not picked up,
--with-perftools may help. (Robin Sommer)
* Renaming the configure option --enable-perftools to
--enable-perftool-debug to indicate that the switch is only
relevant for debugging the heap. It's not needed to pick up
tcmalloc for better performance. (Robin Sommer)
2.0-184 | 2012-03-28 15:11:11 -0700
* Improve handling of IPv6 Routing Type 0 headers. (Jon Siwek)

18
CMakeLists.txt
View file

@ -89,11 +89,20 @@ if (LIBGEOIP_FOUND)
endif ()
set(USE_PERFTOOLS false)
if (ENABLE_PERFTOOLS)
find_package(GooglePerftools)
if (GOOGLEPERFTOOLS_FOUND)
set(USE_PERFTOOLS true)
set(USE_PERFTOOLS_DEBUG false)
find_package(GooglePerftools)
if (GOOGLEPERFTOOLS_FOUND)
include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
set(USE_PERFTOOLS true)
if (ENABLE_PERFTOOLS_DEBUG)
# Enable heap debugging with perftools.
set(USE_PERFTOOLS_DEBUG true)
list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
else ()
# Link in tcmalloc for better performance.
list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
endif ()
endif ()
@ -183,6 +192,7 @@ message(
"\n"
"\nGeoIP: ${USE_GEOIP}"
"\nGoogle perftools: ${USE_PERFTOOLS}"
"\n debugging: ${USE_PERFTOOLS_DEBUG}"
"\n"
"\n================================================================\n"
)

12
NEWS
View file

@ -13,6 +13,10 @@ Bro 2.1
* Bro now requires CMake >= 2.6.3.
* Bro now links in tcmalloc (part of Google perftools) if found at
configure time. Doing so can significantly improve memory and
CPU use.
- Bro now supports IPv6 out of the box; the configure switch
--enable-brov6 is gone.
@ -31,6 +35,14 @@ Bro 2.1
- The syntax for IPv6 literals changed from "2607:f8b0:4009:802::1012"
to "[2607:f8b0:4009:802::1012]".
- Bro now spawn threads for doing its logging. From a user's
perspective not much should change, except that the OS may now show
a bunch of Bro threads.
- We renamed the configure option --enable-perftools to
--enable-perftool-debug to indicate that the switch is only relevant
for debugging the heap.
TODO: Extend.
Bro 2.0

2
VERSION
View file

@ -1 +1 @@
2.0-184
2.0-245

9
configure vendored
View file

@ -27,7 +27,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
Optional Features:
--enable-debug compile in debugging mode
--enable-perftools use Google's perftools
--enable-perftools-debug use Google's perftools for debugging
--disable-broccoli don't build or install the Broccoli library
--disable-broctl don't install Broctl
--disable-auxtools don't build or install auxilliary tools
@ -91,7 +91,7 @@ append_cache_entry BRO_ROOT_DIR PATH /usr/local/bro
append_cache_entry PY_MOD_INSTALL_DIR PATH /usr/local/bro/lib/broctl
append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING /usr/local/bro/share/bro
append_cache_entry ENABLE_DEBUG BOOL false
append_cache_entry ENABLE_PERFTOOLS BOOL false
append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
append_cache_entry BinPAC_SKIP_INSTALL BOOL true
append_cache_entry BUILD_SHARED_LIBS BOOL true
append_cache_entry INSTALL_AUX_TOOLS BOOL true
@ -132,8 +132,8 @@ while [ $# -ne 0 ]; do
--enable-debug)
append_cache_entry ENABLE_DEBUG BOOL true
;;
--enable-perftools)
append_cache_entry ENABLE_PERFTOOLS BOOL true
--enable-perftools-debug)
append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL true
;;
--disable-broccoli)
append_cache_entry INSTALL_BROCCOLI BOOL false
@ -178,7 +178,6 @@ while [ $# -ne 0 ]; do
append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg
;;
--with-perftools=*)
append_cache_entry ENABLE_PERFTOOLS BOOL true
append_cache_entry GooglePerftools_ROOT_DIR PATH $optarg
;;
--with-python=*)

2
src/Anon.cc
View file

@ -154,7 +154,7 @@ void AnonymizeIPAddr_A50::init()
int AnonymizeIPAddr_A50::PreservePrefix(ipaddr32_t input, int num_bits)
{
DEBUG_MSG("%s/%d\n",
IPAddr(IPAddr::IPv4, &input, IPAddr::Network).AsString().c_str(),
IPAddr(IPv4, &input, IPAddr::Network).AsString().c_str(),
num_bits);
if ( ! before_anonymization )

4
src/Attr.cc
View file

@ -5,7 +5,7 @@
#include "Attr.h"
#include "Expr.h"
#include "Serializer.h"
#include "LogMgr.h"
#include "threading/SerialTypes.h"
const char* attr_name(attr_tag t)
{
@ -416,7 +416,7 @@ void Attributes::CheckAttr(Attr* a)
break;
case ATTR_LOG:
if ( ! LogVal::IsCompatibleType(type) )
if ( ! threading::Value::IsCompatibleType(type) )
Error("&log applied to a type that cannot be logged");
break;

20
src/CMakeLists.txt
View file

@ -213,6 +213,8 @@ binpac_target(syslog.pac
########################################################################
## bro target
find_package (Threads)
# This macro stores associated headers for any C/C++ source files given
# as arguments (past _var) as a list in the CMake variable named "_var".
macro(COLLECT_HEADERS _var)
@ -335,10 +337,6 @@ set(bro_SRCS
IRC.cc
List.cc
Reporter.cc
LogMgr.cc
LogWriter.cc
LogWriterAscii.cc
LogWriterNone.cc
Login.cc
MIME.cc
NCP.cc
@ -411,6 +409,18 @@ set(bro_SRCS
PacketDumper.cc
strsep.c
modp_numtoa.c
threading/BasicThread.cc
threading/Manager.cc
threading/MsgThread.cc
threading/SerialTypes.cc
logging/Manager.cc
logging/WriterBackend.cc
logging/WriterFrontend.cc
logging/writers/Ascii.cc
logging/writers/None.cc
${dns_SRCS}
${openssl_SRCS}
)
@ -423,7 +433,7 @@ add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}")
add_executable(bro ${bro_SRCS} ${bro_HEADERS})
target_link_libraries(bro ${brodeps})
target_link_libraries(bro ${brodeps} ${CMAKE_THREAD_LIBS_INIT})
install(TARGETS bro DESTINATION bin)
install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)

2
src/CompHash.cc
View file

@ -709,7 +709,7 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0,
const uint32* const kp = AlignType<uint32>(kp0);
kp1 = reinterpret_cast<const char*>(kp+4);
IPAddr addr(IPAddr::IPv6, kp, IPAddr::Network);
IPAddr addr(IPv6, kp, IPAddr::Network);
switch ( tag ) {
case TYPE_ADDR:

4
src/DCE_RPC.cc
View file

@ -137,7 +137,7 @@ static bool is_mapped_dce_rpc_endpoint(const dce_rpc_endpoint_addr& addr)
bool is_mapped_dce_rpc_endpoint(const ConnID* id, TransportProto proto)
{
if ( id->dst_addr.GetFamily() == IPAddr::IPv6 )
if ( id->dst_addr.GetFamily() == IPv6 )
// TODO: Does the protocol support v6 addresses? #773
return false;
@ -414,7 +414,7 @@ void DCE_RPC_Session::DeliverEpmapperMapResponse(
case binpac::DCE_RPC_Simple::EPM_PROTOCOL_IP:
uint32 hostip = floor->rhs()->data()->ip();
mapped.addr.addr = IPAddr(IPAddr::IPv4, &hostip, IPAddr::Host);
mapped.addr.addr = IPAddr(IPv4, &hostip, IPAddr::Host);
break;
}
}

4
src/DNS_Mgr.cc
View file

@ -321,10 +321,10 @@ void DNS_Mapping::Init(struct hostent* h)
addrs = new IPAddr[num_addrs];
for ( int i = 0; i < num_addrs; ++i )
if ( h->h_addrtype == AF_INET )
addrs[i] = IPAddr(IPAddr::IPv4, (uint32*)h->h_addr_list[i],
addrs[i] = IPAddr(IPv4, (uint32*)h->h_addr_list[i],
IPAddr::Network);
else if ( h->h_addrtype == AF_INET6 )
addrs[i] = IPAddr(IPAddr::IPv6, (uint32*)h->h_addr_list[i],
addrs[i] = IPAddr(IPv6, (uint32*)h->h_addr_list[i],
IPAddr::Network);
}
else

2
src/DPM.cc
View file

@ -74,7 +74,7 @@ void DPM::PostScriptInit()
void DPM::AddConfig(const Analyzer::Config& cfg)
{
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
HeapLeakChecker::Disabler disabler;
#endif

2
src/DebugLogger.cc
View file

@ -15,7 +15,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
{ "compressor", 0, false }, {"string", 0, false },
{ "notifiers", 0, false }, { "main-loop", 0, false },
{ "dpd", 0, false }, { "tm", 0, false },
{ "logging", 0, false }
{ "logging", 0, false }, { "threading", 0, false }
};
DebugLogger::DebugLogger(const char* filename)

1
src/DebugLogger.h
View file

@ -24,6 +24,7 @@ enum DebugStream {
DBG_DPD, // Dynamic application detection framework
DBG_TM, // Time-machine packet input via Brocolli
DBG_LOGGING, // Logging streams
DBG_THREADING, // Threading system
NUM_DBGS // Has to be last
};

10
src/Desc.cc
View file

@ -157,6 +157,16 @@ void ODesc::Add(double d)
}
}
void ODesc::Add(const IPAddr& addr)
{
Add(addr.AsString());
}
void ODesc::Add(const IPPrefix& prefix)
{
Add(prefix.AsString());
}
void ODesc::AddCS(const char* s)
{
int n = strlen(s);

7
src/Desc.h
View file

@ -8,7 +8,6 @@
#include <utility>
#include "BroString.h"
#include "IPAddr.h"
typedef enum {
DESC_READABLE,
@ -23,6 +22,8 @@ typedef enum {
} desc_style;
class BroFile;
class IPAddr;
class IPPrefix;
class ODesc {
public:
@ -76,8 +77,8 @@ public:
void Add(int64 i);
void Add(uint64 u);
void Add(double d);
void Add(const IPAddr& addr) { Add(addr.AsString()); }
void Add(const IPPrefix& prefix) { Add(prefix.AsString()); }
void Add(const IPAddr& addr);
void Add(const IPPrefix& prefix);
// Add s as a counted string.
void AddCS(const char* s);

4
src/File.cc
View file

@ -232,7 +232,7 @@ BroFile::~BroFile()
delete [] access;
delete [] cipher_buffer;
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
heap_checker->UnIgnoreObject(this);
#endif
}
@ -255,7 +255,7 @@ void BroFile::Init()
cipher_ctx = 0;
cipher_buffer = 0;
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
heap_checker->IgnoreObject(this);
#endif
}

2
src/ID.cc
View file

@ -372,7 +372,7 @@ ID* ID::Unserialize(UnserialInfo* info)
Ref(id);
global_scope()->Insert(id->Name(), id);
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
heap_checker->IgnoreObject(id);
#endif
}

6
src/IP.cc
View file

@ -74,8 +74,8 @@ RecordVal* IPv6_Hdr::BuildRecordVal(VectorVal* chain) const
rv->Assign(2, new Val(ntohs(ip6->ip6_plen), TYPE_COUNT));
rv->Assign(3, new Val(ip6->ip6_nxt, TYPE_COUNT));
rv->Assign(4, new Val(ip6->ip6_hlim, TYPE_COUNT));
rv->Assign(5, new AddrVal(ip6->ip6_src));
rv->Assign(6, new AddrVal(ip6->ip6_dst));
rv->Assign(5, new AddrVal(IPAddr(ip6->ip6_src)));
rv->Assign(6, new AddrVal(IPAddr(ip6->ip6_dst)));
if ( ! chain )
chain = new VectorVal(new VectorType(
hdrType(ip6_ext_hdr_type, "ip6_ext_hdr")->Ref()));
@ -314,7 +314,7 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, bool set_next, uint16 next)
if ( ((const struct ip6_rthdr*)hdrs)->ip6r_segleft > 0 )
{
const in6_addr* a = (const in6_addr*)(hdrs+len-16);
reporter->Weird(src, *a, "routing0_segleft");
reporter->Weird(src, IPAddr(*a), "routing0_segleft");
}
else
{

4
src/IPAddr.cc
View file

@ -251,7 +251,7 @@ IPPrefix::IPPrefix(const in6_addr& in6, uint8_t length)
IPPrefix::IPPrefix(const IPAddr& addr, uint8_t length)
: prefix(addr)
{
if ( prefix.GetFamily() == IPAddr::IPv4 )
if ( prefix.GetFamily() == IPv4 )
{
if ( length > 32 )
reporter->InternalError("Bad IPAddr(v4) IPPrefix length : %d",
@ -276,7 +276,7 @@ string IPPrefix::AsString() const
{
char l[16];
if ( prefix.GetFamily() == IPAddr::IPv4 )
if ( prefix.GetFamily() == IPv4 )
modp_uitoa10(length - 96, l);
else
modp_uitoa10(length, l);

47
src/IPAddr.h
View file

@ -10,6 +10,8 @@
#include "BroString.h"
#include "Hash.h"
#include "util.h"
#include "Type.h"
#include "threading/SerialTypes.h"
struct ConnID;
class ExpectedConn;
@ -25,7 +27,7 @@ public:
/**
* Address family.
*/
enum Family { IPv4, IPv6 };
typedef IPFamily Family;
/**
* Byte order.
@ -45,7 +47,7 @@ public:
*
* @param in6 The IPv6 address.
*/
IPAddr(const in4_addr& in4)
explicit IPAddr(const in4_addr& in4)
{
memcpy(in6.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
memcpy(&in6.s6_addr[12], &in4.s_addr, sizeof(in4.s_addr));
@ -56,7 +58,7 @@ public:
*
* @param in6 The IPv6 address.
*/
IPAddr(const in6_addr& arg_in6) : in6(arg_in6) { }
explicit IPAddr(const in6_addr& arg_in6) : in6(arg_in6) { }
/**
* Constructs an address instance from a string representation.
@ -318,14 +320,19 @@ public:
return memcmp(&addr1.in6, &addr2.in6, sizeof(in6_addr)) < 0;
}
/** Converts the address into the type used internally by the
* inter-thread communication.
*/
void ConvertToThreadingValue(threading::Value::addr_t* v) const;
friend HashKey* BuildConnIDHashKey(const ConnID& id);
friend HashKey* BuildExpectedConnHashKey(const ExpectedConn& c);
friend class IPPrefix;
unsigned int MemoryAllocation() const { return padded_sizeof(*this); }
private:
friend class IPPrefix;
/**
* Initializes an address instance from a string representation.
*
@ -384,6 +391,25 @@ inline bool IPAddr::IsLoopback() const
&& (in6.s6_addr[14] == 0) && (in6.s6_addr[15] == 1));
}
inline void IPAddr::ConvertToThreadingValue(threading::Value::addr_t* v) const
{
v->family = GetFamily();
switch ( v->family ) {
case IPv4:
CopyIPv4(&v->in.in4);
return;
case IPv6:
CopyIPv6(&v->in.in6);
return;
// Can't be reached.
abort();
}
}
/**
* Returns a hash key for a given ConnID. Passes ownership to caller.
*/
@ -459,7 +485,7 @@ public:
*/
uint8_t Length() const
{
return prefix.GetFamily() == IPAddr::IPv4 ? length - 96 : length;
return prefix.GetFamily() == IPv4 ? length - 96 : length;
}
/**
@ -516,6 +542,15 @@ public:
return new HashKey(&key, sizeof(key));
}
/** Converts the prefix into the type used internally by the
* inter-thread communication.
*/
void ConvertToThreadingValue(threading::Value::subnet_t* v) const
{
v->length = length;
prefix.ConvertToThreadingValue(&v->prefix);
}
unsigned int MemoryAllocation() const { return padded_sizeof(*this); }
/**

143
src/LogMgr.h
View file

@ -1,143 +0,0 @@
// See the file "COPYING" in the main distribution directory for copyright.
//
// A class managing log writers and filters.
#ifndef LOGMGR_H
#define LOGMGR_H
#include "Val.h"
#include "EventHandler.h"
#include "RemoteSerializer.h"
#include "IPAddr.h"
class SerializationFormat;
// Description of a log field.
struct LogField {
string name;
TypeTag type;
// inner type of sets
TypeTag subtype;
LogField() { subtype = TYPE_VOID; }
LogField(const LogField& other)
: name(other.name), type(other.type), subtype(other.subtype) { }
// (Un-)serialize.
bool Read(SerializationFormat* fmt);
bool Write(SerializationFormat* fmt) const;
};
// Values as logged by a writer.
struct LogVal {
TypeTag type;
bool present; // False for unset fields.
// The following union is a subset of BroValUnion, including only the
// types we can log directly.
struct set_t { bro_int_t size; LogVal** vals; };
typedef set_t vec_t;
union _val {
bro_int_t int_val;
bro_uint_t uint_val;
IPAddr* addr_val;
IPPrefix* subnet_val;
double double_val;
string* string_val;
set_t set_val;
vec_t vector_val;
} val;
LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true)
: type(arg_type), present(arg_present) {}
~LogVal();
// (Un-)serialize.
bool Read(SerializationFormat* fmt);
bool Write(SerializationFormat* fmt) const;
// Returns true if the type can be logged the framework. If
// `atomic_only` is true, will not permit composite types.
static bool IsCompatibleType(BroType* t, bool atomic_only=false);
private:
LogVal(const LogVal& other) { }
};
class LogWriter;
class RemoteSerializer;
class RotationTimer;
class LogMgr {
public:
LogMgr();
~LogMgr();
// These correspond to the BiFs visible on the scripting layer. The
// actual BiFs just forward here.
bool CreateStream(EnumVal* id, RecordVal* stream);
bool EnableStream(EnumVal* id);
bool DisableStream(EnumVal* id);
bool AddFilter(EnumVal* id, RecordVal* filter);
bool RemoveFilter(EnumVal* id, StringVal* name);
bool RemoveFilter(EnumVal* id, string name);
bool Write(EnumVal* id, RecordVal* columns);
bool SetBuf(EnumVal* id, bool enabled); // Adjusts all writers.
bool Flush(EnumVal* id); // Flushes all writers..
protected:
friend class LogWriter;
friend class RemoteSerializer;
friend class RotationTimer;
//// Function also used by the RemoteSerializer.
// Takes ownership of fields.
LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path,
int num_fields, LogField** fields);
// Takes ownership of values..
bool Write(EnumVal* id, EnumVal* writer, string path,
int num_fields, LogVal** vals);
// Announces all instantiated writers to peer.
void SendAllWritersTo(RemoteSerializer::PeerID peer);
//// Functions safe to use by writers.
// Signals that a file has been rotated.
bool FinishedRotation(LogWriter* writer, string new_name, string old_name,
double open, double close, bool terminating);
// Reports an error for the given writer.
void Error(LogWriter* writer, const char* msg);
// Deletes the values as passed into Write().
void DeleteVals(int num_fields, LogVal** vals);
private:
struct Filter;
struct Stream;
struct WriterInfo;
bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
TableVal* include, TableVal* exclude, string path, list<int> indices);
LogVal** RecordToFilterVals(Stream* stream, Filter* filter,
RecordVal* columns);
LogVal* ValToLogVal(Val* val, BroType* ty = 0);
Stream* FindStream(EnumVal* id);
void RemoveDisabledWriters(Stream* stream);
void InstallRotationTimer(WriterInfo* winfo);
void Rotate(WriterInfo* info);
Filter* FindFilter(EnumVal* id, StringVal* filter);
WriterInfo* FindWriter(LogWriter* writer);
vector<Stream *> streams; // Indexed by stream enum.
};
extern LogMgr* log_mgr;
#endif

158
src/LogWriter.cc
View file

@ -1,158 +0,0 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "util.h"
#include "LogWriter.h"
LogWriter::LogWriter()
{
buf = 0;
buf_len = 1024;
buffering = true;
disabled = false;
}
LogWriter::~LogWriter()
{
if ( buf )
free(buf);
for(int i = 0; i < num_fields; ++i)
delete fields[i];
delete [] fields;
}
bool LogWriter::Init(string arg_path, int arg_num_fields,
const LogField* const * arg_fields)
{
path = arg_path;
num_fields = arg_num_fields;
fields = arg_fields;
if ( ! DoInit(arg_path, arg_num_fields, arg_fields) )
{
disabled = true;
return false;
}
return true;
}
bool LogWriter::Write(int arg_num_fields, LogVal** vals)
{
// Double-check that the arguments match. If we get this from remote,
// something might be mixed up.
if ( num_fields != arg_num_fields )
{
DBG_LOG(DBG_LOGGING, "Number of fields don't match in LogWriter::Write() (%d vs. %d)",
arg_num_fields, num_fields);
DeleteVals(vals);
return false;
}
for ( int i = 0; i < num_fields; ++i )
{
if ( vals[i]->type != fields[i]->type )
{
DBG_LOG(DBG_LOGGING, "Field type doesn't match in LogWriter::Write() (%d vs. %d)",
vals[i]->type, fields[i]->type);
DeleteVals(vals);
return false;
}
}
bool result = DoWrite(num_fields, fields, vals);
DeleteVals(vals);
if ( ! result )
disabled = true;
return result;
}
bool LogWriter::SetBuf(bool enabled)
{
if ( enabled == buffering )
// No change.
return true;
buffering = enabled;
if ( ! DoSetBuf(enabled) )
{
disabled = true;
return false;
}
return true;
}
bool LogWriter::Rotate(string rotated_path, double open,
double close, bool terminating)
{
if ( ! DoRotate(rotated_path, open, close, terminating) )
{
disabled = true;
return false;
}
return true;
}
bool LogWriter::Flush()
{
if ( ! DoFlush() )
{
disabled = true;
return false;
}
return true;
}
void LogWriter::Finish()
{
DoFinish();
}
const char* LogWriter::Fmt(const char* format, ...)
{
if ( ! buf )
buf = (char*) malloc(buf_len);
va_list al;
va_start(al, format);
int n = safe_vsnprintf(buf, buf_len, format, al);
va_end(al);
if ( (unsigned int) n >= buf_len )
{ // Not enough room, grow the buffer.
buf_len = n + 32;
buf = (char*) realloc(buf, buf_len);
// Is it portable to restart?
va_start(al, format);
n = safe_vsnprintf(buf, buf_len, format, al);
va_end(al);
}
return buf;
}
void LogWriter::Error(const char *msg)
{
log_mgr->Error(this, msg);
}
void LogWriter::DeleteVals(LogVal** vals)
{
log_mgr->DeleteVals(num_fields, vals);
}
bool LogWriter::FinishedRotation(string new_name, string old_name, double open,
double close, bool terminating)
{
return log_mgr->FinishedRotation(this, new_name, old_name, open, close, terminating);
}

192
src/LogWriter.h
View file

@ -1,192 +0,0 @@
// See the file "COPYING" in the main distribution directory for copyright.
//
// Interface API for a log writer backend. The LogMgr creates a separate
// writer instance of pair of (writer type, output path).
//
// Note thay classes derived from LogWriter must be fully thread-safe and not
// use any non-thread-safe Bro functionality (which includes almost
// everything ...). In particular, do not use fmt() but LogWriter::Fmt()!.
//
// The one exception to this rule is the constructor: it is guaranteed to be
// executed inside the main thread and can thus in particular access global
// script variables.
#ifndef LOGWRITER_H
#define LOGWRITER_H
#include "LogMgr.h"
#include "BroString.h"
class LogWriter {
public:
LogWriter();
virtual ~LogWriter();
//// Interface methods to interact with the writer. Note that these
//// methods are not necessarily thread-safe and must be called only
//// from the main thread (which will typically mean only from the
//// LogMgr). In particular, they must not be called from the
//// writer's derived implementation.
// One-time initialization of the writer to define the logged fields.
// Interpretation of "path" is left to the writer, and will be
// corresponding the value configured on the script-level.
//
// Returns false if an error occured, in which case the writer must
// not be used further.
//
// The new instance takes ownership of "fields", and will delete them
// when done.
bool Init(string path, int num_fields, const LogField* const * fields);
// Writes one log entry. The method takes ownership of "vals" and
// will return immediately after queueing the write request, which is
// potentially before output has actually been written out.
//
// num_fields and the types of the LogVals must match what was passed
// to Init().
//
// Returns false if an error occured, in which case the writer must
// not be used any further.
bool Write(int num_fields, LogVal** vals);
// Sets the buffering status for the writer, if the writer supports
// that. (If not, it will be ignored).
bool SetBuf(bool enabled);
// Flushes any currently buffered output, if the writer supports
// that. (If not, it will be ignored).
bool Flush();
// Triggers rotation, if the writer supports that. (If not, it will
// be ignored).
bool Rotate(string rotated_path, double open, double close, bool terminating);
// Finishes writing to this logger regularly. Must not be called if
// an error has been indicated earlier. After calling this, no
// further writing must be performed.
void Finish();
//// Thread-safe methods that may be called from the writer
//// implementation.
// The following methods return the information as passed to Init().
const string Path() const { return path; }
int NumFields() const { return num_fields; }
const LogField* const * Fields() const { return fields; }
protected:
// Methods for writers to override. If any of these returs false, it
// will be assumed that a fatal error has occured that prevents the
// writer from further operation. It will then be disabled and
// deleted. When return false, the writer should also report the
// error via Error(). Note that even if a writer does not support the
// functionality for one these methods (like rotation), it must still
// return true if that is not to be considered a fatal error.
//
// Called once for initialization of the writer.
virtual bool DoInit(string path, int num_fields,
const LogField* const * fields) = 0;
// Called once per log entry to record.
virtual bool DoWrite(int num_fields, const LogField* const * fields,
LogVal** vals) = 0;
// Called when the buffering status for this writer is changed. If
// buffering is disabled, the writer should attempt to write out
// information as quickly as possible even if doing so may have a
// performance impact. If enabled (which is the default), it may
// buffer data as helpful and write it out later in a way optimized
// for performance. The current buffering state can be queried via
// IsBuf().
//
// A writer may ignore buffering changes if it doesn't fit with its
// semantics (but must still return true in that case).
virtual bool DoSetBuf(bool enabled) = 0;
// Called to flush any currently buffered output.
//
// A writer may ignore flush requests if it doesn't fit with its
// semantics (but must still return true in that case).
virtual bool DoFlush() = 0;
// Called when a log output is to be rotated. Most directly this only
// applies to writers writing into files, which should then close the
// current file and open a new one. However, a writer may also
// trigger other apppropiate actions if semantics are similar.
//
// Once rotation has finished, the implementation should call
// RotationDone() to signal the log manager that potential
// postprocessors can now run.
//
// "rotate_path" reflects the path to where the rotated output is to
// be moved, with specifics depending on the writer. It should
// generally be interpreted in a way consistent with that of "path"
// as passed into DoInit(). As an example, for file-based output,
// "rotate_path" could be the original filename extended with a
// timestamp indicating the time of the rotation.
//
// "open" and "close" are the network time's when the *current* file
// was opened and closed, respectively.
//
// "terminating" indicated whether the rotation request occurs due
// the main Bro prcoess terminating (and not because we've reach a
// regularly scheduled time for rotation).
//
// A writer may ignore rotation requests if it doesn't fit with its
// semantics (but must still return true in that case).
virtual bool DoRotate(string rotated_path, double open, double close,
bool terminating) = 0;
// Called once on termination. Not called when any of the other
// methods has previously signaled an error, i.e., executing this
// method signals a regular shutdown of the writer.
virtual void DoFinish() = 0;
//// Methods for writers to use. These are thread-safe.
// A thread-safe version of fmt().
const char* Fmt(const char* format, ...);
// Returns the current buffering state.
bool IsBuf() { return buffering; }
// Reports an error to the user.
void Error(const char *msg);
// Signals to the log manager that a file has been rotated.
//
// new_name: The filename of the rotated file. old_name: The filename
// of the origina file.
//
// open/close: The timestamps when the original file was opened and
// closed, respectively.
//
// terminating: True if rotation request occured due to the main Bro
// process shutting down.
bool FinishedRotation(string new_name, string old_name, double open,
double close, bool terminating);
private:
friend class LogMgr;
// When an error occurs, we call this method to set a flag marking
// the writer as disabled. The LogMgr will check the flag later and
// remove the writer.
bool Disabled() { return disabled; }
// Deletes the values as passed into Write().
void DeleteVals(LogVal** vals);
string path;
int num_fields;
const LogField* const * fields;
bool buffering;
bool disabled;
// For implementing Fmt().
char* buf;
unsigned int buf_len;
};
#endif

30
src/LogWriterNone.h
View file

@ -1,30 +0,0 @@
// See the file "COPYING" in the main distribution directory for copyright.
//
// Dummy log writer that just discards everything (but still pretends to rotate).
#ifndef LOGWRITERNONE_H
#define LOGWRITERNONE_H
#include "LogWriter.h"
class LogWriterNone : public LogWriter {
public:
LogWriterNone() {}
~LogWriterNone() {};
static LogWriter* Instantiate() { return new LogWriterNone; }
protected:
virtual bool DoInit(string path, int num_fields,
const LogField* const * fields) { return true; }
virtual bool DoWrite(int num_fields, const LogField* const * fields,
LogVal** vals) { return true; }
virtual bool DoSetBuf(bool enabled) { return true; }
virtual bool DoRotate(string rotated_path, double open, double close,
bool terminating);
virtual bool DoFlush() { return true; }
virtual void DoFinish() {}
};
#endif

2
src/Login.cc
View file

@ -38,7 +38,7 @@ Login_Analyzer::Login_Analyzer(AnalyzerTag::Tag tag, Connection* conn)
if ( ! re_skip_authentication )
{
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
HeapLeakChecker::Disabler disabler;
#endif
re_skip_authentication = init_RE(skip_authentication);

3
src/Net.cc
View file

@ -455,6 +455,7 @@ void net_run()
// date on timers and events.
network_time = ct;
expire_timers();
usleep(1); // Just yield.
}
}
@ -484,6 +485,8 @@ void net_run()
// since Bro timers are not high-precision anyway.)
if ( ! using_communication )
usleep(100000);
else
usleep(1000);
// Flawfinder says about usleep:
//

2
src/PersistenceSerializer.cc
View file

@ -137,7 +137,7 @@ bool PersistenceSerializer::CheckForFile(UnserialInfo* info, const char* file,
bool PersistenceSerializer::ReadAll(bool is_init, bool delete_files)
{
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
HeapLeakChecker::Disabler disabler;
#endif

58
src/RemoteSerializer.cc
View file

@ -183,8 +183,9 @@
#include "Sessions.h"
#include "File.h"
#include "Conn.h"
#include "LogMgr.h"
#include "Reporter.h"
#include "threading/SerialTypes.h"
#include "logging/Manager.h"
#include "IPAddr.h"
#include "bro_inet_ntop.h"
@ -532,6 +533,7 @@ RemoteSerializer::RemoteSerializer()
terminating = false;
in_sync = 0;
last_flush = 0;
received_logs = 0;
}
RemoteSerializer::~RemoteSerializer()
@ -681,7 +683,7 @@ RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip,
if ( ! initialized )
reporter->InternalError("remote serializer not initialized");
if ( ip.GetFamily() == IPAddr::IPv6 )
if ( ip.GetFamily() == IPv6 )
Error("inter-Bro communication not supported over IPv6");
const uint32* bytes;
@ -1238,7 +1240,7 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl)
if ( ! initialized )
reporter->InternalError("remote serializer not initialized");
if ( ip.GetFamily() == IPAddr::IPv6 )
if ( ip.GetFamily() == IPv6 )
Error("inter-Bro communication not supported over IPv6");
const uint32* bytes;
@ -1353,6 +1355,14 @@ double RemoteSerializer::NextTimestamp(double* local_network_time)
{
Poll(false);
if ( received_logs > 0 )
{
// If we processed logs last time, assume there's more.
idle = false;
received_logs = 0;
return timer_mgr->Time();
}
double et = events.length() ? events[0]->time : -1;
double pt = packets.length() ? packets[0]->time : -1;
@ -2470,7 +2480,7 @@ bool RemoteSerializer::ProcessRemotePrint()
return true;
}
bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields)
bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields)
{
loop_over_list(peers, i)
{
@ -2480,7 +2490,7 @@ bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string
return true;
}
bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields)
bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields)
{
SetErrorDescr("logging");
@ -2534,7 +2544,7 @@ error:
return false;
}
bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals)
bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals)
{
loop_over_list(peers, i)
{
@ -2544,7 +2554,7 @@ bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, i
return true;
}
bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals)
bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals)
{
if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING )
return false;
@ -2552,7 +2562,9 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st
if ( ! peer->logs_requested )
return false;
assert(peer->log_buffer);
if ( ! peer->log_buffer )
// Peer shutting down.
return false;
// Serialize the log record entry.
@ -2587,8 +2599,11 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st
if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) || (network_time - last_flush > 1.0) )
{
if ( ! FlushLogBuffer(peer) )
{
delete [] data;
return false;
}
}
// If the data is actually larger than our complete buffer, just send it out.
if ( len > LOG_BUFFER_SIZE )
@ -2631,11 +2646,17 @@ bool RemoteSerializer::ProcessLogCreateWriter()
if ( current_peer->state == Peer::CLOSING )
return false;
#ifdef USE_PERFTOOLS_DEBUG
// Don't track allocations here, they'll be released only after the
// main loop exists. And it's just a tiny amount anyway.
HeapLeakChecker::Disabler disabler;
#endif
assert(current_args);
EnumVal* id_val = 0;
EnumVal* writer_val = 0;
LogField** fields = 0;
threading::Field** fields = 0;
BinarySerializationFormat fmt;
fmt.StartRead(current_args->data, current_args->len);
@ -2652,11 +2673,11 @@ bool RemoteSerializer::ProcessLogCreateWriter()
if ( ! success )
goto error;
fields = new LogField* [num_fields];
fields = new threading::Field* [num_fields];
for ( int i = 0; i < num_fields; i++ )
{
fields[i] = new LogField;
fields[i] = new threading::Field;
if ( ! fields[i]->Read(&fmt) )
goto error;
}
@ -2666,7 +2687,7 @@ bool RemoteSerializer::ProcessLogCreateWriter()
id_val = new EnumVal(id, BifType::Enum::Log::ID);
writer_val = new EnumVal(writer, BifType::Enum::Log::Writer);
if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields) )
if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields, true, false) )
goto error;
Unref(id_val);
@ -2697,7 +2718,7 @@ bool RemoteSerializer::ProcessLogWrite()
// Unserialize one entry.
EnumVal* id_val = 0;
EnumVal* writer_val = 0;
LogVal** vals = 0;
threading::Value** vals = 0;
int id, writer;
string path;
@ -2711,11 +2732,11 @@ bool RemoteSerializer::ProcessLogWrite()
if ( ! success )
goto error;
vals = new LogVal* [num_fields];
vals = new threading::Value* [num_fields];
for ( int i = 0; i < num_fields; i++ )
{
vals[i] = new LogVal;
vals[i] = new threading::Value;
if ( ! vals[i]->Read(&fmt) )
goto error;
}
@ -2735,6 +2756,8 @@ bool RemoteSerializer::ProcessLogWrite()
fmt.EndRead();
++received_logs;
return true;
error:
@ -2844,7 +2867,7 @@ void RemoteSerializer::GotID(ID* id, Val* val)
(desc && *desc) ? desc : "not set"),
current_peer);
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
// May still be cached, but we don't care.
heap_checker->IgnoreObject(id);
#endif
@ -3376,6 +3399,9 @@ void SocketComm::Run()
small_timeout.tv_usec =
io->CanWrite() || io->CanRead() ? 1 : 10;
if ( ! io->CanWrite() )
usleep(10);
int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except,
&small_timeout);

16
src/RemoteSerializer.h
View file

@ -14,8 +14,11 @@
// FIXME: Change this to network byte order
class IncrementalSendTimer;
class LogField;
class LogVal;
namespace threading {
class Field;
class Value;
}
// This class handles the communication done in Bro's main loop.
class RemoteSerializer : public Serializer, public IOSource {
@ -99,13 +102,13 @@ public:
bool SendPrintHookEvent(BroFile* f, const char* txt, size_t len);
// Send a request to create a writer on a remote side.
bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields);
bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields);
// Broadcasts a request to create a writer.
bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields);
bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields);
// Broadcast a log entry to everybody interested.
bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals);
bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals);
// Synchronzizes time with all connected peers. Returns number of
// current sync-point, or -1 on error.
@ -300,7 +303,7 @@ protected:
bool SendID(SerialInfo* info, Peer* peer, const ID& id);
bool SendCapabilities(Peer* peer);
bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p);
bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals);
bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals);
void UnregisterHandlers(Peer* peer);
void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0);
@ -335,6 +338,7 @@ private:
int propagate_accesses;
bool ignore_accesses;
bool terminating;
int received_logs;
Peer* source_peer;
PeerID id_counter; // Keeps track of assigned IDs.
uint32 current_sync_point;

4
src/RuleMatcher.cc
View file

@ -191,7 +191,7 @@ void RuleMatcher::Delete(RuleHdrTest* node)
bool RuleMatcher::ReadFiles(const name_list& files)
{
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
HeapLeakChecker::Disabler disabler;
#endif
@ -1082,7 +1082,7 @@ static bool val_to_maskedval(Val* v, maskedvalue_list* append_to)
bool is_v4_mask = m[0] == 0xffffffff &&
m[1] == m[0] && m[2] == m[0];
if ( v->AsSubNet().Prefix().GetFamily() == IPAddr::IPv4 &&
if ( v->AsSubNet().Prefix().GetFamily() == IPv4 &&
is_v4_mask )
{
mval->val = ntohl(*n);

77
src/SerializationFormat.cc
View file

@ -250,9 +250,9 @@ bool BinarySerializationFormat::Read(IPAddr* addr, const char* tag)
}
if ( n == 1 )
*addr = IPAddr(IPAddr::IPv4, raw, IPAddr::Network);
*addr = IPAddr(IPv4, raw, IPAddr::Network);
else
*addr = IPAddr(IPAddr::IPv6, raw, IPAddr::Network);
*addr = IPAddr(IPv6, raw, IPAddr::Network);
return true;
}
@ -269,6 +269,32 @@ bool BinarySerializationFormat::Read(IPPrefix* prefix, const char* tag)
return true;
}
bool BinarySerializationFormat::Read(struct in_addr* addr, const char* tag)
{
uint32_t* bytes = (uint32_t*) &addr->s_addr;
if ( ! Read(&bytes[0], "addr4") )
return false;
bytes[0] = htonl(bytes[0]);
return true;
}
bool BinarySerializationFormat::Read(struct in6_addr* addr, const char* tag)
{
uint32_t* bytes = (uint32_t*) &addr->s6_addr;
for ( int i = 0; i < 4; ++i )
{
if ( ! Read(&bytes[i], "addr6-part") )
return false;
bytes[i] = htonl(bytes[i]);
}
return true;
}
bool BinarySerializationFormat::Write(char v, const char* tag)
{
DBG_LOG(DBG_SERIAL, "Write char %s [%s]", fmt_bytes(&v, 1), tag);
@ -362,6 +388,29 @@ bool BinarySerializationFormat::Write(const IPPrefix& prefix, const char* tag)
return Write(prefix.Prefix(), "prefix") && Write(prefix.Length(), "width");
}
bool BinarySerializationFormat::Write(const struct in_addr& addr, const char* tag)
{
const uint32_t* bytes = (uint32_t*) &addr.s_addr;
if ( ! Write(ntohl(bytes[0]), "addr4") )
return false;
return true;
}
bool BinarySerializationFormat::Write(const struct in6_addr& addr, const char* tag)
{
const uint32_t* bytes = (uint32_t*) &addr.s6_addr;
for ( int i = 0; i < 4; ++i )
{
if ( ! Write(ntohl(bytes[i]), "addr6-part") )
return false;
}
return true;
}
bool BinarySerializationFormat::WriteOpenTag(const char* tag)
{
return true;
@ -464,6 +513,18 @@ bool XMLSerializationFormat::Read(IPPrefix* prefix, const char* tag)
return false;
}
bool XMLSerializationFormat::Read(struct in_addr* addr, const char* tag)
{
reporter->InternalError("no reading of xml");
return false;
}
bool XMLSerializationFormat::Read(struct in6_addr* addr, const char* tag)
{
reporter->InternalError("no reading of xml");
return false;
}
bool XMLSerializationFormat::Write(char v, const char* tag)
{
return WriteElem(tag, "char", &v, 1);
@ -556,6 +617,18 @@ bool XMLSerializationFormat::Write(const IPPrefix& prefix, const char* tag)
return false;
}
bool XMLSerializationFormat::Write(const struct in_addr& addr, const char* tag)
{
reporter->InternalError("XML output of in_addr not implemented");
return false;
}
bool XMLSerializationFormat::Write(const struct in6_addr& addr, const char* tag)
{
reporter->InternalError("XML output of in6_addr not implemented");
return false;
}
bool XMLSerializationFormat::WriteEncodedString(const char* s, int len)
{
while ( len-- )

15
src/SerializationFormat.h
View file

@ -9,6 +9,9 @@ using namespace std;
#include "util.h"
class IPAddr;
class IPPrefix;
// Abstract base class.
class SerializationFormat {
public:
@ -30,6 +33,8 @@ public:
virtual bool Read(string* s, const char* tag) = 0;
virtual bool Read(IPAddr* addr, const char* tag) = 0;
virtual bool Read(IPPrefix* prefix, const char* tag) = 0;
virtual bool Read(struct in_addr* addr, const char* tag) = 0;
virtual bool Read(struct in6_addr* addr, const char* tag) = 0;
// Returns number of raw bytes read since last call to StartRead().
int BytesRead() const { return bytes_read; }
@ -54,6 +59,8 @@ public:
virtual bool Write(const string& s, const char* tag) = 0;
virtual bool Write(const IPAddr& addr, const char* tag) = 0;
virtual bool Write(const IPPrefix& prefix, const char* tag) = 0;
virtual bool Write(const struct in_addr& addr, const char* tag) = 0;
virtual bool Write(const struct in6_addr& addr, const char* tag) = 0;
virtual bool WriteOpenTag(const char* tag) = 0;
virtual bool WriteCloseTag(const char* tag) = 0;
@ -96,6 +103,8 @@ public:
virtual bool Read(string* s, const char* tag);
virtual bool Read(IPAddr* addr, const char* tag);
virtual bool Read(IPPrefix* prefix, const char* tag);
virtual bool Read(struct in_addr* addr, const char* tag);
virtual bool Read(struct in6_addr* addr, const char* tag);
virtual bool Write(int v, const char* tag);
virtual bool Write(uint16 v, const char* tag);
virtual bool Write(uint32 v, const char* tag);
@ -109,6 +118,8 @@ public:
virtual bool Write(const string& s, const char* tag);
virtual bool Write(const IPAddr& addr, const char* tag);
virtual bool Write(const IPPrefix& prefix, const char* tag);
virtual bool Write(const struct in_addr& addr, const char* tag);
virtual bool Write(const struct in6_addr& addr, const char* tag);
virtual bool WriteOpenTag(const char* tag);
virtual bool WriteCloseTag(const char* tag);
virtual bool WriteSeparator();
@ -133,6 +144,8 @@ public:
virtual bool Write(const string& s, const char* tag);
virtual bool Write(const IPAddr& addr, const char* tag);
virtual bool Write(const IPPrefix& prefix, const char* tag);
virtual bool Write(const struct in_addr& addr, const char* tag);
virtual bool Write(const struct in6_addr& addr, const char* tag);
virtual bool WriteOpenTag(const char* tag);
virtual bool WriteCloseTag(const char* tag);
virtual bool WriteSeparator();
@ -150,6 +163,8 @@ public:
virtual bool Read(string* s, const char* tag);
virtual bool Read(IPAddr* addr, const char* tag);
virtual bool Read(IPPrefix* prefix, const char* tag);
virtual bool Read(struct in_addr* addr, const char* tag);
virtual bool Read(struct in6_addr* addr, const char* tag);
private:
// Encodes non-printable characters.

2
src/StateAccess.cc
View file

@ -678,7 +678,7 @@ bool StateAccess::DoUnserialize(UnserialInfo* info)
target.id = new ID(name, SCOPE_GLOBAL, true);
Ref(target.id);
global_scope()->Insert(name, target.id);
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
heap_checker->IgnoreObject(target.id);
#endif
}

21
src/Stats.cc
View file

@ -8,7 +8,7 @@
#include "cq.h"
#include "DNS_Mgr.h"
#include "Trigger.h"
#include "threading/Manager.h"
int killed_by_inactivity = 0;
@ -203,6 +203,25 @@ void ProfileLogger::Log()
current_timers[i]));
}
file->Write(fmt("%0.6f Threads: current=%d\n", network_time, thread_mgr->NumThreads()));
const threading::Manager::msg_stats_list& thread_stats = thread_mgr->GetMsgThreadStats();
for ( threading::Manager::msg_stats_list::const_iterator i = thread_stats.begin();
i != thread_stats.end(); ++i )
{
threading::MsgThread::Stats s = i->second;
file->Write(fmt("%0.6f %-25s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64
" (#queue r/w: in=%" PRIu64 "/%" PRIu64 " out=%" PRIu64 "/%" PRIu64 ")"
"\n",
network_time,
i->first.c_str(),
s.sent_in, s.sent_out,
s.pending_in, s.pending_out,
s.queue_in_stats.num_reads, s.queue_in_stats.num_writes,
s.queue_out_stats.num_reads, s.queue_out_stats.num_writes
));
}
// Script-level state.
unsigned int size, mem = 0;
PDict(ID)* globals = global_scope()->Vars();

26
src/Val.cc
View file

@ -606,7 +606,7 @@ ID* MutableVal::Bind() const
ip = htonl(0x7f000001); // 127.0.0.1
safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#",
IPAddr(IPAddr::IPv4, &ip, IPAddr::Network)->AsString().c_str(),
IPAddr(IPv4, &ip, IPAddr::Network)->AsString().c_str(),
getpid());
#else
safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#", host, getpid());
@ -864,12 +864,12 @@ AddrVal::AddrVal(const char* text) : Val(TYPE_ADDR)
AddrVal::AddrVal(uint32 addr) : Val(TYPE_ADDR)
{
// ### perhaps do gethostbyaddr here?
val.addr_val = new IPAddr(IPAddr::IPv4, &addr, IPAddr::Network);
val.addr_val = new IPAddr(IPv4, &addr, IPAddr::Network);
}
AddrVal::AddrVal(const uint32 addr[4]) : Val(TYPE_ADDR)
{
val.addr_val = new IPAddr(IPAddr::IPv6, addr, IPAddr::Network);
val.addr_val = new IPAddr(IPv6, addr, IPAddr::Network);
}
AddrVal::AddrVal(const IPAddr& addr) : Val(TYPE_ADDR)
@ -889,7 +889,7 @@ unsigned int AddrVal::MemoryAllocation() const
Val* AddrVal::SizeVal() const
{
if ( val.addr_val->GetFamily() == IPAddr::IPv4 )
if ( val.addr_val->GetFamily() == IPv4 )
return new Val(32, TYPE_COUNT);
else
return new Val(128, TYPE_COUNT);
@ -933,13 +933,13 @@ SubNetVal::SubNetVal(const char* text, int width) : Val(TYPE_SUBNET)
SubNetVal::SubNetVal(uint32 addr, int width) : Val(TYPE_SUBNET)
{
IPAddr a(IPAddr::IPv4, &addr, IPAddr::Network);
IPAddr a(IPv4, &addr, IPAddr::Network);
val.subnet_val = new IPPrefix(a, width);
}
SubNetVal::SubNetVal(const uint32* addr, int width) : Val(TYPE_SUBNET)
{
IPAddr a(IPAddr::IPv6, addr, IPAddr::Network);
IPAddr a(IPv6, addr, IPAddr::Network);
val.subnet_val = new IPPrefix(a, width);
}
@ -953,6 +953,16 @@ SubNetVal::~SubNetVal()
delete val.subnet_val;
}
const IPAddr& SubNetVal::Prefix() const
{
return val.subnet_val->Prefix();
}
int SubNetVal::Width() const
{
return val.subnet_val->Length();
}
unsigned int SubNetVal::MemoryAllocation() const
{
return padded_sizeof(*this) + val.subnet_val->MemoryAllocation();
@ -978,7 +988,7 @@ IPAddr SubNetVal::Mask() const
uint32 m[4];
for ( unsigned int i = 0; i < 4; ++i )
m[i] = 0;
IPAddr rval(IPAddr::IPv6, m, IPAddr::Host);
IPAddr rval(IPv6, m, IPAddr::Host);
return rval;
}
@ -994,7 +1004,7 @@ IPAddr SubNetVal::Mask() const
while ( ++mp < m + 4 )
*mp = 0;
IPAddr rval(IPAddr::IPv6, m, IPAddr::Host);
IPAddr rval(IPv6, m, IPAddr::Host);
return rval;
}

8
src/Val.h
View file

@ -513,10 +513,6 @@ protected:
#define UDP_PORT_MASK 0x20000
#define ICMP_PORT_MASK 0x30000
typedef enum {
TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP,
} TransportProto;
class PortVal : public Val {
public:
// Constructors - both take the port number in host order.
@ -588,8 +584,8 @@ public:
Val* SizeVal() const;
const IPAddr& Prefix() const { return val.subnet_val->Prefix(); }
int Width() const { return val.subnet_val->Length(); }
const IPAddr& Prefix() const;
int Width() const;
IPAddr Mask() const;
bool Contains(const IPAddr& addr) const;

24
src/bro.bif
View file

@ -1949,7 +1949,7 @@ function is_local_interface%(ip: addr%) : bool
if ( ent )
{
for ( unsigned int len = 0; ent->h_addr_list[len]; ++len )
addrs.push_back(IPAddr(IPAddr::IPv4, (uint32*)ent->h_addr_list[len],
addrs.push_back(IPAddr(IPv4, (uint32*)ent->h_addr_list[len],
IPAddr::Network));
}
@ -1958,7 +1958,7 @@ function is_local_interface%(ip: addr%) : bool
if ( ent )
{
for ( unsigned int len = 0; ent->h_addr_list[len]; ++len )
addrs.push_back(IPAddr(IPAddr::IPv6, (uint32*)ent->h_addr_list[len],
addrs.push_back(IPAddr(IPv6, (uint32*)ent->h_addr_list[len],
IPAddr::Network));
}
@ -2024,7 +2024,7 @@ function gethostname%(%) : string
## Returns: true if *a* is an IPv4 address, else false.
function is_v4_addr%(a: addr%): bool
%{
if ( a->AsAddr().GetFamily() == IPAddr::IPv4 )
if ( a->AsAddr().GetFamily() == IPv4 )
return new Val(1, TYPE_BOOL);
else
return new Val(0, TYPE_BOOL);
@ -2037,7 +2037,7 @@ function is_v4_addr%(a: addr%): bool
## Returns: true if *a* is an IPv6 address, else false.
function is_v6_addr%(a: addr%): bool
%{
if ( a->AsAddr().GetFamily() == IPAddr::IPv6 )
if ( a->AsAddr().GetFamily() == IPv6 )
return new Val(1, TYPE_BOOL);
else
return new Val(0, TYPE_BOOL);
@ -2070,7 +2070,7 @@ function routing0_data_to_addrs%(s: string%): addr_vec
while ( len > 0 )
{
IPAddr a(IPAddr::IPv6, (const uint32*) bytes, IPAddr::Network);
IPAddr a(IPv6, (const uint32*) bytes, IPAddr::Network);
rval->Assign(rval->Size(), new AddrVal(a), 0);
bytes += 16;
len -= 16;
@ -3552,7 +3552,7 @@ function lookup_location%(a: addr%) : geo_location
}
#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6
if ( geoip_v6 && a->AsAddr().GetFamily() == IPAddr::IPv6 )
if ( geoip_v6 && a->AsAddr().GetFamily() == IPv6 )
{
geoipv6_t ga;
a->AsAddr().CopyIPv6(&ga);
@ -3564,7 +3564,7 @@ function lookup_location%(a: addr%) : geo_location
else
#endif
if ( geoip && a->AsAddr().GetFamily() == IPAddr::IPv4 )
if ( geoip && a->AsAddr().GetFamily() == IPv4 )
{
const uint32* bytes;
a->AsAddr().GetBytes(&bytes);
@ -3647,7 +3647,7 @@ function lookup_asn%(a: addr%) : count
{
// IPv6 support showed up in 1.4.5.
#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6
if ( a->AsAddr().GetFamily() == IPAddr::IPv6 )
if ( a->AsAddr().GetFamily() == IPv6 )
{
geoipv6_t ga;
a->AsAddr().CopyIPv6(&ga);
@ -3656,7 +3656,7 @@ function lookup_asn%(a: addr%) : count
else
#endif
if ( a->AsAddr().GetFamily() == IPAddr::IPv4 )
if ( a->AsAddr().GetFamily() == IPv4 )
{
const uint32* bytes;
a->AsAddr().GetBytes(&bytes);
@ -5383,7 +5383,7 @@ function preserve_prefix%(a: addr, width: count%): any
AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50];
if ( ip_anon )
{
if ( a->AsAddr().GetFamily() == IPAddr::IPv6 )
if ( a->AsAddr().GetFamily() == IPv6 )
builtin_error("preserve_prefix() not supported for IPv6 addresses");
else
{
@ -5412,7 +5412,7 @@ function preserve_subnet%(a: subnet%): any
AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50];
if ( ip_anon )
{
if ( a->AsSubNet().Prefix().GetFamily() == IPAddr::IPv6 )
if ( a->AsSubNet().Prefix().GetFamily() == IPv6 )
builtin_error("preserve_subnet() not supported for IPv6 addresses");
else
{
@ -5448,7 +5448,7 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr
if ( anon_class < 0 || anon_class >= NUM_ADDR_ANONYMIZATION_CLASSES )
builtin_error("anonymize_addr(): invalid ip addr anonymization class");
if ( a->AsAddr().GetFamily() == IPAddr::IPv6 )
if ( a->AsAddr().GetFamily() == IPv6 )
{
builtin_error("anonymize_addr() not supported for IPv6 addresses");
return 0;

3
src/logging.bif
View file

@ -3,8 +3,9 @@
module Log;
%%{
#include "LogMgr.h"
#include "NetVar.h"
#include "logging/Manager.h"
%%}
type Filter: record;

630
src/LogMgr.cc → src/logging/Manager.cc
View file

File diff suppressed because it is too large Load diff

205
src/logging/Manager.h Normal file
View file

@ -0,0 +1,205 @@
// See the file "COPYING" in the main distribution directory for copyright.
//
// A class managing log writers and filters.
#ifndef LOGGING_MANAGER_H
#define LOGGING_MANAGER_H
#include "../Val.h"
#include "../EventHandler.h"
#include "../RemoteSerializer.h"
class SerializationFormat;
class RemoteSerializer;
class RotationTimer;
namespace logging {
class WriterBackend;
class WriterFrontend;
class RotationFinishedMessage;
/**
* Singleton class for managing log streams.
*/
class Manager {
public:
/**
* Constructor.
*/
Manager();
/**
* Destructor.
*/
~Manager();
/**
* Creates a new log stream.
*
* @param id The enum value corresponding the log stream.
*
* @param stream A record of script type \c Log::Stream.
*
* This method corresponds directly to the internal BiF defined in
* logging.bif, which just forwards here.
*/
bool CreateStream(EnumVal* id, RecordVal* stream);
/**
* Enables a log log stream.
*
* @param id The enum value corresponding the log stream.
*
* This method corresponds directly to the internal BiF defined in
* logging.bif, which just forwards here.
*/
bool EnableStream(EnumVal* id);
/**
* Disables a log stream.
*
* @param id The enum value corresponding the log stream.
*
* This methods corresponds directly to the internal BiF defined in
* logging.bif, which just forwards here.
*/
bool DisableStream(EnumVal* id);
/**
* Adds a filter to a log stream.
*
* @param id The enum value corresponding the log stream.
*
* @param filter A record of script type \c Log::Filter.
*
* This methods corresponds directly to the internal BiF defined in
* logging.bif, which just forwards here.
*/
bool AddFilter(EnumVal* id, RecordVal* filter);
/**
* Removes a filter from a log stream.
*
* @param id The enum value corresponding the log stream.
*
* @param name The name of the filter to remove.
*
* This methods corresponds directly to the internal BiF defined in
* logging.bif, which just forwards here.
*/
bool RemoveFilter(EnumVal* id, StringVal* name);
/**
* Removes a filter from a log stream.
*
* @param id The enum value corresponding the log stream.
*
* @param name The name of the filter to remove.
*
* This methods corresponds directly to the internal BiF defined in
* logging.bif, which just forwards here.
*/
bool RemoveFilter(EnumVal* id, string name);
/**
* Write a record to a log stream.
*
* @param id The enum value corresponding the log stream.
*
* @param colums A record of the type defined for the stream's
* columns.
*
* This methods corresponds directly to the internal BiF defined in
* logging.bif, which just forwards here.
*/
bool Write(EnumVal* id, RecordVal* columns);
/**
* Sets log streams buffering state. This adjusts all associated
* writers to the new state.
*
* @param id The enum value corresponding the log stream.
*
* @param enabled False to disable buffering (default is enabled).
*
* This methods corresponds directly to the internal BiF defined in
* logging.bif, which just forwards here.
*/
bool SetBuf(EnumVal* id, bool enabled);
/**
* Flushes a log stream. This flushed all associated writers.
*
* @param id The enum value corresponding the log stream.
*
* This methods corresponds directly to the internal BiF defined in
* logging.bif, which just forwards here.
*/
bool Flush(EnumVal* id);
/**
* Prepares the log manager to terminate. This will flush all log
* stream.
*/
void Terminate();
protected:
friend class WriterFrontend;
friend class RotationFinishedMessage;
friend class ::RemoteSerializer;
friend class ::RotationTimer;
// Instantiates a new WriterBackend of the given type (note that
// doing so creates a new thread!).
WriterBackend* CreateBackend(WriterFrontend* frontend, bro_int_t type);
//// Function also used by the RemoteSerializer.
// Takes ownership of fields.
WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path,
int num_fields, const threading::Field* const* fields,
bool local, bool remote);
// Takes ownership of values..
bool Write(EnumVal* id, EnumVal* writer, string path,
int num_fields, threading::Value** vals);
// Announces all instantiated writers to peer.
void SendAllWritersTo(RemoteSerializer::PeerID peer);
// Signals that a file has been rotated.
bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
double open, double close, bool terminating);
// Deletes the values as passed into Write().
void DeleteVals(int num_fields, threading::Value** vals);
private:
struct Filter;
struct Stream;
struct WriterInfo;
bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
TableVal* include, TableVal* exclude, string path, list<int> indices);
threading::Value** RecordToFilterVals(Stream* stream, Filter* filter,
RecordVal* columns);
threading::Value* ValToLogVal(Val* val, BroType* ty = 0);
Stream* FindStream(EnumVal* id);
void RemoveDisabledWriters(Stream* stream);
void InstallRotationTimer(WriterInfo* winfo);
void Rotate(WriterInfo* info);
Filter* FindFilter(EnumVal* id, StringVal* filter);
WriterInfo* FindWriter(WriterFrontend* writer);
vector<Stream *> streams; // Indexed by stream enum.
};
}
extern logging::Manager* log_mgr;
#endif

281
src/logging/WriterBackend.cc Normal file
View file

@ -0,0 +1,281 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "util.h"
#include "threading/SerialTypes.h"
#include "WriterBackend.h"
#include "WriterFrontend.h"
// Messages sent from backend to frontend (i.e., "OutputMessages").
using threading::Value;
using threading::Field;
namespace logging {
class RotationFinishedMessage : public threading::OutputMessage<WriterFrontend>
{
public:
RotationFinishedMessage(WriterFrontend* writer, string new_name, string old_name,
double open, double close, bool terminating)
: threading::OutputMessage<WriterFrontend>("RotationFinished", writer),
new_name(new_name), old_name(old_name), open(open),
close(close), terminating(terminating) { }
virtual bool Process()
{
return log_mgr->FinishedRotation(Object(), new_name, old_name, open, close, terminating);
}
private:
string new_name;
string old_name;
double open;
double close;
bool terminating;
};
class FlushWriteBufferMessage : public threading::OutputMessage<WriterFrontend>
{
public:
FlushWriteBufferMessage(WriterFrontend* writer)
: threading::OutputMessage<WriterFrontend>("FlushWriteBuffer", writer) {}
virtual bool Process() { Object()->FlushWriteBuffer(); return true; }
};
class DisableMessage : public threading::OutputMessage<WriterFrontend>
{
public:
DisableMessage(WriterFrontend* writer)
: threading::OutputMessage<WriterFrontend>("Disable", writer) {}
virtual bool Process() { Object()->SetDisable(); return true; }
};
}
// Backend methods.
using namespace logging;
WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread()
{
path = "<path not yet set>";
num_fields = 0;
fields = 0;
buffering = true;
frontend = arg_frontend;
SetName(frontend->Name());
}
WriterBackend::~WriterBackend()
{
if ( fields )
{
for(int i = 0; i < num_fields; ++i)
delete fields[i];
delete [] fields;
}
}
void WriterBackend::DeleteVals(int num_writes, Value*** vals)
{
for ( int j = 0; j < num_writes; ++j )
{
// Note this code is duplicated in Manager::DeleteVals().
for ( int i = 0; i < num_fields; i++ )
delete vals[j][i];
delete [] vals[j];
}
delete [] vals;
}
bool WriterBackend::FinishedRotation(string new_name, string old_name,
double open, double close, bool terminating)
{
SendOut(new RotationFinishedMessage(frontend, new_name, old_name, open, close, terminating));
return true;
}
void WriterBackend::DisableFrontend()
{
SendOut(new DisableMessage(frontend));
}
bool WriterBackend::Init(string arg_path, int arg_num_fields, const Field* const* arg_fields)
{
path = arg_path;
num_fields = arg_num_fields;
fields = arg_fields;
string name = Fmt("%s/%s", path.c_str(), frontend->Name().c_str());
SetName(name);
if ( ! DoInit(arg_path, arg_num_fields, arg_fields) )
{
DisableFrontend();
return false;
}
return true;
}
bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals)
{
// Double-check that the arguments match. If we get this from remote,
// something might be mixed up.
if ( num_fields != arg_num_fields )
{
#ifdef DEBUG
const char* msg = Fmt("Number of fields don't match in WriterBackend::Write() (%d vs. %d)",
arg_num_fields, num_fields);
Debug(DBG_LOGGING, msg);
#endif
DeleteVals(num_writes, vals);
DisableFrontend();
return false;
}
#ifdef DEBUG
// Double-check all the types match.
for ( int j = 0; j < num_writes; j++ )
{
for ( int i = 0; i < num_fields; ++i )
{
if ( vals[j][i]->type != fields[i]->type )
{
const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)",
vals[j][i]->type, fields[i]->type);
Debug(DBG_LOGGING, msg);
DisableFrontend();
DeleteVals(num_writes, vals);
return false;
}
}
}
#endif
bool success = true;
for ( int j = 0; j < num_writes; j++ )
{
success = DoWrite(num_fields, fields, vals[j]);
if ( ! success )
break;
}
DeleteVals(num_writes, vals);
if ( ! success )
DisableFrontend();
return success;
}
bool WriterBackend::SetBuf(bool enabled)
{
if ( enabled == buffering )
// No change.
return true;
buffering = enabled;
if ( ! DoSetBuf(enabled) )
{
DisableFrontend();
return false;
}
return true;
}
bool WriterBackend::Rotate(string rotated_path, double open,
double close, bool terminating)
{
if ( ! DoRotate(rotated_path, open, close, terminating) )
{
DisableFrontend();
return false;
}
return true;
}
bool WriterBackend::Flush()
{
if ( ! DoFlush() )
{
DisableFrontend();
return false;
}
return true;
}
bool WriterBackend::Finish()
{
if ( ! DoFlush() )
{
DisableFrontend();
return false;
}
return true;
}
bool WriterBackend::DoHeartbeat(double network_time, double current_time)
{
MsgThread::DoHeartbeat(network_time, current_time);
SendOut(new FlushWriteBufferMessage(frontend));
return true;
}
string WriterBackend::Render(const threading::Value::addr_t& addr) const
{
if ( addr.family == IPv4 )
{
char s[INET_ADDRSTRLEN];
if ( inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) == NULL )
return "<bad IPv4 address conversion>";
else
return s;
}
else
{
char s[INET6_ADDRSTRLEN];
if ( inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) == NULL )
return "<bad IPv6 address conversion>";
else
return s;
}
}
string WriterBackend::Render(const threading::Value::subnet_t& subnet) const
{
char l[16];
if ( subnet.prefix.family == IPv4 )
modp_uitoa10(subnet.length - 96, l);
else
modp_uitoa10(subnet.length, l);
string s = Render(subnet.prefix) + "/" + l;
return s;
}

319
src/logging/WriterBackend.h Normal file
View file

@ -0,0 +1,319 @@
// See the file "COPYING" in the main distribution directory for copyright.
//
// Bridge class between main process and writer threads.
#ifndef LOGGING_WRITERBACKEND_H
#define LOGGING_WRITERBACKEND_H
#include "Manager.h"
#include "threading/MsgThread.h"
namespace logging {
/**
* Base class for writer implementation. When the logging::Manager creates a
* new logging filter, it instantiates a WriterFrontend. That then in turn
* creates a WriterBackend of the right type. The frontend then forwards
* messages over the backend as its methods are called.
*
* All of this methods must be called only from the corresponding child
* thread (the constructor and destructor are the exceptions.)
*/
class WriterBackend : public threading::MsgThread
{
public:
/**
* Constructor.
*
* @param frontend The frontend writer that created this backend. The
* *only* purpose of this value is to be passed back via messages as
* a argument to callbacks. One must not otherwise access the
* frontend, it's running in a different thread.
*
* @param name A descriptive name for writer's type (e.g., \c Ascii).
*
*/
WriterBackend(WriterFrontend* frontend);
/**
* Destructor.
*/
virtual ~WriterBackend();
/**
* One-time initialization of the writer to define the logged fields.
*
* @param path A string left to the interpretation of the writer
* implementation; it corresponds to the value configured on the
* script-level for the logging filter.
*
* @param num_fields The number of log fields for the stream.
*
* @param fields An array of size \a num_fields with the log fields.
* The methods takes ownership of the array.
*
* @return False if an error occured.
*/
bool Init(string path, int num_fields, const threading::Field* const* fields);
/**
* Writes one log entry.
*
* @param num_fields: The number of log fields for this stream. The
* value must match what was passed to Init().
*
* @param An array of size \a num_fields with the log values. Their
* types musst match with the field passed to Init(). The method
* takes ownership of \a vals..
*
* Returns false if an error occured, in which case the writer must
* not be used any further.
*
* @return False if an error occured.
*/
bool Write(int num_fields, int num_writes, threading::Value*** vals);
/**
* Sets the buffering status for the writer, assuming the writer
* supports that. (If not, it will be ignored).
*
* @param enabled False if buffering is to be disabled (by default
* it's on).
*
* @return False if an error occured.
*/
bool SetBuf(bool enabled);
/**
* Flushes any currently buffered output, assuming the writer
* supports that. (If not, it will be ignored).
*
* @return False if an error occured.
*/
bool Flush();
/**
* Triggers rotation, if the writer supports that. (If not, it will
* be ignored).
*
* @return False if an error occured.
*/
bool Rotate(string rotated_path, double open, double close, bool terminating);
/**
* Finishes writing to this logger in a regularl fashion. Must not be
* called if an error has been indicated earlier. After calling this,
* no further writing must be performed.
*
* @return False if an error occured.
*/
bool Finish();
/**
* Disables the frontend that has instantiated this backend. Once
* disabled,the frontend will not send any further message over.
*/
void DisableFrontend();
/**
* Returns the log path as passed into the constructor.
*/
const string Path() const { return path; }
/**
* Returns the number of log fields as passed into the constructor.
*/
int NumFields() const { return num_fields; }
/**
* Returns the log fields as passed into the constructor.
*/
const threading::Field* const * Fields() const { return fields; }
/**
* Returns the current buffering state.
*
* @return True if buffering is enabled.
*/
bool IsBuf() { return buffering; }
/**
* Signals that a file has been rotated. This must be called by a
* writer's implementation of DoRotate() once rotation has finished.
*
* Most of the parameters should be passed through from DoRotate().
*
* @param new_name The filename of the rotated file.
*
* @param old_name The filename of the original file.
*
* @param open: The timestamp when the original file was opened.
*
* @param close: The timestamp when the origina file was closed.
*
* @param terminating: True if the original rotation request occured
* due to the main Bro process shutting down.
*/
bool FinishedRotation(string new_name, string old_name,
double open, double close, bool terminating);
/** Helper method to render an IP address as a string.
*
* @param addr The address.
*
* @return An ASCII representation of the address.
*/
string Render(const threading::Value::addr_t& addr) const;
/** Helper method to render an subnet value as a string.
*
* @param addr The address.
*
* @return An ASCII representation of the address.
*/
string Render(const threading::Value::subnet_t& subnet) const;
protected:
/**
* Writer-specific intialization method.
*
* A writer implementation must override this method. If it returns
* false, it will be assumed that a fatal error has occured that
* prevents the writer from further operation; it will then be
* disabled and eventually deleted. When returning false, an
* implementation should also call Error() to indicate what happened.
*/
virtual bool DoInit(string path, int num_fields,
const threading::Field* const* fields) = 0;
/**
* Writer-specific output method implementing recording of fone log
* entry.
*
* A writer implementation must override this method. If it returns
* false, it will be assumed that a fatal error has occured that
* prevents the writer from further operation; it will then be
* disabled and eventually deleted. When returning false, an
* implementation should also call Error() to indicate what happened.
*/
virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
threading::Value** vals) = 0;
/**
* Writer-specific method implementing a change of fthe buffering
* state. If buffering is disabled, the writer should attempt to
* write out information as quickly as possible even if doing so may
* have a performance impact. If enabled (which is the default), it
* may buffer data as helpful and write it out later in a way
* optimized for performance. The current buffering state can be
* queried via IsBuf().
*
* A writer implementation must override this method but it can just
* ignore calls if buffering doesn't align with its semantics.
*
* If the method returns false, it will be assumed that a fatal error
* has occured that prevents the writer from further operation; it
* will then be disabled and eventually deleted. When returning
* false, an implementation should also call Error() to indicate what
* happened.
*/
virtual bool DoSetBuf(bool enabled) = 0;
/**
* Writer-specific method implementing flushing of its output.
*
* A writer implementation must override this method but it can just
* ignore calls if flushing doesn't align with its semantics.
*
* If the method returns false, it will be assumed that a fatal error
* has occured that prevents the writer from further operation; it
* will then be disabled and eventually deleted. When returning
* false, an implementation should also call Error() to indicate what
* happened.
*/
virtual bool DoFlush() = 0;
/**
* Writer-specific method implementing log rotation. Most directly
* this only applies to writers writing into files, which should then
* close the current file and open a new one. However, a writer may
* also trigger other apppropiate actions if semantics are similar. *
* Once rotation has finished, the implementation must call
* FinishedRotation() to signal the log manager that potential
* postprocessors can now run.
*
* A writer implementation must override this method but it can just
* ignore calls if flushing doesn't align with its semantics. It
* still needs to call FinishedRotation() though.
*
* If the method returns false, it will be assumed that a fatal error
* has occured that prevents the writer from further operation; it
* will then be disabled and eventually deleted. When returning
* false, an implementation should also call Error() to indicate what
* happened.
*
* @param rotate_path Reflects the path to where the rotated output
* is to be moved, with specifics depending on the writer. It should
* generally be interpreted in a way consistent with that of \c path
* as passed into DoInit(). As an example, for file-based output, \c
* rotate_path could be the original filename extended with a
* timestamp indicating the time of the rotation.
*
* @param open The network time when the *current* file was opened.
*
* @param close The network time when the *current* file was closed.
*
* @param terminating Indicates whether the rotation request occurs
* due the main Bro prcoess terminating (and not because we've
* reached a regularly scheduled time for rotation).
*/
virtual bool DoRotate(string rotated_path, double open, double close,
bool terminating) = 0;
/**
* Writer-specific method implementing log output finalization at
* termination. Not called when any of the other methods has
* previously signaled an error, i.e., executing this method signals
* a regular shutdown of the writer.
*
* A writer implementation must override this method but it can just
* ignore calls if flushing doesn't align with its semantics.
*
* If the method returns false, it will be assumed that a fatal error
* has occured that prevents the writer from further operation; it
* will then be disabled and eventually deleted. When returning
* false, an implementation should also call Error() to indicate what
* happened.
*/
virtual bool DoFinish() = 0;
/**
* Triggered by regular heartbeat messages from the main thread.
*
* This method can be overridden but once must call
* WriterBackend::DoHeartbeat().
*/
virtual bool DoHeartbeat(double network_time, double current_time);
private:
/**
* Deletes the values as passed into Write().
*/
void DeleteVals(int num_writes, threading::Value*** vals);
// Frontend that instantiated us. This object must not be access from
// this class, it's running in a different thread!
WriterFrontend* frontend;
string path; // Log path.
int num_fields; // Number of log fields.
const threading::Field* const* fields; // Log fields.
bool buffering; // True if buffering is enabled.
};
}
#endif

279
src/logging/WriterFrontend.cc Normal file
View file

@ -0,0 +1,279 @@
#include "Net.h"
#include "threading/SerialTypes.h"
#include "WriterFrontend.h"
#include "WriterBackend.h"
using threading::Value;
using threading::Field;
namespace logging {
// Messages sent from frontend to backend (i.e., "InputMessages").
class InitMessage : public threading::InputMessage<WriterBackend>
{
public:
InitMessage(WriterBackend* backend, const string path, const int num_fields, const Field* const* fields)
: threading::InputMessage<WriterBackend>("Init", backend),
path(path), num_fields(num_fields), fields(fields) { }
virtual bool Process() { return Object()->Init(path, num_fields, fields); }
private:
const string path;
const int num_fields;
const Field * const* fields;
};
class RotateMessage : public threading::InputMessage<WriterBackend>
{
public:
RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const string rotated_path, const double open,
const double close, const bool terminating)
: threading::InputMessage<WriterBackend>("Rotate", backend),
frontend(frontend),
rotated_path(rotated_path), open(open),
close(close), terminating(terminating) { }
virtual bool Process() { return Object()->Rotate(rotated_path, open, close, terminating); }
private:
WriterFrontend* frontend;
const string rotated_path;
const double open;
const double close;
const bool terminating;
};
class WriteMessage : public threading::InputMessage<WriterBackend>
{
public:
WriteMessage(WriterBackend* backend, int num_fields, int num_writes, Value*** vals)
: threading::InputMessage<WriterBackend>("Write", backend),
num_fields(num_fields), num_writes(num_writes), vals(vals) {}
virtual bool Process() { return Object()->Write(num_fields, num_writes, vals); }
private:
int num_fields;
int num_writes;
Value ***vals;
};
class SetBufMessage : public threading::InputMessage<WriterBackend>
{
public:
SetBufMessage(WriterBackend* backend, const bool enabled)
: threading::InputMessage<WriterBackend>("SetBuf", backend),
enabled(enabled) { }
virtual bool Process() { return Object()->SetBuf(enabled); }
private:
const bool enabled;
};
class FlushMessage : public threading::InputMessage<WriterBackend>
{
public:
FlushMessage(WriterBackend* backend)
: threading::InputMessage<WriterBackend>("Flush", backend) {}
virtual bool Process() { return Object()->Flush(); }
};
class FinishMessage : public threading::InputMessage<WriterBackend>
{
public:
FinishMessage(WriterBackend* backend)
: threading::InputMessage<WriterBackend>("Finish", backend) {}
virtual bool Process() { return Object()->Finish(); }
};
}
// Frontend methods.
using namespace logging;
WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote)
{
stream = arg_stream;
writer = arg_writer;
Ref(stream);
Ref(writer);
disabled = initialized = false;
buf = true;
local = arg_local;
remote = arg_remote;
write_buffer = 0;
write_buffer_pos = 0;
ty_name = "<not set>";
if ( local )
{
backend = log_mgr->CreateBackend(this, writer->AsEnum());
assert(backend);
backend->Start();
}
else
backend = 0;
}
WriterFrontend::~WriterFrontend()
{
Unref(stream);
Unref(writer);
}
string WriterFrontend::Name() const
{
if ( path.size() )
return ty_name;
return ty_name + "/" + path;
}
void WriterFrontend::Stop()
{
FlushWriteBuffer();
SetDisable();
if ( backend )
backend->Stop();
}
void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* const * arg_fields)
{
if ( disabled )
return;
if ( initialized )
reporter->InternalError("writer initialize twice");
path = arg_path;
num_fields = arg_num_fields;
fields = arg_fields;
initialized = true;
if ( backend )
backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields));
if ( remote )
remote_serializer->SendLogCreateWriter(stream,
writer,
arg_path,
arg_num_fields,
arg_fields);
}
void WriterFrontend::Write(int num_fields, Value** vals)
{
if ( disabled )
return;
if ( remote )
remote_serializer->SendLogWrite(stream,
writer,
path,
num_fields,
vals);
if ( ! backend )
{
DeleteVals(vals);
return;
}
if ( ! write_buffer )
{
// Need new buffer.
write_buffer = new Value**[WRITER_BUFFER_SIZE];
write_buffer_pos = 0;
}
write_buffer[write_buffer_pos++] = vals;
if ( write_buffer_pos >= WRITER_BUFFER_SIZE || ! buf || terminating )
// Buffer full (or no bufferin desired or termiating).
FlushWriteBuffer();
}
void WriterFrontend::FlushWriteBuffer()
{
if ( ! write_buffer_pos )
// Nothing to do.
return;
if ( backend )
backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer));
// Clear buffer (no delete, we pass ownership to child thread.)
write_buffer = 0;
write_buffer_pos = 0;
}
void WriterFrontend::SetBuf(bool enabled)
{
if ( disabled )
return;
buf = enabled;
if ( backend )
backend->SendIn(new SetBufMessage(backend, enabled));
if ( ! buf )
// Make sure no longer buffer any still queued data.
FlushWriteBuffer();
}
void WriterFrontend::Flush()
{
if ( disabled )
return;
FlushWriteBuffer();
if ( backend )
backend->SendIn(new FlushMessage(backend));
}
void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating)
{
if ( disabled )
return;
FlushWriteBuffer();
if ( backend )
backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating));
}
void WriterFrontend::Finish()
{
if ( disabled )
return;
FlushWriteBuffer();
if ( backend )
backend->SendIn(new FinishMessage(backend));
}
void WriterFrontend::DeleteVals(Value** vals)
{
// Note this code is duplicated in Manager::DeleteVals().
for ( int i = 0; i < num_fields; i++ )
delete vals[i];
delete [] vals;
}

222
src/logging/WriterFrontend.h Normal file
View file

@ -0,0 +1,222 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef LOGGING_WRITERFRONTEND_H
#define LOGGING_WRITERFRONTEND_H
#include "Manager.h"
#include "threading/MsgThread.h"
namespace logging {
class WriterBackend;
/**
* Bridge class between the logging::Manager and backend writer threads. The
* Manager instantiates one \a WriterFrontend for each open logging filter.
* Each frontend in turns instantiates a WriterBackend-derived class
* internally that's specific to the particular output format. That backend
* runs in a new thread, and it receives messages from the frontend that
* correspond to method called by the manager.
*
*/
class WriterFrontend {
public:
/**
* Constructor.
*
* stream: The logging stream.
*
* writer: The backend writer type, with the value corresponding to the
* script-level \c Log::Writer enum (e.g., \a WRITER_ASCII). The
* frontend will internally instantiate a WriterBackend of the
* corresponding type.
*
* local: If true, the writer will instantiate a local backend.
*
* remote: If true, the writer will forward all data to remote
* clients.
*
* Frontends must only be instantiated by the main thread.
*/
WriterFrontend(EnumVal* stream, EnumVal* writer, bool local, bool remote);
/**
* Destructor.
*
* Frontends must only be destroyed by the main thread.
*/
virtual ~WriterFrontend();
/**
* Stops all output to this writer. Calling this methods disables all
* message forwarding to the backend and stops the backend thread.
*
* This method must only be called from the main thread.
*/
void Stop();
/**
* Initializes the writer.
*
* This method generates a message to the backend writer and triggers
* the corresponding message there. If the backend method fails, it
* sends a message back that will asynchronously call Disable().
*
* See WriterBackend::Init() for arguments. The method takes
* ownership of \a fields.
*
* This method must only be called from the main thread.
*/
void Init(string path, int num_fields, const threading::Field* const* fields);
/**
* Write out a record.
*
* This method generates a message to the backend writer and triggers
* the corresponding message there. If the backend method fails, it
* sends a message back that will asynchronously call Disable().
*
* As an optimization, if buffering is enabled (which is the default)
* this method may buffer several writes and send them over to the
* backend in bulk with a single message. An explicit bulk write of
* all currently buffered data can be triggered with
* FlushWriteBuffer(). The backend writer triggers this with a
* message at every heartbeat.
*
* See WriterBackend::Writer() for arguments (except that this method
* takes only a single record, not an array). The method takes
* ownership of \a vals.
*
* This method must only be called from the main thread.
*/
void Write(int num_fields, threading::Value** vals);
/**
* Sets the buffering state.
*
* This method generates a message to the backend writer and triggers
* the corresponding message there. If the backend method fails, it
* sends a message back that will asynchronously call Disable().
*
* See WriterBackend::SetBuf() for arguments.
*
* This method must only be called from the main thread.
*/
void SetBuf(bool enabled);
/**
* Flushes the output.
*
* This method generates a message to the backend writer and triggers
* the corresponding message there. In addition, it also triggers
* FlushWriteBuffer(). If the backend method fails, it sends a
* message back that will asynchronously call Disable().
*
* This method must only be called from the main thread.
*/
void Flush();
/**
* Triggers log rotation.
*
* This method generates a message to the backend writer and triggers
* the corresponding message there. If the backend method fails, it
* sends a message back that will asynchronously call Disable().
*
* See WriterBackend::Rotate() for arguments.
*
* This method must only be called from the main thread.
*/
void Rotate(string rotated_path, double open, double close, bool terminating);
/**
* Finalizes writing to this tream.
*
* This method generates a message to the backend writer and triggers
* the corresponding message there. If the backend method fails, it
* sends a message back that will asynchronously call Disable().
*
* This method must only be called from the main thread.
*/
void Finish();
/**
* Explicitly triggers a transfer of all potentially buffered Write()
* operations over to the backend.
*
* This method must only be called from the main thread.
*/
void FlushWriteBuffer();
/**
* Disables the writer frontend. From now on, all method calls that
* would normally send message over to the backend, turn into no-ops.
* Note though that it does not stop the backend itself, use Stop()
* to do thast as well (this method is primarily for use as callback
* when the backend wants to disable the frontend).
*
* Disabled frontend will eventually be discarded by the
* logging::Manager.
*
* This method must only be called from the main thread.
*/
void SetDisable() { disabled = true; }
/**
* Returns true if the writer frontend has been disabled with SetDisable().
*/
bool Disabled() { return disabled; }
/**
* Returns the log path as passed into the constructor.
*/
const string Path() const { return path; }
/**
* Returns the number of log fields as passed into the constructor.
*/
int NumFields() const { return num_fields; }
/**
* Returns a descriptive name for the writer, including the type of
* the backend and the path used.
*
* This method is safe to call from any thread.
*/
string Name() const;
/**
* Returns the log fields as passed into the constructor.
*/
const threading::Field* const * Fields() const { return fields; }
protected:
friend class Manager;
void DeleteVals(threading::Value** vals);
EnumVal* stream;
EnumVal* writer;
WriterBackend* backend; // The backend we have instanatiated.
bool disabled; // True if disabled.
bool initialized; // True if initialized.
bool buf; // True if buffering is enabled (default).
bool local; // True if logging locally.
bool remote; // True if loggin remotely.
string ty_name; // Name of the backend type. Set by the manager.
string path; // The log path.
int num_fields; // The number of log fields.
const threading::Field* const* fields; // The log fields.
// Buffer for bulk writes.
static const int WRITER_BUFFER_SIZE = 1000;
int write_buffer_pos; // Position of next write in buffer.
threading::Value*** write_buffer; // Buffer of size WRITER_BUFFER_SIZE.
};
}
#endif

48
src/LogWriterAscii.cc → src/logging/writers/Ascii.cc
View file

@ -3,10 +3,17 @@
#include <string>
#include <errno.h>
#include "LogWriterAscii.h"
#include "NetVar.h"
#include "threading/SerialTypes.h"
LogWriterAscii::LogWriterAscii()
#include "Ascii.h"
using namespace logging;
using namespace writer;
using threading::Value;
using threading::Field;
Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
{
file = 0;
@ -42,7 +49,7 @@ LogWriterAscii::LogWriterAscii()
desc.AddEscapeSequence(separator, separator_len);
}
LogWriterAscii::~LogWriterAscii()
Ascii::~Ascii()
{
if ( file )
fclose(file);
@ -54,7 +61,7 @@ LogWriterAscii::~LogWriterAscii()
delete [] header_prefix;
}
bool LogWriterAscii::WriteHeaderField(const string& key, const string& val)
bool Ascii::WriteHeaderField(const string& key, const string& val)
{
string str = string(header_prefix, header_prefix_len) +
key + string(separator, separator_len) + val + "\n";
@ -62,8 +69,8 @@ bool LogWriterAscii::WriteHeaderField(const string& key, const string& val)
return (fwrite(str.c_str(), str.length(), 1, file) == 1);
}
bool LogWriterAscii::DoInit(string path, int num_fields,
const LogField* const * fields)
bool Ascii::DoInit(string path, int num_fields,
const Field* const * fields)
{
if ( output_to_stdout )
path = "/dev/stdout";
@ -108,7 +115,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields,
types += string(separator, separator_len);
}
const LogField* field = fields[i];
const Field* field = fields[i];
names += field->name;
types += type_name(field->type);
if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) )
@ -131,17 +138,18 @@ write_error:
return false;
}
bool LogWriterAscii::DoFlush()
bool Ascii::DoFlush()
{
fflush(file);
return true;
}
void LogWriterAscii::DoFinish()
bool Ascii::DoFinish()
{
return true;
}
bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
{
if ( ! val->present )
{
@ -161,16 +169,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
case TYPE_COUNT:
case TYPE_COUNTER:
case TYPE_PORT:
desc->Add(val->val.uint_val);
break;
case TYPE_PORT:
desc->Add(val->val.port_val.port);
break;
case TYPE_SUBNET:
desc->Add(*val->val.subnet_val);
desc->Add(Render(val->val.subnet_val));
break;
case TYPE_ADDR:
desc->Add(*val->val.addr_val);
desc->Add(Render(val->val.addr_val));
break;
case TYPE_TIME:
@ -279,8 +290,8 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
return true;
}
bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields,
LogVal** vals)
bool Ascii::DoWrite(int num_fields, const Field* const * fields,
Value** vals)
{
if ( ! file )
DoInit(Path(), NumFields(), Fields());
@ -310,8 +321,7 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields,
return true;
}
bool LogWriterAscii::DoRotate(string rotated_path, double open,
double close, bool terminating)
bool Ascii::DoRotate(string rotated_path, double open, double close, bool terminating)
{
// Don't rotate special files or if there's not one currently open.
if ( ! file || IsSpecial(Path()) )
@ -332,13 +342,13 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open,
return true;
}
bool LogWriterAscii::DoSetBuf(bool enabled)
bool Ascii::DoSetBuf(bool enabled)
{
// Nothing to do.
return true;
}
string LogWriterAscii::LogExt()
string Ascii::LogExt()
{
const char* ext = getenv("BRO_LOG_SUFFIX");
if ( ! ext ) ext = "log";

35
src/LogWriterAscii.h → src/logging/writers/Ascii.h
View file

@ -2,33 +2,36 @@
//
// Log writer for delimiter-separated ASCII logs.
#ifndef LOGWRITERASCII_H
#define LOGWRITERASCII_H
#ifndef LOGGING_WRITER_ASCII_H
#define LOGGING_WRITER_ASCII_H
#include "LogWriter.h"
#include "../WriterBackend.h"
class LogWriterAscii : public LogWriter {
namespace logging { namespace writer {
class Ascii : public WriterBackend {
public:
LogWriterAscii();
~LogWriterAscii();
Ascii(WriterFrontend* frontend);
~Ascii();
static LogWriter* Instantiate() { return new LogWriterAscii; }
static WriterBackend* Instantiate(WriterFrontend* frontend)
{ return new Ascii(frontend); }
static string LogExt();
protected:
virtual bool DoInit(string path, int num_fields,
const LogField* const * fields);
virtual bool DoWrite(int num_fields, const LogField* const * fields,
LogVal** vals);
const threading::Field* const* fields);
virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
threading::Value** vals);
virtual bool DoSetBuf(bool enabled);
virtual bool DoRotate(string rotated_path, double open, double close,
bool terminating);
virtual bool DoRotate(string rotated_path, double open,
double close, bool terminating);
virtual bool DoFlush();
virtual void DoFinish();
virtual bool DoFinish();
private:
bool IsSpecial(string path) { return path.find("/dev/") == 0; }
bool DoWriteOne(ODesc* desc, LogVal* val, const LogField* field);
bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field);
bool WriteHeaderField(const string& key, const string& value);
FILE* file;
@ -55,4 +58,8 @@ private:
int header_prefix_len;
};
}
}
#endif

8
src/LogWriterNone.cc → src/logging/writers/None.cc
View file

@ -1,8 +1,10 @@
#include "LogWriterNone.h"
#include "None.h"
bool LogWriterNone::DoRotate(string rotated_path, double open,
double close, bool terminating)
using namespace logging;
using namespace writer;
bool None::DoRotate(string rotated_path, double open, double close, bool terminating)
{
if ( ! FinishedRotation(string("/dev/null"), Path(), open, close, terminating))
{

36
src/logging/writers/None.h Normal file
View file

@ -0,0 +1,36 @@
// See the file "COPYING" in the main distribution directory for copyright.
//
// Dummy log writer that just discards everything (but still pretends to rotate).
#ifndef LOGGING_WRITER_NONE_H
#define LOGGING_WRITER_NONE_H
#include "../WriterBackend.h"
namespace logging { namespace writer {
class None : public WriterBackend {
public:
None(WriterFrontend* frontend) : WriterBackend(frontend) {}
~None() {};
static WriterBackend* Instantiate(WriterFrontend* frontend)
{ return new None(frontend); }
protected:
virtual bool DoInit(string path, int num_fields,
const threading::Field* const * fields) { return true; }
virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
threading::Value** vals) { return true; }
virtual bool DoSetBuf(bool enabled) { return true; }
virtual bool DoRotate(string rotated_path, double open,
double close, bool terminating);
virtual bool DoFlush() { return true; }
virtual bool DoFinish() { return true; }
};
}
}
#endif

46
src/main.cc
View file

@ -29,7 +29,6 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
#include "Event.h"
#include "File.h"
#include "Reporter.h"
#include "LogMgr.h"
#include "Net.h"
#include "NetVar.h"
#include "Var.h"
@ -47,7 +46,10 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
#include "DPM.h"
#include "BroDoc.h"
#include "Brofiler.h"
#include "LogWriterAscii.h"
#include "threading/Manager.h"
#include "logging/Manager.h"
#include "logging/writers/Ascii.h"
#include "binpac_bro.h"
@ -63,7 +65,7 @@ extern "C" {
#include "setsignal.h"
};
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
HeapLeakChecker* heap_checker = 0;
int perftools_leaks = 0;
int perftools_profile = 0;
@ -74,7 +76,8 @@ char* writefile = 0;
name_list prefixes;
DNS_Mgr* dns_mgr;
TimerMgr* timer_mgr;
LogMgr* log_mgr;
logging::Manager* log_mgr = 0;
threading::Manager* thread_mgr = 0;
Stmt* stmts;
EventHandlerPtr net_done = 0;
RuleMatcher* rule_matcher = 0;
@ -174,7 +177,7 @@ void usage()
fprintf(stderr, " -W|--watchdog | activate watchdog timer\n");
fprintf(stderr, " -Z|--doc-scripts | generate documentation for all loaded scripts\n");
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
fprintf(stderr, " -m|--mem-leaks | show leaks [perftools]\n");
fprintf(stderr, " -M|--mem-profile | record heap [perftools]\n");
#endif
@ -195,7 +198,7 @@ void usage()
fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes());
fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake());
fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n");
fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", LogWriterAscii::LogExt().c_str());
fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str());
fprintf(stderr, " $BRO_PROFILER_FILE | Output file for script execution statistics (not set)\n");
exit(1);
@ -241,7 +244,7 @@ void done_with_network()
net_finish(1);
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
if ( perftools_profile )
{
@ -285,6 +288,9 @@ void terminate_bro()
if ( remote_serializer )
remote_serializer->LogStats();
log_mgr->Terminate();
thread_mgr->Terminate();
delete timer_mgr;
delete dns_mgr;
delete persistence_serializer;
@ -296,6 +302,7 @@ void terminate_bro()
delete remote_serializer;
delete dpm;
delete log_mgr;
delete thread_mgr;
delete reporter;
}
@ -324,6 +331,13 @@ RETSIGTYPE sig_handler(int signo)
{
set_processing_status("TERMINATING", "sig_handler");
signal_val = signo;
if ( thread_mgr->Terminating() && (signal_val == SIGTERM || signal_val == SIGINT) )
// If the thread manager is already terminating (i.e.,
// waiting for child threads to exit), another term signal
// will send the threads a kill.
thread_mgr->KillThreads();
return RETSIGVAL;
}
@ -410,7 +424,7 @@ int main(int argc, char** argv)
#ifdef USE_IDMEF
{"idmef-dtd", required_argument, 0, 'n'},
#endif
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
{"mem-leaks", no_argument, 0, 'm'},
{"mem-profile", no_argument, 0, 'M'},
#endif
@ -452,7 +466,7 @@ int main(int argc, char** argv)
safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLOPSWbdghvZ",
sizeof(opts));
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
strncat(opts, "mM", 2);
#endif
@ -608,7 +622,7 @@ int main(int argc, char** argv)
exit(0);
break;
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
case 'm':
perftools_leaks = 1;
break;
@ -658,7 +672,9 @@ int main(int argc, char** argv)
set_processing_status("INITIALIZING", "main");
bro_start_time = current_time(true);
reporter = new Reporter();
thread_mgr = new threading::Manager();
#ifdef DEBUG
if ( debug_streams )
@ -724,7 +740,7 @@ int main(int argc, char** argv)
persistence_serializer = new PersistenceSerializer();
remote_serializer = new RemoteSerializer();
event_registry = new EventRegistry();
log_mgr = new LogMgr();
log_mgr = new logging::Manager();
if ( events_file )
event_player = new EventPlayer(events_file);
@ -742,14 +758,14 @@ int main(int argc, char** argv)
// nevertheless reported; see perftools docs), thus
// we suppress some messages here.
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
{
HeapLeakChecker::Disabler disabler;
#endif
yyparse();
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
}
#endif
@ -996,12 +1012,14 @@ int main(int argc, char** argv)
have_pending_timers = ! reading_traces && timer_mgr->Size() > 0;
io_sources.Register(thread_mgr, true);
if ( io_sources.Size() > 0 || have_pending_timers )
{
if ( profiling_logger )
profiling_logger->Log();
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
if ( perftools_leaks )
heap_checker = new HeapLeakChecker("net_run");

5
src/net_util.cc
View file

@ -135,12 +135,11 @@ const char* fmt_conn_id(const IPAddr& src_addr, uint32 src_port,
const char* fmt_conn_id(const uint32* src_addr, uint32 src_port,
const uint32* dst_addr, uint32 dst_port)
{
IPAddr src(IPAddr::IPv6, src_addr, IPAddr::Network);
IPAddr dst(IPAddr::IPv6, dst_addr, IPAddr::Network);
IPAddr src(IPv6, src_addr, IPAddr::Network);
IPAddr dst(IPv6, dst_addr, IPAddr::Network);
return fmt_conn_id(src, src_port, dst, dst_port);
}
uint32 extract_uint32(const u_char* data)
{
uint32 val;

10
src/net_util.h
View file

@ -5,6 +5,13 @@
#include "config.h"
// Define first.
typedef enum {
TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP,
} TransportProto;
typedef enum { IPv4, IPv6 } IPFamily;
#include <assert.h>
#include <sys/types.h>
@ -21,7 +28,6 @@
#include <netinet/ip_icmp.h>
#include "util.h"
#include "IPAddr.h"
#ifdef HAVE_NETINET_IP6_H
#include <netinet/ip6.h>
@ -58,6 +64,8 @@ inline int seq_delta(uint32 a, uint32 b)
return int(a-b);
}
class IPAddr;
// Returns the ones-complement checksum of a chunk of b short-aligned bytes.
extern int ones_complement_checksum(const void* p, int b, uint32 sum);
extern int ones_complement_checksum(const IPAddr& a, uint32 sum);

174
src/threading/BasicThread.cc Normal file
View file

@ -0,0 +1,174 @@
#include <sys/signal.h>
#include <signal.h>
#include "config.h"
#include "BasicThread.h"
#include "Manager.h"
#ifdef HAVE_LINUX
#include <sys/prctl.h>
#endif
using namespace threading;
uint64_t BasicThread::thread_counter = 0;
BasicThread::BasicThread()
{
started = false;
terminating = false;
pthread = 0;
buf_len = 2048;
buf = (char*) malloc(buf_len);
name = Fmt("thread-%d", ++thread_counter);
thread_mgr->AddThread(this);
}
BasicThread::~BasicThread()
{
if ( buf )
free(buf);
}
void BasicThread::SetName(const string& arg_name)
{
// Slight race condition here with reader threads, but shouldn't matter.
name = arg_name;
}
void BasicThread::SetOSName(const string& name)
{
#ifdef HAVE_LINUX
prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
#endif
#ifdef __APPLE__
pthread_setname_np(name.c_str());
#endif
#ifdef FREEBSD
pthread_set_name_np(pthread_self(), name, name.c_str());
#endif
}
const char* BasicThread::Fmt(const char* format, ...)
{
va_list al;
va_start(al, format);
int n = safe_vsnprintf(buf, buf_len, format, al);
va_end(al);
if ( (unsigned int) n >= buf_len )
{ // Not enough room, grow the buffer.
int tmp_len = n + 32;
char* tmp = (char*) malloc(tmp_len);
// Is it portable to restart?
va_start(al, format);
n = safe_vsnprintf(tmp, tmp_len, format, al);
va_end(al);
free(tmp);
}
return buf;
}
void BasicThread::Start()
{
if ( started )
return;
if ( pthread_mutex_init(&terminate, 0) != 0 )
reporter->FatalError("Cannot create terminate mutex for thread %s", name.c_str());
// We use this like a binary semaphore and acquire it immediately.
if ( pthread_mutex_lock(&terminate) != 0 )
reporter->FatalError("Cannot aquire terminate mutex for thread %s", name.c_str());
if ( pthread_create(&pthread, 0, BasicThread::launcher, this) != 0 )
reporter->FatalError("Cannot create thread %s", name.c_str());
DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str());
started = true;
OnStart();
}
void BasicThread::Stop()
{
if ( ! started )
return;
if ( terminating )
return;
DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str());
// Signal that it's ok for the thread to exit now by unlocking the
// mutex.
if ( pthread_mutex_unlock(&terminate) != 0 )
reporter->FatalError("Failure flagging terminate condition for thread %s", name.c_str());
terminating = true;
OnStop();
}
void BasicThread::Join()
{
if ( ! started )
return;
if ( ! terminating )
Stop();
DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str());
if ( pthread_join(pthread, 0) != 0 )
reporter->FatalError("Failure joining thread %s", name.c_str());
pthread_mutex_destroy(&terminate);
DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str());
pthread = 0;
}
void BasicThread::Kill()
{
if ( ! (started && pthread) )
return;
// I believe this is safe to call from a signal handler ... Not error
// checking so that killing doesn't bail out if we have already
// terminated.
pthread_kill(pthread, SIGKILL);
}
void* BasicThread::launcher(void *arg)
{
BasicThread* thread = (BasicThread *)arg;
// Block signals in thread. We handle signals only in the main
// process.
sigset_t mask_set;
sigfillset(&mask_set);
int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0);
assert(res == 0); //
// Run thread's main function.
thread->Run();
// Wait until somebody actually wants us to terminate.
if ( pthread_mutex_lock(&thread->terminate) != 0 )
reporter->FatalError("Failure acquiring terminate mutex at end of thread %s", thread->Name().c_str());
return 0;
}

170
src/threading/BasicThread.h Normal file
View file

@ -0,0 +1,170 @@
#ifndef THREADING_BASICTHREAD_H
#define THREADING_BASICTHREAD_H
#include <pthread.h>
#include <semaphore.h>
#include "Queue.h"
#include "util.h"
using namespace std;
namespace threading {
class Manager;
/**
* Base class for all threads.
*
* This class encapsulates all the OS-level thread handling. All thread
* instances are automatically added to the threading::Manager for management. The
* manager also takes care of deleting them (which must not be done
* manually).
*/
class BasicThread
{
public:
/**
* Creates a new thread object. Instantiating the object does however
* not yet start the actual OS thread, that requires calling Start().
*
* Only Bro's main thread may create new thread instances.
*
* @param name A descriptive name for thread the thread. This may
* show up in messages to the user.
*/
BasicThread();
/**
* Returns a descriptive name for the thread. If not set via
* SetName(). If not set, a default name is choosen automatically.
*
* This method is safe to call from any thread.
*/
const string& Name() const { return name; }
/**
* Sets a descriptive name for the thread. This should be a string
* that's useful in output presented to the user and uniquely
* identifies the thread.
*
* This method must be called only from the thread itself.
*/
void SetName(const string& name);
/**
* Set the name shown by the OS as the thread's description. Not
* supported on all OSs.
*/
void SetOSName(const string& name);
/**
* Starts the thread. Calling this methods will spawn a new OS thread
* executing Run(). Note that one can't restart a thread after a
* Stop(), doing so will be ignored.
*
* Only Bro's main thread must call this method.
*/
void Start();
/**
* Signals the thread to stop. The method lets Terminating() now
* return true. It does however not force the thread to terminate.
* It's up to the Run() method to to query Terminating() and exit
* eventually.
*
* Calling this method has no effect if Start() hasn't been executed
* yet.
*
* Only Bro's main thread must call this method.
*/
void Stop();
/**
* Returns true if Stop() has been called.
*
* This method is safe to call from any thread.
*/
bool Terminating() const { return terminating; }
/**
* A version of fmt() that the thread can safely use.
*
* This is safe to call from Run() but must not be used from any
* other thread than the current one.
*/
const char* Fmt(const char* format, ...);
protected:
friend class Manager;
/**
* Entry point for the thread. This must be overridden by derived
* classes and will execute in a separate thread once Start() is
* called. The thread will not terminate before this method finishes.
* An implementation should regularly check Terminating() to see if
* exiting has been requested.
*/
virtual void Run() = 0;
/**
* Executed with Start(). This is a hook into starting the thread. It
* will be called from Bro's main thread after the OS thread has been
* started.
*/
virtual void OnStart() {}
/**
* Executed with Stop(). This is a hook into stopping the thread. It
* will be called from Bro's main thread after the thread has been
* signaled to stop.
*/
virtual void OnStop() {}
/**
* Destructor. This will be called by the manager.
*
* Only Bro's main thread may delete thread instances.
*
*/
virtual ~BasicThread();
/**
* Waits until the thread's Run() method has finished and then joins
* it. This is called from the threading::Manager.
*/
void Join();
/**
* Kills the thread immediately. One still needs to call Join()
* afterwards.
*
* This is called from the threading::Manager and safe to execute
* during a signal handler.
*/
void Kill();
private:
// pthread entry function.
static void* launcher(void *arg);
string name;
pthread_t pthread;
bool started; // Set to to true once running.
bool terminating; // Set to to true to signal termination.
// Used as a semaphore to tell the pthread thread when it may
// terminate.
pthread_mutex_t terminate;
// For implementing Fmt().
char* buf;
unsigned int buf_len;
static uint64_t thread_counter;
};
}
#endif

147
src/threading/Manager.cc Normal file
View file

@ -0,0 +1,147 @@
#include "Manager.h"
using namespace threading;
Manager::Manager()
{
DBG_LOG(DBG_THREADING, "Creating thread manager ...");
did_process = true;
next_beat = 0;
terminating = false;
idle = true;
}
Manager::~Manager()
{
if ( all_threads.size() )
Terminate();
}
void Manager::Terminate()
{
DBG_LOG(DBG_THREADING, "Terminating thread manager ...");
terminating = true;
// First process remaining thread output for the message threads.
do Process(); while ( did_process );
// Signal all to stop.
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->Stop();
// Then join them all.
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
{
(*i)->Join();
delete *i;
}
all_threads.clear();
msg_threads.clear();
idle = true;
closed = true;
terminating = false;
}
void Manager::KillThreads()
{
DBG_LOG(DBG_THREADING, "Killing threads ...");
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->Kill();
}
void Manager::AddThread(BasicThread* thread)
{
DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str());
all_threads.push_back(thread);
idle = false;
}
void Manager::AddMsgThread(MsgThread* thread)
{
DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name().c_str());
msg_threads.push_back(thread);
}
void Manager::GetFds(int* read, int* write, int* except)
{
}
double Manager::NextTimestamp(double* network_time)
{
// fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat);
if ( ::network_time && (did_process || ::network_time > next_beat || ! next_beat) )
// If we had something to process last time (or out heartbeat
// is due or not set yet), we want to check for more asap.
return timer_mgr->Time();
return -1.0;
}
void Manager::Process()
{
bool do_beat = false;
if ( network_time && (network_time > next_beat || ! next_beat) )
{
do_beat = true;
next_beat = ::network_time + HEART_BEAT_INTERVAL;
}
did_process = false;
for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ )
{
MsgThread* t = *i;
if ( do_beat )
t->Heartbeat();
while ( t->HasOut() )
{
Message* msg = t->RetrieveOut();
if ( msg->Process() )
{
if ( network_time )
did_process = true;
}
else
{
string s = msg->Name() + " failed, terminating thread";
reporter->Error("%s", s.c_str());
t->Stop();
}
delete msg;
}
}
// fprintf(stderr, "P %.6f %.6f do_beat=%d did_process=%d next_next=%.6f\n", network_time, timer_mgr->Time(), do_beat, (int)did_process, next_beat);
}
const threading::Manager::msg_stats_list& threading::Manager::GetMsgThreadStats()
{
stats.clear();
for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ )
{
MsgThread* t = *i;
MsgThread::Stats s;
t->GetStats(&s);
stats.push_back(std::make_pair(t->Name(),s));
}
return stats;
}

146
src/threading/Manager.h Normal file
View file

@ -0,0 +1,146 @@
#ifndef THREADING_MANAGER_H
#define THREADING_MANAGER_H
#include <list>
#include "IOSource.h"
#include "BasicThread.h"
#include "MsgThread.h"
namespace threading {
/**
* The thread manager coordinates all child threads. Once a BasicThread is
* instantitated, it gets addedd to the manager, which will delete it later
* once it has terminated.
*
* In addition to basic threads, the manager also provides additional
* functionality specific to MsgThread instances. In particular, it polls
* their outgoing message queue on a regular basis and feeds data sent into
* the rest of Bro. It also triggers the regular heartbeats.
*/
class Manager : public IOSource
{
public:
/**
* Constructor. Only a single instance of the manager must be
* created.
*/
Manager();
/**
* Destructir.
*/
~Manager();
/**
* Terminates the manager's processor. The method signals all threads
* to terminates and wait for them to do so. It then joins them and
* returns to the caller. Afterwards, no more thread instances may be
* created.
*/
void Terminate();
/**
* Returns True if we are currently in Terminate() waiting for
* threads to exit.
*/
bool Terminating() const { return terminating; }
/**
* Immediately kills all child threads. It does however not yet join
* them, one still needs to call Terminate() for that.
*
* This method is safe to call from a signal handler, and can in fact
* be called while Terminate() is already in progress.
*/
void KillThreads();
typedef std::list<std::pair<string, MsgThread::Stats> > msg_stats_list;
/**
* Returns statistics from all current MsgThread instances.
*
* @return A list of statistics, with one entry for each MsgThread.
* Each entry is a tuple of thread name and statistics. The list
* reference remains valid until the next call to this method (or
* termination of the manager).
*/
const msg_stats_list& GetMsgThreadStats();
/**
* Returns the number of currently active threads. This counts all
* threads that are not yet joined, includingt any potentially in
* Terminating() state.
*/
int NumThreads() const { return all_threads.size(); }
protected:
friend class BasicThread;
friend class MsgThread;
/**
* Registers a new basic thread with the manager. This is
* automatically called by the thread's constructor.
*
* @param thread The thread.
*/
void AddThread(BasicThread* thread);
/**
* Registers a new message thread with the manager. This is
* automatically called by the thread's constructor. This must be
* called \a in \a addition to AddThread(BasicThread* thread). The
* MsgThread constructor makes sure to do so.
*
* @param thread The thread.
*/
void AddMsgThread(MsgThread* thread);
/**
* Part of the IOSource interface.
*/
virtual void GetFds(int* read, int* write, int* except);
/**
* Part of the IOSource interface.
*/
virtual double NextTimestamp(double* network_time);
/**
* Part of the IOSource interface.
*/
virtual void Process();
/**
* Part of the IOSource interface.
*/
virtual const char* Tag() { return "threading::Manager"; }
private:
static const int HEART_BEAT_INTERVAL = 1;
typedef std::list<BasicThread*> all_thread_list;
all_thread_list all_threads;
typedef std::list<MsgThread*> msg_thread_list;
msg_thread_list msg_threads;
bool did_process; // True if the last Process() found some work to do.
double next_beat; // Timestamp when the next heartbeat will be sent.
bool terminating; // True if we are in Terminate().
msg_stats_list stats;
};
}
/**
* A singleton instance of the thread manager. All methods must only be
* called from Bro's main thread.
*/
extern threading::Manager* thread_mgr;
#endif

289
src/threading/MsgThread.cc Normal file
View file

@ -0,0 +1,289 @@
#include "DebugLogger.h"
#include "MsgThread.h"
#include "Manager.h"
using namespace threading;
namespace threading {
////// Messages.
// Signals child thread to terminate. This is actually a no-op; its only
// purpose is unblock the current read operation so that the child's Run()
// methods can check the termination status.
class TerminateMessage : public InputMessage<MsgThread>
{
public:
TerminateMessage(MsgThread* thread) : InputMessage<MsgThread>("Terminate", thread) { }
virtual bool Process() { return true; }
};
/// Sends a heartbeat to the child thread.
class HeartbeatMessage : public InputMessage<MsgThread>
{
public:
HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time)
: InputMessage<MsgThread>("Heartbeat", thread)
{ network_time = arg_network_time; current_time = arg_current_time; }
virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); }
private:
double network_time;
double current_time;
};
// A message from the child to be passed on to the Reporter.
class ReporterMessage : public OutputMessage<MsgThread>
{
public:
enum Type {
INFO, WARNING, ERROR, FATAL_ERROR, FATAL_ERROR_WITH_CORE,
INTERNAL_WARNING, INTERNAL_ERROR
};
ReporterMessage(Type arg_type, MsgThread* thread, const string& arg_msg)
: OutputMessage<MsgThread>("ReporterMessage", thread)
{ type = arg_type; msg = arg_msg; }
virtual bool Process();
private:
string msg;
Type type;
};
#ifdef DEBUG
// A debug message from the child to be passed on to the DebugLogger.
class DebugMessage : public OutputMessage<MsgThread>
{
public:
DebugMessage(DebugStream arg_stream, MsgThread* thread, const string& arg_msg)
: OutputMessage<MsgThread>("DebugMessage", thread)
{ stream = arg_stream; msg = arg_msg; }
virtual bool Process()
{
string s = Object()->Name() + ": " + msg;
debug_logger.Log(stream, "%s", s.c_str());
return true;
}
private:
string msg;
DebugStream stream;
};
#endif
}
////// Methods.
Message::~Message()
{
}
bool ReporterMessage::Process()
{
string s = Object()->Name() + ": " + msg;
const char* cmsg = s.c_str();
switch ( type ) {
case INFO:
reporter->Info("%s", cmsg);
break;
case WARNING:
reporter->Warning("%s", cmsg);
break;
case ERROR:
reporter->Error("%s", cmsg);
break;
case FATAL_ERROR:
reporter->FatalError("%s", cmsg);
break;
case FATAL_ERROR_WITH_CORE:
reporter->FatalErrorWithCore("%s", cmsg);
break;
case INTERNAL_WARNING:
reporter->InternalWarning("%s", cmsg);
break;
case INTERNAL_ERROR :
reporter->InternalError("%s", cmsg);
break;
default:
reporter->InternalError("unknown ReporterMessage type %d", type);
}
return true;
}
MsgThread::MsgThread() : BasicThread()
{
cnt_sent_in = cnt_sent_out = 0;
thread_mgr->AddMsgThread(this);
}
void MsgThread::OnStop()
{
// This is to unblock the current queue read operation.
SendIn(new TerminateMessage(this), true);
}
void MsgThread::Heartbeat()
{
SendIn(new HeartbeatMessage(this, network_time, current_time()));
}
bool MsgThread::DoHeartbeat(double network_time, double current_time)
{
string n = Name();
n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", n.c_str(),
cnt_sent_in - queue_in.Size(),
cnt_sent_out - queue_out.Size());
SetOSName(n.c_str());
return true;
}
void MsgThread::Info(const char* msg)
{
SendOut(new ReporterMessage(ReporterMessage::INFO, this, msg));
}
void MsgThread::Warning(const char* msg)
{
SendOut(new ReporterMessage(ReporterMessage::WARNING, this, msg));
}
void MsgThread::Error(const char* msg)
{
SendOut(new ReporterMessage(ReporterMessage::ERROR, this, msg));
}
void MsgThread::FatalError(const char* msg)
{
SendOut(new ReporterMessage(ReporterMessage::FATAL_ERROR, this, msg));
}
void MsgThread::FatalErrorWithCore(const char* msg)
{
SendOut(new ReporterMessage(ReporterMessage::FATAL_ERROR_WITH_CORE, this, msg));
}
void MsgThread::InternalWarning(const char* msg)
{
SendOut(new ReporterMessage(ReporterMessage::INTERNAL_WARNING, this, msg));
}
void MsgThread::InternalError(const char* msg)
{
SendOut(new ReporterMessage(ReporterMessage::INTERNAL_ERROR, this, msg));
}
#ifdef DEBUG
void MsgThread::Debug(DebugStream stream, const char* msg)
{
SendOut(new DebugMessage(stream, this, msg));
}
#endif
void MsgThread::SendIn(BasicInputMessage* msg, bool force)
{
if ( Terminating() && ! force )
{
delete msg;
return;
}
DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str());
queue_in.Put(msg);
++cnt_sent_in;
}
void MsgThread::SendOut(BasicOutputMessage* msg, bool force)
{
if ( Terminating() && ! force )
{
delete msg;
return;
}
queue_out.Put(msg);
++cnt_sent_out;
}
BasicOutputMessage* MsgThread::RetrieveOut()
{
BasicOutputMessage* msg = queue_out.Get();
assert(msg);
DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str());
return msg;
}
BasicInputMessage* MsgThread::RetrieveIn()
{
BasicInputMessage* msg = queue_in.Get();
assert(msg);
#ifdef DEBUG
string s = Fmt("Retrieved '%s' in %s", msg->Name().c_str(), Name().c_str());
Debug(DBG_THREADING, s.c_str());
#endif
return msg;
}
void MsgThread::Run()
{
while ( true )
{
// When requested to terminate, we only do so when
// all input has been processed.
if ( Terminating() && ! queue_in.Ready() )
break;
BasicInputMessage* msg = RetrieveIn();
bool result = msg->Process();
if ( ! result )
{
string s = msg->Name() + " failed, terminating thread";
Error(s.c_str());
Stop();
break;
}
delete msg;
}
}
void MsgThread::GetStats(Stats* stats)
{
stats->sent_in = cnt_sent_in;
stats->sent_out = cnt_sent_out;
stats->pending_in = queue_in.Size();
stats->pending_out = queue_out.Size();
queue_in.GetStats(&stats->queue_in_stats);
queue_out.GetStats(&stats->queue_out_stats);
}

402
src/threading/MsgThread.h Normal file
View file

@ -0,0 +1,402 @@
#ifndef THREADING_MSGTHREAD_H
#define THREADING_MSGTHREAD_H
#include <pthread.h>
#include "DebugLogger.h"
#include "BasicThread.h"
#include "Queue.h"
namespace threading {
class BasicInputMessage;
class BasicOutputMessage;
class HeartbeatMessage;
/**
* A specialized thread that provides bi-directional message passing between
* Bro's main thread and the child thread. Messages are instances of
* BasicInputMessage and BasicOutputMessage for message sent \a to the child
* thread and received \a from the child thread, respectively.
*
* The thread's Run() method implements main loop that processes incoming
* messages until Terminating() indicates that execution should stop. Once
* that happens, the thread stops accepting any new messages, finishes
* processes all remaining ones still in the queue, and then exits.
*/
class MsgThread : public BasicThread
{
public:
/**
* Constructor. It automatically registers the thread with the
* threading::Manager.
*
* Only Bro's main thread may instantiate a new thread.
*/
MsgThread();
/**
* Sends a message to the child thread. The message will be proceesed
* once the thread has retrieved it from its incoming queue.
*
* Only the main thread may call this method.
*
* @param msg The message.
*/
void SendIn(BasicInputMessage* msg) { return SendIn(msg, false); }
/**
* Sends a message from the child thread to the main thread.
*
* Only the child thread may call this method.
*
* @param msg The mesasge.
*/
void SendOut(BasicOutputMessage* msg) { return SendOut(msg, false); }
/**
* Reports an informational message from the child thread. The main
* thread will pass this to the Reporter once received.
*
* Only the child thread may call this method.
*
* @param msg The message. It will be prefixed with the thread's name.
*/
void Info(const char* msg);
/**
* Reports a warning from the child thread that may indicate a
* problem. The main thread will pass this to the Reporter once
* received.
*
* Only the child thread may call this method.
*
* @param msg The message. It will be prefixed with the thread's name.
*/
void Warning(const char* msg);
/**
* Reports a non-fatal error from the child thread. The main thread
* will pass this to the Reporter once received. Processing proceeds
* normally after the error has been reported.
*
* Only the child thread may call this method.
*
* @param msg The message. It will be prefixed with the thread's name.
*/
void Error(const char* msg);
/**
* Reports a fatal error from the child thread. The main thread will
* pass this to the Reporter once received. Bro will terminate after
* the message has been reported.
*
* Only the child thread may call this method.
*
* @param msg The message. It will be prefixed with the thread's name.
*/
void FatalError(const char* msg);
/**
* Reports a fatal error from the child thread. The main thread will
* pass this to the Reporter once received. Bro will terminate with a
* core dump after the message has been reported.
*
* Only the child thread may call this method.
*
* @param msg The message. It will be prefixed with the thread's name.
*/
void FatalErrorWithCore(const char* msg);
/**
* Reports a potential internal problem from the child thread. The
* main thread will pass this to the Reporter once received. Bro will
* continue normally.
*
* Only the child thread may call this method.
*
* @param msg The message. It will be prefixed with the thread's name.
*/
void InternalWarning(const char* msg);
/**
* Reports an internal program error from the child thread. The main
* thread will pass this to the Reporter once received. Bro will
* terminate with a core dump after the message has been reported.
*
* Only the child thread may call this method.
*
* @param msg The message. It will be prefixed with the thread's name.
*/
void InternalError(const char* msg);
#ifdef DEBUG
/**
* Records a debug message for the given stream from the child
* thread. The main thread will pass this to the DebugLogger once
* received.
*
* Only the child thread may call this method.
*
* @param msg The message. It will be prefixed with the thread's name.
*/
void Debug(DebugStream stream, const char* msg);
#endif
/**
* Statistics about inter-thread communication.
*/
struct Stats
{
uint64_t sent_in; //! Number of messages sent to the child thread.
uint64_t sent_out; //! Number of messages sent from the child thread to the main thread
uint64_t pending_in; //! Number of messages sent to the child but not yet processed.
uint64_t pending_out; //! Number of messages sent from the child but not yet processed by the main thread.
/// Statistics from our queues.
Queue<BasicInputMessage *>::Stats queue_in_stats;
Queue<BasicOutputMessage *>::Stats queue_out_stats;
};
/**
* Returns statistics about the inter-thread communication.
*
* @param stats A pointer to a structure that will be filled with
* current numbers.
*/
void GetStats(Stats* stats);
protected:
friend class Manager;
friend class HeartbeatMessage;
/**
* Pops a message sent by the child from the child-to-main queue.
*
* This is method is called regularly by the threading::Manager.
*
* @return The message, wth ownership passed to caller. Returns null
* if the queue is empty.
*/
BasicOutputMessage* RetrieveOut();
/**
* Triggers a heartbeat message being sent to the client thread.
*
* This is method is called regularly by the threading::Manager.
*
* Can be overriden in derived classed to hook into the heart beat,
* but must call the parent implementation. Note that this method is
* always called by the main thread and must not access data of the
* child thread directly. See DoHeartbeat() if you want to do
* something on the child-side.
*/
virtual void Heartbeat();
/**
* Overriden from BasicThread.
*
*/
virtual void Run();
virtual void OnStop();
/**
* Regulatly triggered for execution in the child thread.
*
* When overriding, one must call the parent class' implementation.
*
* network_time: The network_time when the heartbeat was trigger by
* the main thread.
*
* current_time: Wall clock when the heartbeat was trigger by the
* main thread.
*/
virtual bool DoHeartbeat(double network_time, double current_time);
private:
/**
* Pops a message sent by the main thread from the main-to-chold
* queue.
*
* Must only be called by the child thread.
*
* @return The message, wth ownership passed to caller. Returns null
* if the queue is empty.
*/
BasicInputMessage* RetrieveIn();
/**
* Queues a message for the child.
*
* Must only be called by the main thread.
*
* @param msg The message.
*
* @param force: If true, the message will be queued even when we're already
* Terminating(). Normally, the message would be discarded in that
* case.
*/
void SendIn(BasicInputMessage* msg, bool force);
/**
* Queues a message for the main thread.
*
* Must only be called by the child thread.
*
* @param msg The message.
*
* @param force: If true, the message will be queued even when we're already
* Terminating(). Normally, the message would be discarded in that
* case.
*/
void SendOut(BasicOutputMessage* msg, bool force);
/**
* Returns true if there's at least one message pending for the child
* thread.
*/
bool HasIn() { return queue_in.Ready(); }
/**
* Returns true if there's at least one message pending for the main
* thread.
*/
bool HasOut() { return queue_out.Ready(); }
Queue<BasicInputMessage *> queue_in;
Queue<BasicOutputMessage *> queue_out;
uint64_t cnt_sent_in; // Counts message sent to child.
uint64_t cnt_sent_out; // Counts message sent by child.
};
/**
* Base class for all message between Bro's main process and a MsgThread.
*/
class Message
{
public:
/**
* Destructor.
*/
virtual ~Message();
/**
* Returns a descriptive name for the message's general type. This is
* what's passed into the constructor and used mainly for debugging
* purposes.
*/
const string& Name() const { return name; }
/**
* Callback that must be overriden for processing a message.
*/
virtual bool Process() = 0; // Thread will be terminated if returngin false.
protected:
/**
* Constructor.
*
* @param arg_name A descriptive name for the type of message. Used
* mainly for debugging purposes.
*/
Message(const string& arg_name) { name = arg_name; }
private:
string name;
};
/**
* Base class for messages sent from Bro's main thread to a child MsgThread.
*/
class BasicInputMessage : public Message
{
protected:
/**
* Constructor.
*
* @param name A descriptive name for the type of message. Used
* mainly for debugging purposes.
*/
BasicInputMessage(const string& name) : Message(name) {}
};
/**
* Base class for messages sent from a child MsgThread to Bro's main thread.
*/
class BasicOutputMessage : public Message
{
protected:
/**
* Constructor.
*
* @param name A descriptive name for the type of message. Used
* mainly for debugging purposes.
*/
BasicOutputMessage(const string& name) : Message(name) {}
};
/**
* A paremeterized InputMessage that stores a pointer to an argument object.
* Normally, the objects will be used from the Process() callback.
*/
template<typename O>
class InputMessage : public BasicInputMessage
{
public:
/**
* Returns the objects passed to the constructor.
*/
O* Object() const { return object; }
protected:
/**
* Constructor.
*
* @param name: A descriptive name for the type of message. Used
* mainly for debugging purposes.
*
* @param arg_object: An object to store with the message.
*/
InputMessage(const string& name, O* arg_object) : BasicInputMessage(name)
{ object = arg_object; }
private:
O* object;
};
/**
* A paremeterized OututMessage that stores a pointer to an argument object.
* Normally, the objects will be used from the Process() callback.
*/
template<typename O>
class OutputMessage : public BasicOutputMessage
{
public:
/**
* Returns the objects passed to the constructor.
*/
O* Object() const { return object; }
protected:
/**
* Constructor.
*
* @param name A descriptive name for the type of message. Used
* mainly for debugging purposes.
*
* @param arg_object An object to store with the message.
*/
OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name)
{ object = arg_object; }
private:
O* object;
};
}
#endif

222
src/threading/Queue.h Normal file
View file

@ -0,0 +1,222 @@
#ifndef THREADING_QUEUE_H
#define THREADING_QUEUE_H
#include <pthread.h>
#include <queue>
#include <deque>
#include <stdint.h>
#include "Reporter.h"
#undef Queue // Defined elsewhere unfortunately.
namespace threading {
/**
* A thread-safe single-reader single-writer queue.
*
* The implementation uses multiple queues and reads/writes in rotary fashion
* in an attempt to limit contention.
*
* All Queue instances must be instantiated by Bro's main thread.
*
* TODO: Unclear how critical performance is for this qeueue. We could like;y
* optimize it further if helpful.
*/
template<typename T>
class Queue
{
public:
/**
* Constructor.
*/
Queue();
/**
* Destructor.
*/
~Queue();
/**
* Retrieves one elment.
*/
T Get();
/**
* Queues one element.
*/
void Put(T data);
/**
* Returns true if the next Get() operation will succeed.
*/
bool Ready();
/**
* Returns the number of queued items not yet retrieved.
*/
uint64_t Size();
/**
* Statistics about inter-thread communication.
*/
struct Stats
{
uint64_t num_reads; //! Number of messages read from the queue.
uint64_t num_writes; //! Number of messages written to the queue.
};
/**
* Returns statistics about the queue's usage.
*
* @param stats A pointer to a structure that will be filled with
* current numbers. */
void GetStats(Stats* stats);
private:
static const int NUM_QUEUES = 8;
pthread_mutex_t mutex[NUM_QUEUES]; // Mutex protected shared accesses.
pthread_cond_t has_data[NUM_QUEUES]; // Signals when data becomes available
std::queue<T> messages[NUM_QUEUES]; // Actually holds the queued messages
int read_ptr; // Where the next operation will read from
int write_ptr; // Where the next operation will write to
// Statistics.
uint64_t num_reads;
uint64_t num_writes;
};
inline static void safe_lock(pthread_mutex_t* mutex)
{
if ( pthread_mutex_lock(mutex) != 0 )
reporter->FatalErrorWithCore("cannot lock mutex");
}
inline static void safe_unlock(pthread_mutex_t* mutex)
{
if ( pthread_mutex_unlock(mutex) != 0 )
reporter->FatalErrorWithCore("cannot unlock mutex");
}
template<typename T>
inline Queue<T>::Queue()
{
read_ptr = 0;
write_ptr = 0;
num_reads = num_writes = 0;
for( int i = 0; i < NUM_QUEUES; ++i )
{
if ( pthread_cond_init(&has_data[i], NULL) != 0 )
reporter->FatalError("cannot init queue condition variable");
if ( pthread_mutex_init(&mutex[i], NULL) != 0 )
reporter->FatalError("cannot init queue mutex");
}
}
template<typename T>
inline Queue<T>::~Queue()
{
for( int i = 0; i < NUM_QUEUES; ++i )
{
pthread_cond_destroy(&has_data[i]);
pthread_mutex_destroy(&mutex[i]);
}
}
template<typename T>
inline T Queue<T>::Get()
{
safe_lock(&mutex[read_ptr]);
int old_read_ptr = read_ptr;
if ( messages[read_ptr].empty() )
pthread_cond_wait(&has_data[read_ptr], &mutex[read_ptr]);
T data = messages[read_ptr].front();
messages[read_ptr].pop();
read_ptr = (read_ptr + 1) % NUM_QUEUES;
++num_reads;
safe_unlock(&mutex[old_read_ptr]);
return data;
}
template<typename T>
inline void Queue<T>::Put(T data)
{
safe_lock(&mutex[write_ptr]);
int old_write_ptr = write_ptr;
bool need_signal = messages[write_ptr].empty();
messages[write_ptr].push(data);
if ( need_signal )
pthread_cond_signal(&has_data[write_ptr]);
write_ptr = (write_ptr + 1) % NUM_QUEUES;
++num_writes;
safe_unlock(&mutex[old_write_ptr]);
}
template<typename T>
inline bool Queue<T>::Ready()
{
safe_lock(&mutex[read_ptr]);
bool ret = (messages[read_ptr].size());
safe_unlock(&mutex[read_ptr]);
return ret;
}
template<typename T>
inline uint64_t Queue<T>::Size()
{
// Need to lock all queues.
for ( int i = 0; i < NUM_QUEUES; i++ )
safe_lock(&mutex[i]);
uint64_t size = 0;
for ( int i = 0; i < NUM_QUEUES; i++ )
size += messages[i].size();
for ( int i = 0; i < NUM_QUEUES; i++ )
safe_unlock(&mutex[i]);
return size;
}
template<typename T>
inline void Queue<T>::GetStats(Stats* stats)
{
// To be safe, we look all queues. That's probably unneccessary, but
// doesn't really hurt.
for ( int i = 0; i < NUM_QUEUES; i++ )
safe_lock(&mutex[i]);
stats->num_reads = num_reads;
stats->num_writes = num_writes;
for ( int i = 0; i < NUM_QUEUES; i++ )
safe_unlock(&mutex[i]);
}
}
#endif

355
src/threading/SerialTypes.cc Normal file
View file

@ -0,0 +1,355 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "SerialTypes.h"
#include "../RemoteSerializer.h"
using namespace threading;
bool Field::Read(SerializationFormat* fmt)
{
int t;
int st;
bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") );
type = (TypeTag) t;
subtype = (TypeTag) st;
return success;
}
bool Field::Write(SerializationFormat* fmt) const
{
return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype"));
}
Value::~Value()
{
if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC)
&& present )
delete val.string_val;
if ( type == TYPE_TABLE && present )
{
for ( int i = 0; i < val.set_val.size; i++ )
delete val.set_val.vals[i];
delete [] val.set_val.vals;
}
if ( type == TYPE_VECTOR && present )
{
for ( int i = 0; i < val.vector_val.size; i++ )
delete val.vector_val.vals[i];
delete [] val.vector_val.vals;
}
}
bool Value::IsCompatibleType(BroType* t, bool atomic_only)
{
if ( ! t )
return false;
switch ( t->Tag() ) {
case TYPE_BOOL:
case TYPE_INT:
case TYPE_COUNT:
case TYPE_COUNTER:
case TYPE_PORT:
case TYPE_SUBNET:
case TYPE_ADDR:
case TYPE_DOUBLE:
case TYPE_TIME:
case TYPE_INTERVAL:
case TYPE_ENUM:
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
return true;
case TYPE_RECORD:
return ! atomic_only;
case TYPE_TABLE:
{
if ( atomic_only )
return false;
if ( ! t->IsSet() )
return false;
return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true);
}
case TYPE_VECTOR:
{
if ( atomic_only )
return false;
return IsCompatibleType(t->AsVectorType()->YieldType(), true);
}
default:
return false;
}
return false;
}
bool Value::Read(SerializationFormat* fmt)
{
int ty;
if ( ! (fmt->Read(&ty, "type") && fmt->Read(&present, "present")) )
return false;
type = (TypeTag)(ty);
if ( ! present )
return true;
switch ( type ) {
case TYPE_BOOL:
case TYPE_INT:
return fmt->Read(&val.int_val, "int");
case TYPE_COUNT:
case TYPE_COUNTER:
return fmt->Read(&val.uint_val, "uint");
case TYPE_PORT: {
int proto;
if ( ! (fmt->Read(&val.port_val.port, "port") && fmt->Read(&proto, "proto") ) ) {
return false;
}
switch ( proto ) {
case 0:
val.port_val.proto = TRANSPORT_UNKNOWN;
break;
case 1:
val.port_val.proto = TRANSPORT_TCP;
break;
case 2:
val.port_val.proto = TRANSPORT_UDP;
break;
case 3:
val.port_val.proto = TRANSPORT_ICMP;
break;
default:
return false;
}
return true;
}
case TYPE_ADDR:
{
char family;
if ( ! fmt->Read(&family, "addr-family") )
return false;
switch ( family ) {
case 4:
val.addr_val.family = IPv4;
return fmt->Read(&val.addr_val.in.in4, "addr-in4");
case 6:
val.addr_val.family = IPv6;
return fmt->Read(&val.addr_val.in.in6, "addr-in6");
}
// Can't be reached.
abort();
}
case TYPE_SUBNET:
{
char length;
char family;
if ( ! (fmt->Read(&length, "subnet-len") && fmt->Read(&family, "subnet-family")) )
return false;
switch ( family ) {
case 4:
val.subnet_val.length = (uint8_t)length;
val.subnet_val.prefix.family = IPv4;
return fmt->Read(&val.subnet_val.prefix.in.in4, "subnet-in4");
case 6:
val.subnet_val.length = (uint8_t)length;
val.subnet_val.prefix.family = IPv6;
return fmt->Read(&val.subnet_val.prefix.in.in6, "subnet-in6");
}
// Can't be reached.
abort();
}
case TYPE_DOUBLE:
case TYPE_TIME:
case TYPE_INTERVAL:
return fmt->Read(&val.double_val, "double");
case TYPE_ENUM:
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
{
val.string_val = new string;
return fmt->Read(val.string_val, "string");
}
case TYPE_TABLE:
{
if ( ! fmt->Read(&val.set_val.size, "set_size") )
return false;
val.set_val.vals = new Value* [val.set_val.size];
for ( int i = 0; i < val.set_val.size; ++i )
{
val.set_val.vals[i] = new Value;
if ( ! val.set_val.vals[i]->Read(fmt) )
return false;
}
return true;
}
case TYPE_VECTOR:
{
if ( ! fmt->Read(&val.vector_val.size, "vector_size") )
return false;
val.vector_val.vals = new Value* [val.vector_val.size];
for ( int i = 0; i < val.vector_val.size; ++i )
{
val.vector_val.vals[i] = new Value;
if ( ! val.vector_val.vals[i]->Read(fmt) )
return false;
}
return true;
}
default:
reporter->InternalError("unsupported type %s in Value::Write", type_name(type));
}
return false;
}
bool Value::Write(SerializationFormat* fmt) const
{
if ( ! (fmt->Write((int)type, "type") &&
fmt->Write(present, "present")) )
return false;
if ( ! present )
return true;
switch ( type ) {
case TYPE_BOOL:
case TYPE_INT:
return fmt->Write(val.int_val, "int");
case TYPE_COUNT:
case TYPE_COUNTER:
return fmt->Write(val.uint_val, "uint");
case TYPE_PORT:
return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto");
case TYPE_ADDR:
{
switch ( val.addr_val.family ) {
case IPv4:
return fmt->Write((char)4, "addr-family")
&& fmt->Write(val.addr_val.in.in4, "addr-in4");
case IPv6:
return fmt->Write((char)6, "addr-family")
&& fmt->Write(val.addr_val.in.in6, "addr-in6");
break;
}
// Can't be reached.
abort();
}
case TYPE_SUBNET:
{
if ( ! fmt->Write((char)val.subnet_val.length, "subnet-length") )
return false;
switch ( val.subnet_val.prefix.family ) {
case IPv4:
return fmt->Write((char)4, "subnet-family")
&& fmt->Write(val.subnet_val.prefix.in.in4, "subnet-in4");
case IPv6:
return fmt->Write((char)6, "subnet-family")
&& fmt->Write(val.subnet_val.prefix.in.in6, "subnet-in6");
break;
}
// Can't be reached.
abort();
}
case TYPE_DOUBLE:
case TYPE_TIME:
case TYPE_INTERVAL:
return fmt->Write(val.double_val, "double");
case TYPE_ENUM:
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
return fmt->Write(*val.string_val, "string");
case TYPE_TABLE:
{
if ( ! fmt->Write(val.set_val.size, "set_size") )
return false;
for ( int i = 0; i < val.set_val.size; ++i )
{
if ( ! val.set_val.vals[i]->Write(fmt) )
return false;
}
return true;
}
case TYPE_VECTOR:
{
if ( ! fmt->Write(val.vector_val.size, "vector_size") )
return false;
for ( int i = 0; i < val.vector_val.size; ++i )
{
if ( ! val.vector_val.vals[i]->Write(fmt) )
return false;
}
return true;
}
default:
reporter->InternalError("unsupported type %s in Value::REad", type_name(type));
}
return false;
}

146
src/threading/SerialTypes.h Normal file
View file

@ -0,0 +1,146 @@
#ifndef THREADING_SERIALIZATIONTYPES_H
#define THREADING_SERIALIZATIONTYPES_H
using namespace std;
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include "Type.h"
#include "net_util.h"
class SerializationFormat;
namespace threading {
/**
* Definition of a log file, i.e., one column of a log stream.
*/
struct Field {
string name; //! Name of the field.
TypeTag type; //! Type of the field.
TypeTag subtype; //! Inner type for sets.
/**
* Constructor.
*/
Field() { subtype = TYPE_VOID; }
/**
* Copy constructor.
*/
Field(const Field& other)
: name(other.name), type(other.type), subtype(other.subtype) { }
/**
* Unserializes a field.
*
* @param fmt The serialization format to use. The format handles
* low-level I/O.
*
* @return False if an error occured.
*/
bool Read(SerializationFormat* fmt);
/**
* Serializes a field.
*
* @param fmt The serialization format to use. The format handles
* low-level I/O.
*
* @return False if an error occured.
*/
bool Write(SerializationFormat* fmt) const;
};
/**
* Definition of a log value, i.e., a entry logged by a stream.
*
* This struct essentialy represents a serialization of a Val instance (for
* those Vals supported).
*/
struct Value {
TypeTag type; //! The type of the value.
bool present; //! False for optional record fields that are not set.
struct set_t { bro_int_t size; Value** vals; };
typedef set_t vec_t;
struct port_t { bro_uint_t port; TransportProto proto; };
struct addr_t {
IPFamily family;
union {
struct in_addr in4;
struct in6_addr in6;
} in;
};
struct subnet_t { addr_t prefix; uint8_t length; };
/**
* This union is a subset of BroValUnion, including only the types we
* can log directly. See IsCompatibleType().
*/
union _val {
bro_int_t int_val;
bro_uint_t uint_val;
port_t port_val;
double double_val;
set_t set_val;
vec_t vector_val;
addr_t addr_val;
subnet_t subnet_val;
string* string_val;
} val;
/**
* Constructor.
*
* arg_type: The type of the value.
*
* arg_present: False if the value represents an optional record field
* that is not set.
*/
Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true)
: type(arg_type), present(arg_present) {}
/**
* Destructor.
*/
~Value();
/**
* Unserializes a value.
*
* @param fmt The serialization format to use. The format handles low-level I/O.
*
* @return False if an error occured.
*/
bool Read(SerializationFormat* fmt);
/**
* Serializes a value.
*
* @param fmt The serialization format to use. The format handles
* low-level I/O.
*
* @return False if an error occured.
*/
bool Write(SerializationFormat* fmt) const;
/**
* Returns true if the type can be represented by a Value. If
* `atomic_only` is true, will not permit composite types.
*/
static bool IsCompatibleType(BroType* t, bool atomic_only=false);
private:
friend class ::IPAddr;
Value(const Value& other) { } // Disabled.
};
}
#endif /* THREADING_SERIALIZATIONTZPES_H */

2
src/util.h
View file

@ -37,7 +37,7 @@
#endif
#ifdef USE_PERFTOOLS
#ifdef USE_PERFTOOLS_DEBUG
#include <google/heap-checker.h>
#include <google/heap-profiler.h>
extern HeapLeakChecker* heap_checker;

10
testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log Normal file
View file

@ -0,0 +1,10 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path metrics
#fields ts metric_id filter_name index.host index.str index.network value
#types time enum string addr string subnet count
1331256494.591966 TEST_METRIC foo-bar 6.5.4.3 - - 4
1331256494.591966 TEST_METRIC foo-bar 7.2.1.5 - - 2
1331256494.591966 TEST_METRIC foo-bar 1.2.3.4 - - 6

10
testing/btest/Baseline/core.leaks.remote/sender.test.failure.log Normal file
View file

@ -0,0 +1,10 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path test.failure
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US
1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK
1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX

12
testing/btest/Baseline/core.leaks.remote/sender.test.log Normal file
View file

@ -0,0 +1,12 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path test
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown
1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US
1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK
1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR
1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX

9
testing/btest/Baseline/core.leaks.remote/sender.test.success.log Normal file
View file

@ -0,0 +1,9 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path test.success
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown
1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR

39
testing/btest/core/leaks/basic-cluster.bro Normal file
View file

@ -0,0 +1,39 @@
# Needs perftools support.
#
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
# @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT
# @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
# @TEST-EXEC: sleep 1
# @TEST-EXEC: btest-bg-run worker-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
# @TEST-EXEC: btest-bg-run worker-2 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
# @TEST-EXEC: btest-bg-wait -k 30
# @TEST-EXEC: btest-diff manager-1/metrics.log
@TEST-START-FILE cluster-layout.bro
redef Cluster::nodes = {
["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
};
@TEST-END-FILE
redef Log::default_rotation_interval = 0secs;
redef enum Metrics::ID += {
TEST_METRIC,
};
event bro_init() &priority=5
{
Metrics::add_filter(TEST_METRIC,
[$name="foo-bar",
$break_interval=3secs]);
if ( Cluster::local_node_type() == Cluster::WORKER )
{
Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
}
}

79
testing/btest/core/leaks/remote.bro Normal file
View file

@ -0,0 +1,79 @@
# Needs perftools support.
#
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
#
# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro
# @TEST-EXEC: sleep 1
# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro
# @TEST-EXEC: sleep 1
# @TEST-EXEC: btest-bg-wait -k 10
# @TEST-EXEC: btest-diff sender/test.log
# @TEST-EXEC: btest-diff sender/test.failure.log
# @TEST-EXEC: btest-diff sender/test.success.log
# @TEST-EXEC: cmp receiver/test.log sender/test.log
# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log
# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log
# This is the common part loaded by both sender and receiver.
module Test;
export {
# Create a new ID for our log stream
redef enum Log::ID += { LOG };
# Define a record with all the columns the log file can have.
# (I'm using a subset of fields from ssh-ext for demonstration.)
type Log: record {
t: time;
id: conn_id; # Will be rolled out into individual columns.
status: string &optional;
country: string &default="unknown";
} &log;
}
event bro_init()
{
Log::create_stream(Test::LOG, [$columns=Log]);
Log::add_filter(Test::LOG, [$name="f1", $path="test.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]);
}
#####
@TEST-START-FILE sender.bro
module Test;
@load frameworks/communication/listen
function fail(rec: Log): bool
{
return rec$status != "success";
}
event remote_connection_handshake_done(p: event_peer)
{
Log::add_filter(Test::LOG, [$name="f2", $path="test.failure", $pred=fail]);
local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
local r: Log = [$t=network_time(), $id=cid, $status="success"];
# Log something.
Log::write(Test::LOG, r);
Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
disconnect(p);
}
@TEST-END-FILE
@TEST-START-FILE receiver.bro
#####
redef Communication::nodes += {
["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
};
@TEST-END-FILE

2
testing/external/scripts/perftools-adapt-paths vendored
View file

@ -7,4 +7,4 @@
cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1
grep -q "No leaks found" $1
grep -qv "detected leaks of" $1