diff --git a/CHANGES b/CHANGES index 9d5c6dc05f..643e4ebfcd 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,63 @@ +2.0-257 | 2012-04-05 15:32:43 -0700 + + * Fix CMake from warning about unused ENABLE_PERFTOOLS_DEBUG + variable. (Jon Siwek) + + * Fix handling of IPv6 atomic fragments. (Jon Siwek) + + * Fix that prevents Bro processes that do neither local logging nor + request remote logs from spawning threads. (Robin Sommer) + + * Fixing perftools-debug support. (Robin Sommer) + + * Reverting SocketComm change tuning I/O behaviour. (Robin Sommer) + + * Adding notice_policy.log canonification for external tests. (Robin Sommer) + + +2.0-245 | 2012-04-04 17:25:20 -0700 + + * Internal restructuring of the logging framework: we now spawn + threads doing the I/O. From a user's perspective not much should + change, except that the OS may now show a bunch of Bro threads. + (Gilbert Clark and Robin Sommer). + + * When building Bro, we now always link in tcmalloc if it's found at + configure time. If it's installed but not picked up, + --with-perftools may help. (Robin Sommer) + + * Renaming the configure option --enable-perftools to + --enable-perftool-debug to indicate that the switch is only + relevant for debugging the heap. It's not needed to pick up + tcmalloc for better performance. (Robin Sommer) + +2.0-184 | 2012-03-28 15:11:11 -0700 + + * Improve handling of IPv6 Routing Type 0 headers. (Jon Siwek) + + - For RH0 headers with non-zero segments left, a + "routing0_segleft" flow_weird event is raised (with a + destination indicating the last address in the routing header), + and an "rh0_segleft" event can also be handled if the other + contents of the packet header are of interest. No further + analysis is done as the complexity required to correctly + identify destination endpoints of connections doesn't seem worth + it as RH0 has been deprecated by RFC 5095. + + - For RH0 headers without any segments left, a "routing0_header" + flow_weird event is raised, but further analysis still occurs as + normal. + +2.0-182 | 2012-03-28 15:01:57 -0700 + + * Remove dead tcp_checksum function from net_util. (Jon Siwek) + + * Change routing0_data_to_addrs BIF to return vector of addresses. + The order of addresses in type 0 routing headers is + interesting/important. (Jon Siwek) + + 2.0-179 | 2012-03-23 17:43:31 -0700 * Remove the default "tcp or udp or icmp" filter. In default mode, diff --git a/CMakeLists.txt b/CMakeLists.txt index de3138c20c..d27fa2d40b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,15 +89,29 @@ if (LIBGEOIP_FOUND) endif () set(USE_PERFTOOLS false) -if (ENABLE_PERFTOOLS) - find_package(GooglePerftools) - if (GOOGLEPERFTOOLS_FOUND) - set(USE_PERFTOOLS true) - include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR}) +set(USE_PERFTOOLS_DEBUG false) + +find_package(GooglePerftools) + +if (GOOGLEPERFTOOLS_FOUND) + include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR}) + set(USE_PERFTOOLS true) + + if (ENABLE_PERFTOOLS_DEBUG) + # Enable heap debugging with perftools. + set(USE_PERFTOOLS_DEBUG true) + list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG}) + else () + # Link in tcmalloc for better performance. list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES}) endif () endif () +if (ENABLE_PERFTOOLS_DEBUG) + # Just a no op to prevent CMake from complaining about manually-specified + # ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found +endif () + set(brodeps ${BinPAC_LIBRARY} ${PCAP_LIBRARY} @@ -183,6 +197,7 @@ message( "\n" "\nGeoIP: ${USE_GEOIP}" "\nGoogle perftools: ${USE_PERFTOOLS}" + "\n debugging: ${USE_PERFTOOLS_DEBUG}" "\n" "\n================================================================\n" ) diff --git a/NEWS b/NEWS index 75f9e4822b..6472ce9093 100644 --- a/NEWS +++ b/NEWS @@ -13,6 +13,10 @@ Bro 2.1 * Bro now requires CMake >= 2.6.3. + * Bro now links in tcmalloc (part of Google perftools) if found at + configure time. Doing so can significantly improve memory and + CPU use. + - Bro now supports IPv6 out of the box; the configure switch --enable-brov6 is gone. @@ -31,6 +35,14 @@ Bro 2.1 - The syntax for IPv6 literals changed from "2607:f8b0:4009:802::1012" to "[2607:f8b0:4009:802::1012]". +- Bro now spawn threads for doing its logging. From a user's + perspective not much should change, except that the OS may now show + a bunch of Bro threads. + +- We renamed the configure option --enable-perftools to + --enable-perftool-debug to indicate that the switch is only relevant + for debugging the heap. + TODO: Extend. Bro 2.0 diff --git a/VERSION b/VERSION index db8d11e293..10cad761b9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-179 +2.0-257 diff --git a/aux/binpac b/aux/binpac index dd1a3a95f0..56ae73ab99 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit dd1a3a95f07082efcd5274b21104a038d523d132 +Subproject commit 56ae73ab995dda665d8918d1a6b3af39b15991e3 diff --git a/aux/bro-aux b/aux/bro-aux index a59b35bdad..12d32194c1 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit a59b35bdada8f70fb1a59bf7bb2976534c86d378 +Subproject commit 12d32194c19d2dce06818588a2aeccf234de1889 diff --git a/aux/broccoli b/aux/broccoli index a4046c2f79..60898666ba 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit a4046c2f79b6ab0ac19ae8be94b79c6ce578bea7 +Subproject commit 60898666ba1df1913c08ad5045b1e56f974060cc diff --git a/aux/broctl b/aux/broctl index 66e9e87bee..4d1a0692a7 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 66e9e87beebce983fa0f479b0284d5690b0290d4 +Subproject commit 4d1a0692a7d7b5229230856a4041f70fd3a6eaa5 diff --git a/aux/btest b/aux/btest index dc78a3ebf5..8da6c55697 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit dc78a3ebf5cd8fbd1b3034990e36fa21a51d1a19 +Subproject commit 8da6c55697ff580600cfff474f4ccba2a592f911 diff --git a/cmake b/cmake index 550ab2c8d9..5ddec45563 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 550ab2c8d95b1d3e18e40a903152650e6c7a3c45 +Subproject commit 5ddec4556338339fc4d1da27bce766a827990543 diff --git a/config.h.in b/config.h.in index e744cb7dbd..0047344c51 100644 --- a/config.h.in +++ b/config.h.in @@ -109,7 +109,7 @@ #cmakedefine HAVE_GEOIP_CITY_EDITION_REV0_V6 /* Use Google's perftools */ -#cmakedefine USE_PERFTOOLS +#cmakedefine USE_PERFTOOLS_DEBUG /* Version number of package */ #define VERSION "@VERSION@" diff --git a/configure b/configure index 43afb4ae99..05aa12815b 100755 --- a/configure +++ b/configure @@ -27,7 +27,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... Optional Features: --enable-debug compile in debugging mode - --enable-perftools use Google's perftools + --enable-perftools-debug use Google's perftools for debugging --disable-broccoli don't build or install the Broccoli library --disable-broctl don't install Broctl --disable-auxtools don't build or install auxilliary tools @@ -91,7 +91,7 @@ append_cache_entry BRO_ROOT_DIR PATH /usr/local/bro append_cache_entry PY_MOD_INSTALL_DIR PATH /usr/local/bro/lib/broctl append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING /usr/local/bro/share/bro append_cache_entry ENABLE_DEBUG BOOL false -append_cache_entry ENABLE_PERFTOOLS BOOL false +append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false append_cache_entry BinPAC_SKIP_INSTALL BOOL true append_cache_entry BUILD_SHARED_LIBS BOOL true append_cache_entry INSTALL_AUX_TOOLS BOOL true @@ -132,8 +132,8 @@ while [ $# -ne 0 ]; do --enable-debug) append_cache_entry ENABLE_DEBUG BOOL true ;; - --enable-perftools) - append_cache_entry ENABLE_PERFTOOLS BOOL true + --enable-perftools-debug) + append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL true ;; --disable-broccoli) append_cache_entry INSTALL_BROCCOLI BOOL false @@ -178,7 +178,6 @@ while [ $# -ne 0 ]; do append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg ;; --with-perftools=*) - append_cache_entry ENABLE_PERFTOOLS BOOL true append_cache_entry GooglePerftools_ROOT_DIR PATH $optarg ;; --with-python=*) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 424c00e188..dad4c71e35 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -46,6 +46,13 @@ type index_vec: vector of count; ## then remove this alias. type string_vec: vector of string; +## A vector of addresses. +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. +type addr_vec: vector of addr; + ## A table of strings indexed by strings. ## ## .. todo:: We need this type definition only for declaring builtin functions via diff --git a/src/Anon.cc b/src/Anon.cc index 4c4146ac3e..d2a28a0e08 100644 --- a/src/Anon.cc +++ b/src/Anon.cc @@ -154,7 +154,7 @@ void AnonymizeIPAddr_A50::init() int AnonymizeIPAddr_A50::PreservePrefix(ipaddr32_t input, int num_bits) { DEBUG_MSG("%s/%d\n", - IPAddr(IPAddr::IPv4, &input, IPAddr::Network).AsString().c_str(), + IPAddr(IPv4, &input, IPAddr::Network).AsString().c_str(), num_bits); if ( ! before_anonymization ) diff --git a/src/Attr.cc b/src/Attr.cc index aed9165182..82d9c9ddc7 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -5,7 +5,7 @@ #include "Attr.h" #include "Expr.h" #include "Serializer.h" -#include "LogMgr.h" +#include "threading/SerialTypes.h" const char* attr_name(attr_tag t) { @@ -416,7 +416,7 @@ void Attributes::CheckAttr(Attr* a) break; case ATTR_LOG: - if ( ! LogVal::IsCompatibleType(type) ) + if ( ! threading::Value::IsCompatibleType(type) ) Error("&log applied to a type that cannot be logged"); break; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 74c3c747ce..ce1b25dd42 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -213,6 +213,8 @@ binpac_target(syslog.pac ######################################################################## ## bro target +find_package (Threads) + # This macro stores associated headers for any C/C++ source files given # as arguments (past _var) as a list in the CMake variable named "_var". macro(COLLECT_HEADERS _var) @@ -335,10 +337,6 @@ set(bro_SRCS IRC.cc List.cc Reporter.cc - LogMgr.cc - LogWriter.cc - LogWriterAscii.cc - LogWriterNone.cc Login.cc MIME.cc NCP.cc @@ -411,6 +409,18 @@ set(bro_SRCS PacketDumper.cc strsep.c modp_numtoa.c + + threading/BasicThread.cc + threading/Manager.cc + threading/MsgThread.cc + threading/SerialTypes.cc + + logging/Manager.cc + logging/WriterBackend.cc + logging/WriterFrontend.cc + logging/writers/Ascii.cc + logging/writers/None.cc + ${dns_SRCS} ${openssl_SRCS} ) @@ -423,7 +433,7 @@ add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}") add_executable(bro ${bro_SRCS} ${bro_HEADERS}) -target_link_libraries(bro ${brodeps}) +target_link_libraries(bro ${brodeps} ${CMAKE_THREAD_LIBS_INIT}) install(TARGETS bro DESTINATION bin) install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base) diff --git a/src/CompHash.cc b/src/CompHash.cc index ae0e082216..86677f9719 100644 --- a/src/CompHash.cc +++ b/src/CompHash.cc @@ -709,7 +709,7 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, const uint32* const kp = AlignType(kp0); kp1 = reinterpret_cast(kp+4); - IPAddr addr(IPAddr::IPv6, kp, IPAddr::Network); + IPAddr addr(IPv6, kp, IPAddr::Network); switch ( tag ) { case TYPE_ADDR: diff --git a/src/DCE_RPC.cc b/src/DCE_RPC.cc index 88cdb48e80..21cb3be9a0 100644 --- a/src/DCE_RPC.cc +++ b/src/DCE_RPC.cc @@ -137,7 +137,7 @@ static bool is_mapped_dce_rpc_endpoint(const dce_rpc_endpoint_addr& addr) bool is_mapped_dce_rpc_endpoint(const ConnID* id, TransportProto proto) { - if ( id->dst_addr.GetFamily() == IPAddr::IPv6 ) + if ( id->dst_addr.GetFamily() == IPv6 ) // TODO: Does the protocol support v6 addresses? #773 return false; @@ -414,7 +414,7 @@ void DCE_RPC_Session::DeliverEpmapperMapResponse( case binpac::DCE_RPC_Simple::EPM_PROTOCOL_IP: uint32 hostip = floor->rhs()->data()->ip(); - mapped.addr.addr = IPAddr(IPAddr::IPv4, &hostip, IPAddr::Host); + mapped.addr.addr = IPAddr(IPv4, &hostip, IPAddr::Host); break; } } diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index a80de42704..9e65d3c9a9 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -321,10 +321,10 @@ void DNS_Mapping::Init(struct hostent* h) addrs = new IPAddr[num_addrs]; for ( int i = 0; i < num_addrs; ++i ) if ( h->h_addrtype == AF_INET ) - addrs[i] = IPAddr(IPAddr::IPv4, (uint32*)h->h_addr_list[i], + addrs[i] = IPAddr(IPv4, (uint32*)h->h_addr_list[i], IPAddr::Network); else if ( h->h_addrtype == AF_INET6 ) - addrs[i] = IPAddr(IPAddr::IPv6, (uint32*)h->h_addr_list[i], + addrs[i] = IPAddr(IPv6, (uint32*)h->h_addr_list[i], IPAddr::Network); } else diff --git a/src/DPM.cc b/src/DPM.cc index fdd7d81222..d7e5cd25ef 100644 --- a/src/DPM.cc +++ b/src/DPM.cc @@ -74,7 +74,7 @@ void DPM::PostScriptInit() void DPM::AddConfig(const Analyzer::Config& cfg) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index d60fdd70c8..c41a0552c6 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -15,7 +15,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, - { "logging", 0, false } + { "logging", 0, false }, { "threading", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index a2dece5b3c..71e21bfa26 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -24,6 +24,7 @@ enum DebugStream { DBG_DPD, // Dynamic application detection framework DBG_TM, // Time-machine packet input via Brocolli DBG_LOGGING, // Logging streams + DBG_THREADING, // Threading system NUM_DBGS // Has to be last }; diff --git a/src/Desc.cc b/src/Desc.cc index 12b4a524eb..9d94321427 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -157,6 +157,16 @@ void ODesc::Add(double d) } } +void ODesc::Add(const IPAddr& addr) + { + Add(addr.AsString()); + } + +void ODesc::Add(const IPPrefix& prefix) + { + Add(prefix.AsString()); + } + void ODesc::AddCS(const char* s) { int n = strlen(s); diff --git a/src/Desc.h b/src/Desc.h index 9f8d7ab109..9c60c68106 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -8,7 +8,6 @@ #include #include "BroString.h" -#include "IPAddr.h" typedef enum { DESC_READABLE, @@ -23,6 +22,8 @@ typedef enum { } desc_style; class BroFile; +class IPAddr; +class IPPrefix; class ODesc { public: @@ -76,8 +77,8 @@ public: void Add(int64 i); void Add(uint64 u); void Add(double d); - void Add(const IPAddr& addr) { Add(addr.AsString()); } - void Add(const IPPrefix& prefix) { Add(prefix.AsString()); } + void Add(const IPAddr& addr); + void Add(const IPPrefix& prefix); // Add s as a counted string. void AddCS(const char* s); diff --git a/src/File.cc b/src/File.cc index 080923ad37..d4e31bcc16 100644 --- a/src/File.cc +++ b/src/File.cc @@ -232,7 +232,7 @@ BroFile::~BroFile() delete [] access; delete [] cipher_buffer; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->UnIgnoreObject(this); #endif } @@ -255,7 +255,7 @@ void BroFile::Init() cipher_ctx = 0; cipher_buffer = 0; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->IgnoreObject(this); #endif } diff --git a/src/Frag.cc b/src/Frag.cc index 9bd16a71c9..04298e14ad 100644 --- a/src/Frag.cc +++ b/src/Frag.cc @@ -52,8 +52,6 @@ FragReassembler::FragReassembler(NetSessions* arg_s, frag_size = 0; // flag meaning "not known" next_proto = ip->NextProto(); - AddFragment(t, ip, pkt); - if ( frag_timeout != 0.0 ) { expire_timer = new FragTimer(this, t + frag_timeout); @@ -61,6 +59,8 @@ FragReassembler::FragReassembler(NetSessions* arg_s, } else expire_timer = 0; + + AddFragment(t, ip, pkt); } FragReassembler::~FragReassembler() diff --git a/src/ID.cc b/src/ID.cc index 3f5c76ca1d..a70aa3fd0e 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -372,7 +372,7 @@ ID* ID::Unserialize(UnserialInfo* info) Ref(id); global_scope()->Insert(id->Name(), id); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->IgnoreObject(id); #endif } diff --git a/src/IP.cc b/src/IP.cc index 4148c58a33..bb60d17f15 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -74,8 +74,8 @@ RecordVal* IPv6_Hdr::BuildRecordVal(VectorVal* chain) const rv->Assign(2, new Val(ntohs(ip6->ip6_plen), TYPE_COUNT)); rv->Assign(3, new Val(ip6->ip6_nxt, TYPE_COUNT)); rv->Assign(4, new Val(ip6->ip6_hlim, TYPE_COUNT)); - rv->Assign(5, new AddrVal(ip6->ip6_src)); - rv->Assign(6, new AddrVal(ip6->ip6_dst)); + rv->Assign(5, new AddrVal(IPAddr(ip6->ip6_src))); + rv->Assign(6, new AddrVal(IPAddr(ip6->ip6_dst))); if ( ! chain ) chain = new VectorVal(new VectorType( hdrType(ip6_ext_hdr_type, "ip6_ext_hdr")->Ref())); @@ -305,6 +305,24 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, bool set_next, uint16 next) chain.push_back(p); + // RFC 5095 deprecates routing type 0 headers, so raise weirds for that. + if ( current_type == IPPROTO_ROUTING && + ((const struct ip6_rthdr*)hdrs)->ip6r_type == 0 ) + { + IPAddr src(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_src); + + if ( ((const struct ip6_rthdr*)hdrs)->ip6r_segleft > 0 ) + { + const in6_addr* a = (const in6_addr*)(hdrs+len-16); + reporter->Weird(src, IPAddr(*a), "routing0_segleft"); + } + else + { + IPAddr dst(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_dst); + reporter->Weird(src, dst, "routing0_header"); + } + } + hdrs += len; length += len; } while ( current_type != IPPROTO_FRAGMENT && diff --git a/src/IP.h b/src/IP.h index cb5bcf77c7..daa508db7f 100644 --- a/src/IP.h +++ b/src/IP.h @@ -171,6 +171,20 @@ public: { return IsFragment() ? (ntohs(GetFragHdr()->ip6f_offlg) & 0x0001) != 0 : 0; } + /** + * Returns whether the chain contains a routing type 0 extension header + * with nonzero segments left. + */ + bool RH0SegLeft() const + { + for ( size_t i = 0; i < chain.size(); ++i ) + if ( chain[i]->Type() == IPPROTO_ROUTING && + ((const struct ip6_rthdr*)chain[i]->Data())->ip6r_type == 0 && + ((const struct ip6_rthdr*)chain[i]->Data())->ip6r_segleft > 0 ) + return true; + return false; + } + /** * Returns a vector of ip6_ext_hdr RecordVals that includes script-layer * representation of all extension headers in the chain. @@ -343,6 +357,13 @@ public: size_t NumHeaders() const { return ip4 ? 1 : ip6_hdrs->Size(); } + /** + * Returns true if this is an IPv6 header containing a routing type 0 + * extension with nonzero segments left, else returns false. + */ + bool RH0SegLeft() const + { return ip4 ? false : ip6_hdrs->RH0SegLeft(); } + /** * Returns an ip_hdr or ip6_hdr_chain RecordVal. */ diff --git a/src/IPAddr.cc b/src/IPAddr.cc index 52c3f9b35c..0ba5589fff 100644 --- a/src/IPAddr.cc +++ b/src/IPAddr.cc @@ -251,7 +251,7 @@ IPPrefix::IPPrefix(const in6_addr& in6, uint8_t length) IPPrefix::IPPrefix(const IPAddr& addr, uint8_t length) : prefix(addr) { - if ( prefix.GetFamily() == IPAddr::IPv4 ) + if ( prefix.GetFamily() == IPv4 ) { if ( length > 32 ) reporter->InternalError("Bad IPAddr(v4) IPPrefix length : %d", @@ -276,7 +276,7 @@ string IPPrefix::AsString() const { char l[16]; - if ( prefix.GetFamily() == IPAddr::IPv4 ) + if ( prefix.GetFamily() == IPv4 ) modp_uitoa10(length - 96, l); else modp_uitoa10(length, l); diff --git a/src/IPAddr.h b/src/IPAddr.h index f0c0ac12c8..8e1921e07b 100644 --- a/src/IPAddr.h +++ b/src/IPAddr.h @@ -10,6 +10,8 @@ #include "BroString.h" #include "Hash.h" #include "util.h" +#include "Type.h" +#include "threading/SerialTypes.h" struct ConnID; class ExpectedConn; @@ -25,7 +27,7 @@ public: /** * Address family. */ - enum Family { IPv4, IPv6 }; + typedef IPFamily Family; /** * Byte order. @@ -45,7 +47,7 @@ public: * * @param in6 The IPv6 address. */ - IPAddr(const in4_addr& in4) + explicit IPAddr(const in4_addr& in4) { memcpy(in6.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix)); memcpy(&in6.s6_addr[12], &in4.s_addr, sizeof(in4.s_addr)); @@ -56,7 +58,7 @@ public: * * @param in6 The IPv6 address. */ - IPAddr(const in6_addr& arg_in6) : in6(arg_in6) { } + explicit IPAddr(const in6_addr& arg_in6) : in6(arg_in6) { } /** * Constructs an address instance from a string representation. @@ -318,14 +320,19 @@ public: return memcmp(&addr1.in6, &addr2.in6, sizeof(in6_addr)) < 0; } + /** Converts the address into the type used internally by the + * inter-thread communication. + */ + void ConvertToThreadingValue(threading::Value::addr_t* v) const; + friend HashKey* BuildConnIDHashKey(const ConnID& id); friend HashKey* BuildExpectedConnHashKey(const ExpectedConn& c); - friend class IPPrefix; - unsigned int MemoryAllocation() const { return padded_sizeof(*this); } private: + friend class IPPrefix; + /** * Initializes an address instance from a string representation. * @@ -384,6 +391,25 @@ inline bool IPAddr::IsLoopback() const && (in6.s6_addr[14] == 0) && (in6.s6_addr[15] == 1)); } +inline void IPAddr::ConvertToThreadingValue(threading::Value::addr_t* v) const + { + v->family = GetFamily(); + + switch ( v->family ) { + + case IPv4: + CopyIPv4(&v->in.in4); + return; + + case IPv6: + CopyIPv6(&v->in.in6); + return; + + // Can't be reached. + abort(); + } + } + /** * Returns a hash key for a given ConnID. Passes ownership to caller. */ @@ -459,7 +485,7 @@ public: */ uint8_t Length() const { - return prefix.GetFamily() == IPAddr::IPv4 ? length - 96 : length; + return prefix.GetFamily() == IPv4 ? length - 96 : length; } /** @@ -516,6 +542,15 @@ public: return new HashKey(&key, sizeof(key)); } + /** Converts the prefix into the type used internally by the + * inter-thread communication. + */ + void ConvertToThreadingValue(threading::Value::subnet_t* v) const + { + v->length = length; + prefix.ConvertToThreadingValue(&v->prefix); + } + unsigned int MemoryAllocation() const { return padded_sizeof(*this); } /** diff --git a/src/LogMgr.h b/src/LogMgr.h deleted file mode 100644 index c66f9ed7f8..0000000000 --- a/src/LogMgr.h +++ /dev/null @@ -1,143 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. -// -// A class managing log writers and filters. - -#ifndef LOGMGR_H -#define LOGMGR_H - -#include "Val.h" -#include "EventHandler.h" -#include "RemoteSerializer.h" -#include "IPAddr.h" - -class SerializationFormat; - -// Description of a log field. -struct LogField { - string name; - TypeTag type; - // inner type of sets - TypeTag subtype; - - LogField() { subtype = TYPE_VOID; } - LogField(const LogField& other) - : name(other.name), type(other.type), subtype(other.subtype) { } - - // (Un-)serialize. - bool Read(SerializationFormat* fmt); - bool Write(SerializationFormat* fmt) const; -}; - -// Values as logged by a writer. -struct LogVal { - TypeTag type; - bool present; // False for unset fields. - - // The following union is a subset of BroValUnion, including only the - // types we can log directly. - struct set_t { bro_int_t size; LogVal** vals; }; - typedef set_t vec_t; - - union _val { - bro_int_t int_val; - bro_uint_t uint_val; - IPAddr* addr_val; - IPPrefix* subnet_val; - double double_val; - string* string_val; - set_t set_val; - vec_t vector_val; - } val; - - LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) - : type(arg_type), present(arg_present) {} - ~LogVal(); - - // (Un-)serialize. - bool Read(SerializationFormat* fmt); - bool Write(SerializationFormat* fmt) const; - - // Returns true if the type can be logged the framework. If - // `atomic_only` is true, will not permit composite types. - static bool IsCompatibleType(BroType* t, bool atomic_only=false); - -private: - LogVal(const LogVal& other) { } -}; - -class LogWriter; -class RemoteSerializer; -class RotationTimer; - -class LogMgr { -public: - LogMgr(); - ~LogMgr(); - - // These correspond to the BiFs visible on the scripting layer. The - // actual BiFs just forward here. - bool CreateStream(EnumVal* id, RecordVal* stream); - bool EnableStream(EnumVal* id); - bool DisableStream(EnumVal* id); - bool AddFilter(EnumVal* id, RecordVal* filter); - bool RemoveFilter(EnumVal* id, StringVal* name); - bool RemoveFilter(EnumVal* id, string name); - bool Write(EnumVal* id, RecordVal* columns); - bool SetBuf(EnumVal* id, bool enabled); // Adjusts all writers. - bool Flush(EnumVal* id); // Flushes all writers.. - -protected: - friend class LogWriter; - friend class RemoteSerializer; - friend class RotationTimer; - - //// Function also used by the RemoteSerializer. - - // Takes ownership of fields. - LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, LogField** fields); - - // Takes ownership of values.. - bool Write(EnumVal* id, EnumVal* writer, string path, - int num_fields, LogVal** vals); - - // Announces all instantiated writers to peer. - void SendAllWritersTo(RemoteSerializer::PeerID peer); - - //// Functions safe to use by writers. - - // Signals that a file has been rotated. - bool FinishedRotation(LogWriter* writer, string new_name, string old_name, - double open, double close, bool terminating); - - // Reports an error for the given writer. - void Error(LogWriter* writer, const char* msg); - - // Deletes the values as passed into Write(). - void DeleteVals(int num_fields, LogVal** vals); - -private: - struct Filter; - struct Stream; - struct WriterInfo; - - bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, - TableVal* include, TableVal* exclude, string path, list indices); - - LogVal** RecordToFilterVals(Stream* stream, Filter* filter, - RecordVal* columns); - - LogVal* ValToLogVal(Val* val, BroType* ty = 0); - Stream* FindStream(EnumVal* id); - void RemoveDisabledWriters(Stream* stream); - void InstallRotationTimer(WriterInfo* winfo); - void Rotate(WriterInfo* info); - Filter* FindFilter(EnumVal* id, StringVal* filter); - WriterInfo* FindWriter(LogWriter* writer); - - vector streams; // Indexed by stream enum. -}; - -extern LogMgr* log_mgr; - -#endif diff --git a/src/LogWriter.cc b/src/LogWriter.cc deleted file mode 100644 index 8584a0b0b5..0000000000 --- a/src/LogWriter.cc +++ /dev/null @@ -1,158 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "util.h" -#include "LogWriter.h" - -LogWriter::LogWriter() - { - buf = 0; - buf_len = 1024; - buffering = true; - disabled = false; - } - -LogWriter::~LogWriter() - { - if ( buf ) - free(buf); - - for(int i = 0; i < num_fields; ++i) - delete fields[i]; - - delete [] fields; - } - -bool LogWriter::Init(string arg_path, int arg_num_fields, - const LogField* const * arg_fields) - { - path = arg_path; - num_fields = arg_num_fields; - fields = arg_fields; - - if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) - { - disabled = true; - return false; - } - - return true; - } - -bool LogWriter::Write(int arg_num_fields, LogVal** vals) - { - // Double-check that the arguments match. If we get this from remote, - // something might be mixed up. - if ( num_fields != arg_num_fields ) - { - DBG_LOG(DBG_LOGGING, "Number of fields don't match in LogWriter::Write() (%d vs. %d)", - arg_num_fields, num_fields); - - DeleteVals(vals); - return false; - } - - for ( int i = 0; i < num_fields; ++i ) - { - if ( vals[i]->type != fields[i]->type ) - { - DBG_LOG(DBG_LOGGING, "Field type doesn't match in LogWriter::Write() (%d vs. %d)", - vals[i]->type, fields[i]->type); - DeleteVals(vals); - return false; - } - } - - bool result = DoWrite(num_fields, fields, vals); - - DeleteVals(vals); - - if ( ! result ) - disabled = true; - - return result; - } - -bool LogWriter::SetBuf(bool enabled) - { - if ( enabled == buffering ) - // No change. - return true; - - buffering = enabled; - - if ( ! DoSetBuf(enabled) ) - { - disabled = true; - return false; - } - - return true; - } - -bool LogWriter::Rotate(string rotated_path, double open, - double close, bool terminating) - { - if ( ! DoRotate(rotated_path, open, close, terminating) ) - { - disabled = true; - return false; - } - - return true; - } - -bool LogWriter::Flush() - { - if ( ! DoFlush() ) - { - disabled = true; - return false; - } - - return true; - } - -void LogWriter::Finish() - { - DoFinish(); - } - -const char* LogWriter::Fmt(const char* format, ...) - { - if ( ! buf ) - buf = (char*) malloc(buf_len); - - va_list al; - va_start(al, format); - int n = safe_vsnprintf(buf, buf_len, format, al); - va_end(al); - - if ( (unsigned int) n >= buf_len ) - { // Not enough room, grow the buffer. - buf_len = n + 32; - buf = (char*) realloc(buf, buf_len); - - // Is it portable to restart? - va_start(al, format); - n = safe_vsnprintf(buf, buf_len, format, al); - va_end(al); - } - - return buf; - } - -void LogWriter::Error(const char *msg) - { - log_mgr->Error(this, msg); - } - -void LogWriter::DeleteVals(LogVal** vals) - { - log_mgr->DeleteVals(num_fields, vals); - } - -bool LogWriter::FinishedRotation(string new_name, string old_name, double open, - double close, bool terminating) - { - return log_mgr->FinishedRotation(this, new_name, old_name, open, close, terminating); - } diff --git a/src/LogWriter.h b/src/LogWriter.h deleted file mode 100644 index 1d2f9fa4b2..0000000000 --- a/src/LogWriter.h +++ /dev/null @@ -1,192 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. -// -// Interface API for a log writer backend. The LogMgr creates a separate -// writer instance of pair of (writer type, output path). -// -// Note thay classes derived from LogWriter must be fully thread-safe and not -// use any non-thread-safe Bro functionality (which includes almost -// everything ...). In particular, do not use fmt() but LogWriter::Fmt()!. -// -// The one exception to this rule is the constructor: it is guaranteed to be -// executed inside the main thread and can thus in particular access global -// script variables. - -#ifndef LOGWRITER_H -#define LOGWRITER_H - -#include "LogMgr.h" -#include "BroString.h" - -class LogWriter { -public: - LogWriter(); - virtual ~LogWriter(); - - //// Interface methods to interact with the writer. Note that these - //// methods are not necessarily thread-safe and must be called only - //// from the main thread (which will typically mean only from the - //// LogMgr). In particular, they must not be called from the - //// writer's derived implementation. - - // One-time initialization of the writer to define the logged fields. - // Interpretation of "path" is left to the writer, and will be - // corresponding the value configured on the script-level. - // - // Returns false if an error occured, in which case the writer must - // not be used further. - // - // The new instance takes ownership of "fields", and will delete them - // when done. - bool Init(string path, int num_fields, const LogField* const * fields); - - // Writes one log entry. The method takes ownership of "vals" and - // will return immediately after queueing the write request, which is - // potentially before output has actually been written out. - // - // num_fields and the types of the LogVals must match what was passed - // to Init(). - // - // Returns false if an error occured, in which case the writer must - // not be used any further. - bool Write(int num_fields, LogVal** vals); - - // Sets the buffering status for the writer, if the writer supports - // that. (If not, it will be ignored). - bool SetBuf(bool enabled); - - // Flushes any currently buffered output, if the writer supports - // that. (If not, it will be ignored). - bool Flush(); - - // Triggers rotation, if the writer supports that. (If not, it will - // be ignored). - bool Rotate(string rotated_path, double open, double close, bool terminating); - - // Finishes writing to this logger regularly. Must not be called if - // an error has been indicated earlier. After calling this, no - // further writing must be performed. - void Finish(); - - //// Thread-safe methods that may be called from the writer - //// implementation. - - // The following methods return the information as passed to Init(). - const string Path() const { return path; } - int NumFields() const { return num_fields; } - const LogField* const * Fields() const { return fields; } - -protected: - // Methods for writers to override. If any of these returs false, it - // will be assumed that a fatal error has occured that prevents the - // writer from further operation. It will then be disabled and - // deleted. When return false, the writer should also report the - // error via Error(). Note that even if a writer does not support the - // functionality for one these methods (like rotation), it must still - // return true if that is not to be considered a fatal error. - // - // Called once for initialization of the writer. - virtual bool DoInit(string path, int num_fields, - const LogField* const * fields) = 0; - - // Called once per log entry to record. - virtual bool DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals) = 0; - - // Called when the buffering status for this writer is changed. If - // buffering is disabled, the writer should attempt to write out - // information as quickly as possible even if doing so may have a - // performance impact. If enabled (which is the default), it may - // buffer data as helpful and write it out later in a way optimized - // for performance. The current buffering state can be queried via - // IsBuf(). - // - // A writer may ignore buffering changes if it doesn't fit with its - // semantics (but must still return true in that case). - virtual bool DoSetBuf(bool enabled) = 0; - - // Called to flush any currently buffered output. - // - // A writer may ignore flush requests if it doesn't fit with its - // semantics (but must still return true in that case). - virtual bool DoFlush() = 0; - - // Called when a log output is to be rotated. Most directly this only - // applies to writers writing into files, which should then close the - // current file and open a new one. However, a writer may also - // trigger other apppropiate actions if semantics are similar. - // - // Once rotation has finished, the implementation should call - // RotationDone() to signal the log manager that potential - // postprocessors can now run. - // - // "rotate_path" reflects the path to where the rotated output is to - // be moved, with specifics depending on the writer. It should - // generally be interpreted in a way consistent with that of "path" - // as passed into DoInit(). As an example, for file-based output, - // "rotate_path" could be the original filename extended with a - // timestamp indicating the time of the rotation. - // - // "open" and "close" are the network time's when the *current* file - // was opened and closed, respectively. - // - // "terminating" indicated whether the rotation request occurs due - // the main Bro prcoess terminating (and not because we've reach a - // regularly scheduled time for rotation). - // - // A writer may ignore rotation requests if it doesn't fit with its - // semantics (but must still return true in that case). - virtual bool DoRotate(string rotated_path, double open, double close, - bool terminating) = 0; - - // Called once on termination. Not called when any of the other - // methods has previously signaled an error, i.e., executing this - // method signals a regular shutdown of the writer. - virtual void DoFinish() = 0; - - //// Methods for writers to use. These are thread-safe. - - // A thread-safe version of fmt(). - const char* Fmt(const char* format, ...); - - // Returns the current buffering state. - bool IsBuf() { return buffering; } - - // Reports an error to the user. - void Error(const char *msg); - - // Signals to the log manager that a file has been rotated. - // - // new_name: The filename of the rotated file. old_name: The filename - // of the origina file. - // - // open/close: The timestamps when the original file was opened and - // closed, respectively. - // - // terminating: True if rotation request occured due to the main Bro - // process shutting down. - bool FinishedRotation(string new_name, string old_name, double open, - double close, bool terminating); - -private: - friend class LogMgr; - - // When an error occurs, we call this method to set a flag marking - // the writer as disabled. The LogMgr will check the flag later and - // remove the writer. - bool Disabled() { return disabled; } - - // Deletes the values as passed into Write(). - void DeleteVals(LogVal** vals); - - string path; - int num_fields; - const LogField* const * fields; - bool buffering; - bool disabled; - - // For implementing Fmt(). - char* buf; - unsigned int buf_len; -}; - -#endif diff --git a/src/LogWriterNone.h b/src/LogWriterNone.h deleted file mode 100644 index 3811a19469..0000000000 --- a/src/LogWriterNone.h +++ /dev/null @@ -1,30 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. -// -// Dummy log writer that just discards everything (but still pretends to rotate). - -#ifndef LOGWRITERNONE_H -#define LOGWRITERNONE_H - -#include "LogWriter.h" - -class LogWriterNone : public LogWriter { -public: - LogWriterNone() {} - ~LogWriterNone() {}; - - static LogWriter* Instantiate() { return new LogWriterNone; } - -protected: - virtual bool DoInit(string path, int num_fields, - const LogField* const * fields) { return true; } - - virtual bool DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals) { return true; } - virtual bool DoSetBuf(bool enabled) { return true; } - virtual bool DoRotate(string rotated_path, double open, double close, - bool terminating); - virtual bool DoFlush() { return true; } - virtual void DoFinish() {} -}; - -#endif diff --git a/src/Login.cc b/src/Login.cc index 56efd12f53..e626fb3a0a 100644 --- a/src/Login.cc +++ b/src/Login.cc @@ -38,7 +38,7 @@ Login_Analyzer::Login_Analyzer(AnalyzerTag::Tag tag, Connection* conn) if ( ! re_skip_authentication ) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif re_skip_authentication = init_RE(skip_authentication); diff --git a/src/Net.cc b/src/Net.cc index 35c3b383f6..5bfae2275b 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -455,6 +455,7 @@ void net_run() // date on timers and events. network_time = ct; expire_timers(); + usleep(1); // Just yield. } } @@ -484,6 +485,8 @@ void net_run() // since Bro timers are not high-precision anyway.) if ( ! using_communication ) usleep(100000); + else + usleep(1000); // Flawfinder says about usleep: // diff --git a/src/PersistenceSerializer.cc b/src/PersistenceSerializer.cc index c757467f90..d9baad05bb 100644 --- a/src/PersistenceSerializer.cc +++ b/src/PersistenceSerializer.cc @@ -137,7 +137,7 @@ bool PersistenceSerializer::CheckForFile(UnserialInfo* info, const char* file, bool PersistenceSerializer::ReadAll(bool is_init, bool delete_files) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 06c04582ed..61be8a9e8f 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -183,8 +183,9 @@ #include "Sessions.h" #include "File.h" #include "Conn.h" -#include "LogMgr.h" #include "Reporter.h" +#include "threading/SerialTypes.h" +#include "logging/Manager.h" #include "IPAddr.h" #include "bro_inet_ntop.h" @@ -532,6 +533,7 @@ RemoteSerializer::RemoteSerializer() terminating = false; in_sync = 0; last_flush = 0; + received_logs = 0; } RemoteSerializer::~RemoteSerializer() @@ -681,7 +683,7 @@ RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip, if ( ! initialized ) reporter->InternalError("remote serializer not initialized"); - if ( ip.GetFamily() == IPAddr::IPv6 ) + if ( ip.GetFamily() == IPv6 ) Error("inter-Bro communication not supported over IPv6"); const uint32* bytes; @@ -1238,7 +1240,7 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl) if ( ! initialized ) reporter->InternalError("remote serializer not initialized"); - if ( ip.GetFamily() == IPAddr::IPv6 ) + if ( ip.GetFamily() == IPv6 ) Error("inter-Bro communication not supported over IPv6"); const uint32* bytes; @@ -1353,6 +1355,14 @@ double RemoteSerializer::NextTimestamp(double* local_network_time) { Poll(false); + if ( received_logs > 0 ) + { + // If we processed logs last time, assume there's more. + idle = false; + received_logs = 0; + return timer_mgr->Time(); + } + double et = events.length() ? events[0]->time : -1; double pt = packets.length() ? packets[0]->time : -1; @@ -2470,7 +2480,7 @@ bool RemoteSerializer::ProcessRemotePrint() return true; } -bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields) +bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields) { loop_over_list(peers, i) { @@ -2480,7 +2490,7 @@ bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string return true; } -bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields) +bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields) { SetErrorDescr("logging"); @@ -2493,6 +2503,9 @@ bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) return false; + if ( ! peer->logs_requested ) + return false; + BinarySerializationFormat fmt; fmt.StartWrite(); @@ -2534,7 +2547,7 @@ error: return false; } -bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals) +bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals) { loop_over_list(peers, i) { @@ -2544,7 +2557,7 @@ bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, i return true; } -bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals) +bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals) { if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) return false; @@ -2552,7 +2565,9 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st if ( ! peer->logs_requested ) return false; - assert(peer->log_buffer); + if ( ! peer->log_buffer ) + // Peer shutting down. + return false; // Serialize the log record entry. @@ -2587,7 +2602,10 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) || (network_time - last_flush > 1.0) ) { if ( ! FlushLogBuffer(peer) ) + { + delete [] data; return false; + } } // If the data is actually larger than our complete buffer, just send it out. @@ -2610,6 +2628,9 @@ error: bool RemoteSerializer::FlushLogBuffer(Peer* p) { + if ( ! p->logs_requested ) + return false; + last_flush = network_time; if ( p->state == Peer::CLOSING ) @@ -2631,11 +2652,17 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( current_peer->state == Peer::CLOSING ) return false; +#ifdef USE_PERFTOOLS_DEBUG + // Don't track allocations here, they'll be released only after the + // main loop exists. And it's just a tiny amount anyway. + HeapLeakChecker::Disabler disabler; +#endif + assert(current_args); EnumVal* id_val = 0; EnumVal* writer_val = 0; - LogField** fields = 0; + threading::Field** fields = 0; BinarySerializationFormat fmt; fmt.StartRead(current_args->data, current_args->len); @@ -2652,11 +2679,11 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( ! success ) goto error; - fields = new LogField* [num_fields]; + fields = new threading::Field* [num_fields]; for ( int i = 0; i < num_fields; i++ ) { - fields[i] = new LogField; + fields[i] = new threading::Field; if ( ! fields[i]->Read(&fmt) ) goto error; } @@ -2666,7 +2693,7 @@ bool RemoteSerializer::ProcessLogCreateWriter() id_val = new EnumVal(id, BifType::Enum::Log::ID); writer_val = new EnumVal(writer, BifType::Enum::Log::Writer); - if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields) ) + if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields, true, false) ) goto error; Unref(id_val); @@ -2697,7 +2724,7 @@ bool RemoteSerializer::ProcessLogWrite() // Unserialize one entry. EnumVal* id_val = 0; EnumVal* writer_val = 0; - LogVal** vals = 0; + threading::Value** vals = 0; int id, writer; string path; @@ -2711,11 +2738,11 @@ bool RemoteSerializer::ProcessLogWrite() if ( ! success ) goto error; - vals = new LogVal* [num_fields]; + vals = new threading::Value* [num_fields]; for ( int i = 0; i < num_fields; i++ ) { - vals[i] = new LogVal; + vals[i] = new threading::Value; if ( ! vals[i]->Read(&fmt) ) goto error; } @@ -2735,6 +2762,8 @@ bool RemoteSerializer::ProcessLogWrite() fmt.EndRead(); + ++received_logs; + return true; error: @@ -2844,7 +2873,7 @@ void RemoteSerializer::GotID(ID* id, Val* val) (desc && *desc) ? desc : "not set"), current_peer); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG // May still be cached, but we don't care. heap_checker->IgnoreObject(id); #endif @@ -3376,6 +3405,11 @@ void SocketComm::Run() small_timeout.tv_usec = io->CanWrite() || io->CanRead() ? 1 : 10; +#if 0 + if ( ! io->CanWrite() ) + usleep(10); +#endif + int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except, &small_timeout); diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index 3c3e103c75..05d25ca525 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -14,8 +14,11 @@ // FIXME: Change this to network byte order class IncrementalSendTimer; -class LogField; -class LogVal; + +namespace threading { + class Field; + class Value; +} // This class handles the communication done in Bro's main loop. class RemoteSerializer : public Serializer, public IOSource { @@ -99,13 +102,13 @@ public: bool SendPrintHookEvent(BroFile* f, const char* txt, size_t len); // Send a request to create a writer on a remote side. - bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields); + bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields); // Broadcasts a request to create a writer. - bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields); + bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields); // Broadcast a log entry to everybody interested. - bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals); + bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals); // Synchronzizes time with all connected peers. Returns number of // current sync-point, or -1 on error. @@ -300,7 +303,7 @@ protected: bool SendID(SerialInfo* info, Peer* peer, const ID& id); bool SendCapabilities(Peer* peer); bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p); - bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals); + bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals); void UnregisterHandlers(Peer* peer); void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0); @@ -335,6 +338,7 @@ private: int propagate_accesses; bool ignore_accesses; bool terminating; + int received_logs; Peer* source_peer; PeerID id_counter; // Keeps track of assigned IDs. uint32 current_sync_point; diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index a90bc83293..c9cf1f5c11 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -191,7 +191,7 @@ void RuleMatcher::Delete(RuleHdrTest* node) bool RuleMatcher::ReadFiles(const name_list& files) { -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; #endif @@ -1082,7 +1082,7 @@ static bool val_to_maskedval(Val* v, maskedvalue_list* append_to) bool is_v4_mask = m[0] == 0xffffffff && m[1] == m[0] && m[2] == m[0]; - if ( v->AsSubNet().Prefix().GetFamily() == IPAddr::IPv4 && + if ( v->AsSubNet().Prefix().GetFamily() == IPv4 && is_v4_mask ) { mval->val = ntohl(*n); diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index af3d9b44c2..10dd4f29ea 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -250,9 +250,9 @@ bool BinarySerializationFormat::Read(IPAddr* addr, const char* tag) } if ( n == 1 ) - *addr = IPAddr(IPAddr::IPv4, raw, IPAddr::Network); + *addr = IPAddr(IPv4, raw, IPAddr::Network); else - *addr = IPAddr(IPAddr::IPv6, raw, IPAddr::Network); + *addr = IPAddr(IPv6, raw, IPAddr::Network); return true; } @@ -269,6 +269,32 @@ bool BinarySerializationFormat::Read(IPPrefix* prefix, const char* tag) return true; } +bool BinarySerializationFormat::Read(struct in_addr* addr, const char* tag) + { + uint32_t* bytes = (uint32_t*) &addr->s_addr; + + if ( ! Read(&bytes[0], "addr4") ) + return false; + + bytes[0] = htonl(bytes[0]); + return true; + } + +bool BinarySerializationFormat::Read(struct in6_addr* addr, const char* tag) + { + uint32_t* bytes = (uint32_t*) &addr->s6_addr; + + for ( int i = 0; i < 4; ++i ) + { + if ( ! Read(&bytes[i], "addr6-part") ) + return false; + + bytes[i] = htonl(bytes[i]); + } + + return true; + } + bool BinarySerializationFormat::Write(char v, const char* tag) { DBG_LOG(DBG_SERIAL, "Write char %s [%s]", fmt_bytes(&v, 1), tag); @@ -362,6 +388,29 @@ bool BinarySerializationFormat::Write(const IPPrefix& prefix, const char* tag) return Write(prefix.Prefix(), "prefix") && Write(prefix.Length(), "width"); } +bool BinarySerializationFormat::Write(const struct in_addr& addr, const char* tag) + { + const uint32_t* bytes = (uint32_t*) &addr.s_addr; + + if ( ! Write(ntohl(bytes[0]), "addr4") ) + return false; + + return true; + } + +bool BinarySerializationFormat::Write(const struct in6_addr& addr, const char* tag) + { + const uint32_t* bytes = (uint32_t*) &addr.s6_addr; + + for ( int i = 0; i < 4; ++i ) + { + if ( ! Write(ntohl(bytes[i]), "addr6-part") ) + return false; + } + + return true; + } + bool BinarySerializationFormat::WriteOpenTag(const char* tag) { return true; @@ -464,6 +513,18 @@ bool XMLSerializationFormat::Read(IPPrefix* prefix, const char* tag) return false; } +bool XMLSerializationFormat::Read(struct in_addr* addr, const char* tag) + { + reporter->InternalError("no reading of xml"); + return false; + } + +bool XMLSerializationFormat::Read(struct in6_addr* addr, const char* tag) + { + reporter->InternalError("no reading of xml"); + return false; + } + bool XMLSerializationFormat::Write(char v, const char* tag) { return WriteElem(tag, "char", &v, 1); @@ -556,6 +617,18 @@ bool XMLSerializationFormat::Write(const IPPrefix& prefix, const char* tag) return false; } +bool XMLSerializationFormat::Write(const struct in_addr& addr, const char* tag) + { + reporter->InternalError("XML output of in_addr not implemented"); + return false; + } + +bool XMLSerializationFormat::Write(const struct in6_addr& addr, const char* tag) + { + reporter->InternalError("XML output of in6_addr not implemented"); + return false; + } + bool XMLSerializationFormat::WriteEncodedString(const char* s, int len) { while ( len-- ) diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index f5eb77c608..f270b61bae 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -9,6 +9,9 @@ using namespace std; #include "util.h" +class IPAddr; +class IPPrefix; + // Abstract base class. class SerializationFormat { public: @@ -30,6 +33,8 @@ public: virtual bool Read(string* s, const char* tag) = 0; virtual bool Read(IPAddr* addr, const char* tag) = 0; virtual bool Read(IPPrefix* prefix, const char* tag) = 0; + virtual bool Read(struct in_addr* addr, const char* tag) = 0; + virtual bool Read(struct in6_addr* addr, const char* tag) = 0; // Returns number of raw bytes read since last call to StartRead(). int BytesRead() const { return bytes_read; } @@ -54,6 +59,8 @@ public: virtual bool Write(const string& s, const char* tag) = 0; virtual bool Write(const IPAddr& addr, const char* tag) = 0; virtual bool Write(const IPPrefix& prefix, const char* tag) = 0; + virtual bool Write(const struct in_addr& addr, const char* tag) = 0; + virtual bool Write(const struct in6_addr& addr, const char* tag) = 0; virtual bool WriteOpenTag(const char* tag) = 0; virtual bool WriteCloseTag(const char* tag) = 0; @@ -96,6 +103,8 @@ public: virtual bool Read(string* s, const char* tag); virtual bool Read(IPAddr* addr, const char* tag); virtual bool Read(IPPrefix* prefix, const char* tag); + virtual bool Read(struct in_addr* addr, const char* tag); + virtual bool Read(struct in6_addr* addr, const char* tag); virtual bool Write(int v, const char* tag); virtual bool Write(uint16 v, const char* tag); virtual bool Write(uint32 v, const char* tag); @@ -109,6 +118,8 @@ public: virtual bool Write(const string& s, const char* tag); virtual bool Write(const IPAddr& addr, const char* tag); virtual bool Write(const IPPrefix& prefix, const char* tag); + virtual bool Write(const struct in_addr& addr, const char* tag); + virtual bool Write(const struct in6_addr& addr, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -133,6 +144,8 @@ public: virtual bool Write(const string& s, const char* tag); virtual bool Write(const IPAddr& addr, const char* tag); virtual bool Write(const IPPrefix& prefix, const char* tag); + virtual bool Write(const struct in_addr& addr, const char* tag); + virtual bool Write(const struct in6_addr& addr, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -150,6 +163,8 @@ public: virtual bool Read(string* s, const char* tag); virtual bool Read(IPAddr* addr, const char* tag); virtual bool Read(IPPrefix* prefix, const char* tag); + virtual bool Read(struct in_addr* addr, const char* tag); + virtual bool Read(struct in6_addr* addr, const char* tag); private: // Encodes non-printable characters. diff --git a/src/Sessions.cc b/src/Sessions.cc index ad7e46589a..e8c3da0755 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -482,6 +482,22 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, return; } + // Stop analyzing IPv6 packets that use routing type 0 headers with segments + // left since RH0 headers are deprecated by RFC 5095 and we'd have to make + // extra effort to get the destination in the connection/flow endpoint right. + if ( ip_hdr->RH0SegLeft() ) + { + dump_this_packet = 1; + if ( rh0_segleft ) + { + val_list* vl = new val_list(); + vl->append(ip_hdr->BuildPktHdrVal()); + mgr.QueueEvent(rh0_segleft, vl); + } + Remove(f); + return; + } + int proto = ip_hdr->NextProto(); if ( CheckHeaderTrunc(proto, len, caplen, hdr, pkt) ) diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 7abef72c46..2d0a8dfc5a 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -678,7 +678,7 @@ bool StateAccess::DoUnserialize(UnserialInfo* info) target.id = new ID(name, SCOPE_GLOBAL, true); Ref(target.id); global_scope()->Insert(name, target.id); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG heap_checker->IgnoreObject(target.id); #endif } diff --git a/src/Stats.cc b/src/Stats.cc index c1abe0cdb0..c3035231e9 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -8,7 +8,7 @@ #include "cq.h" #include "DNS_Mgr.h" #include "Trigger.h" - +#include "threading/Manager.h" int killed_by_inactivity = 0; @@ -203,6 +203,25 @@ void ProfileLogger::Log() current_timers[i])); } + file->Write(fmt("%0.6f Threads: current=%d\n", network_time, thread_mgr->NumThreads())); + + const threading::Manager::msg_stats_list& thread_stats = thread_mgr->GetMsgThreadStats(); + for ( threading::Manager::msg_stats_list::const_iterator i = thread_stats.begin(); + i != thread_stats.end(); ++i ) + { + threading::MsgThread::Stats s = i->second; + file->Write(fmt("%0.6f %-25s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 + " (#queue r/w: in=%" PRIu64 "/%" PRIu64 " out=%" PRIu64 "/%" PRIu64 ")" + "\n", + network_time, + i->first.c_str(), + s.sent_in, s.sent_out, + s.pending_in, s.pending_out, + s.queue_in_stats.num_reads, s.queue_in_stats.num_writes, + s.queue_out_stats.num_reads, s.queue_out_stats.num_writes + )); + } + // Script-level state. unsigned int size, mem = 0; PDict(ID)* globals = global_scope()->Vars(); diff --git a/src/Val.cc b/src/Val.cc index db6e9eb23a..83bbc59b9d 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -606,7 +606,7 @@ ID* MutableVal::Bind() const ip = htonl(0x7f000001); // 127.0.0.1 safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#", - IPAddr(IPAddr::IPv4, &ip, IPAddr::Network)->AsString().c_str(), + IPAddr(IPv4, &ip, IPAddr::Network)->AsString().c_str(), getpid()); #else safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#", host, getpid()); @@ -864,12 +864,12 @@ AddrVal::AddrVal(const char* text) : Val(TYPE_ADDR) AddrVal::AddrVal(uint32 addr) : Val(TYPE_ADDR) { // ### perhaps do gethostbyaddr here? - val.addr_val = new IPAddr(IPAddr::IPv4, &addr, IPAddr::Network); + val.addr_val = new IPAddr(IPv4, &addr, IPAddr::Network); } AddrVal::AddrVal(const uint32 addr[4]) : Val(TYPE_ADDR) { - val.addr_val = new IPAddr(IPAddr::IPv6, addr, IPAddr::Network); + val.addr_val = new IPAddr(IPv6, addr, IPAddr::Network); } AddrVal::AddrVal(const IPAddr& addr) : Val(TYPE_ADDR) @@ -889,7 +889,7 @@ unsigned int AddrVal::MemoryAllocation() const Val* AddrVal::SizeVal() const { - if ( val.addr_val->GetFamily() == IPAddr::IPv4 ) + if ( val.addr_val->GetFamily() == IPv4 ) return new Val(32, TYPE_COUNT); else return new Val(128, TYPE_COUNT); @@ -933,13 +933,13 @@ SubNetVal::SubNetVal(const char* text, int width) : Val(TYPE_SUBNET) SubNetVal::SubNetVal(uint32 addr, int width) : Val(TYPE_SUBNET) { - IPAddr a(IPAddr::IPv4, &addr, IPAddr::Network); + IPAddr a(IPv4, &addr, IPAddr::Network); val.subnet_val = new IPPrefix(a, width); } SubNetVal::SubNetVal(const uint32* addr, int width) : Val(TYPE_SUBNET) { - IPAddr a(IPAddr::IPv6, addr, IPAddr::Network); + IPAddr a(IPv6, addr, IPAddr::Network); val.subnet_val = new IPPrefix(a, width); } @@ -953,6 +953,16 @@ SubNetVal::~SubNetVal() delete val.subnet_val; } +const IPAddr& SubNetVal::Prefix() const + { + return val.subnet_val->Prefix(); + } + +int SubNetVal::Width() const + { + return val.subnet_val->Length(); + } + unsigned int SubNetVal::MemoryAllocation() const { return padded_sizeof(*this) + val.subnet_val->MemoryAllocation(); @@ -978,7 +988,7 @@ IPAddr SubNetVal::Mask() const uint32 m[4]; for ( unsigned int i = 0; i < 4; ++i ) m[i] = 0; - IPAddr rval(IPAddr::IPv6, m, IPAddr::Host); + IPAddr rval(IPv6, m, IPAddr::Host); return rval; } @@ -994,7 +1004,7 @@ IPAddr SubNetVal::Mask() const while ( ++mp < m + 4 ) *mp = 0; - IPAddr rval(IPAddr::IPv6, m, IPAddr::Host); + IPAddr rval(IPv6, m, IPAddr::Host); return rval; } diff --git a/src/Val.h b/src/Val.h index 64c19b2d92..ef1c1257c8 100644 --- a/src/Val.h +++ b/src/Val.h @@ -514,10 +514,6 @@ protected: #define ICMP_PORT_MASK 0x30000 -typedef enum { - TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP, -} TransportProto; - class PortVal : public Val { public: // Constructors - both take the port number in host order. @@ -589,8 +585,8 @@ public: Val* SizeVal() const; - const IPAddr& Prefix() const { return val.subnet_val->Prefix(); } - int Width() const { return val.subnet_val->Length(); } + const IPAddr& Prefix() const; + int Width() const; IPAddr Mask() const; bool Contains(const IPAddr& addr) const; diff --git a/src/bro.bif b/src/bro.bif index 64ed7d1f2f..fa6766a7bf 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1949,7 +1949,7 @@ function is_local_interface%(ip: addr%) : bool if ( ent ) { for ( unsigned int len = 0; ent->h_addr_list[len]; ++len ) - addrs.push_back(IPAddr(IPAddr::IPv4, (uint32*)ent->h_addr_list[len], + addrs.push_back(IPAddr(IPv4, (uint32*)ent->h_addr_list[len], IPAddr::Network)); } @@ -1958,7 +1958,7 @@ function is_local_interface%(ip: addr%) : bool if ( ent ) { for ( unsigned int len = 0; ent->h_addr_list[len]; ++len ) - addrs.push_back(IPAddr(IPAddr::IPv6, (uint32*)ent->h_addr_list[len], + addrs.push_back(IPAddr(IPv6, (uint32*)ent->h_addr_list[len], IPAddr::Network)); } @@ -2024,7 +2024,7 @@ function gethostname%(%) : string ## Returns: true if *a* is an IPv4 address, else false. function is_v4_addr%(a: addr%): bool %{ - if ( a->AsAddr().GetFamily() == IPAddr::IPv4 ) + if ( a->AsAddr().GetFamily() == IPv4 ) return new Val(1, TYPE_BOOL); else return new Val(0, TYPE_BOOL); @@ -2037,7 +2037,7 @@ function is_v4_addr%(a: addr%): bool ## Returns: true if *a* is an IPv6 address, else false. function is_v6_addr%(a: addr%): bool %{ - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) return new Val(1, TYPE_BOOL); else return new Val(0, TYPE_BOOL); @@ -2050,18 +2050,15 @@ function is_v6_addr%(a: addr%): bool # =========================================================================== ## Converts the *data* field of :bro:type:`ip6_routing` records that have -## *rtype* of 0 into a set of addresses. +## *rtype* of 0 into a vector of addresses. ## ## s: The *data* field of an :bro:type:`ip6_routing` record that has ## an *rtype* of 0. ## -## Returns: The set of addresses contained in the routing header data. -function routing0_data_to_addrs%(s: string%): addr_set +## Returns: The vector of addresses contained in the routing header data. +function routing0_data_to_addrs%(s: string%): addr_vec %{ - BroType* index_type = base_type(TYPE_ADDR); - TypeList* set_index = new TypeList(index_type); - set_index->Append(index_type); - TableVal* tv = new TableVal(new SetType(set_index, 0)); + VectorVal* rval = new VectorVal(new VectorType(base_type(TYPE_ADDR))); int len = s->Len(); const u_char* bytes = s->Bytes(); @@ -2073,13 +2070,13 @@ function routing0_data_to_addrs%(s: string%): addr_set while ( len > 0 ) { - IPAddr a(IPAddr::IPv6, (const uint32*) bytes, IPAddr::Network); - tv->Assign(new AddrVal(a), 0); + IPAddr a(IPv6, (const uint32*) bytes, IPAddr::Network); + rval->Assign(rval->Size(), new AddrVal(a), 0); bytes += 16; len -= 16; } - return tv; + return rval; %} ## Converts a :bro:type:`addr` to a :bro:type:`index_vec`. @@ -3555,7 +3552,7 @@ function lookup_location%(a: addr%) : geo_location } #ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( geoip_v6 && a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( geoip_v6 && a->AsAddr().GetFamily() == IPv6 ) { geoipv6_t ga; a->AsAddr().CopyIPv6(&ga); @@ -3567,7 +3564,7 @@ function lookup_location%(a: addr%) : geo_location else #endif - if ( geoip && a->AsAddr().GetFamily() == IPAddr::IPv4 ) + if ( geoip && a->AsAddr().GetFamily() == IPv4 ) { const uint32* bytes; a->AsAddr().GetBytes(&bytes); @@ -3650,7 +3647,7 @@ function lookup_asn%(a: addr%) : count { // IPv6 support showed up in 1.4.5. #ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) { geoipv6_t ga; a->AsAddr().CopyIPv6(&ga); @@ -3659,7 +3656,7 @@ function lookup_asn%(a: addr%) : count else #endif - if ( a->AsAddr().GetFamily() == IPAddr::IPv4 ) + if ( a->AsAddr().GetFamily() == IPv4 ) { const uint32* bytes; a->AsAddr().GetBytes(&bytes); @@ -5386,7 +5383,7 @@ function preserve_prefix%(a: addr, width: count%): any AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; if ( ip_anon ) { - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) builtin_error("preserve_prefix() not supported for IPv6 addresses"); else { @@ -5415,7 +5412,7 @@ function preserve_subnet%(a: subnet%): any AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; if ( ip_anon ) { - if ( a->AsSubNet().Prefix().GetFamily() == IPAddr::IPv6 ) + if ( a->AsSubNet().Prefix().GetFamily() == IPv6 ) builtin_error("preserve_subnet() not supported for IPv6 addresses"); else { @@ -5451,7 +5448,7 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr if ( anon_class < 0 || anon_class >= NUM_ADDR_ANONYMIZATION_CLASSES ) builtin_error("anonymize_addr(): invalid ip addr anonymization class"); - if ( a->AsAddr().GetFamily() == IPAddr::IPv6 ) + if ( a->AsAddr().GetFamily() == IPv6 ) { builtin_error("anonymize_addr() not supported for IPv6 addresses"); return 0; diff --git a/src/event.bif b/src/event.bif index 8e7b0be8a8..afe5073649 100644 --- a/src/event.bif +++ b/src/event.bif @@ -478,6 +478,14 @@ event ipv6_ext_headers%(c: connection, p: pkt_hdr%); ## .. bro:see:: new_packet tcp_packet ipv6_ext_headers event esp_packet%(p: pkt_hdr%); +## Generated for any packets using an IPv6 Routing Type 0 extension header +## with non-zero segments left. +## +## p: Information from the header of the packet that triggered the event. +## +## .. bro:see:: new_packet tcp_packet ipv6_ext_headers +event rh0_segleft%(p: pkt_hdr%); + ## Generated for every packet that has non-empty transport-layer payload. This is a ## very low-level and expensive event that should be avoided when at all possible. ## It's usually infeasible to handle when processing even medium volumes of diff --git a/src/logging.bif b/src/logging.bif index 31e1bebacd..c8960b4e38 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -3,8 +3,9 @@ module Log; %%{ -#include "LogMgr.h" #include "NetVar.h" + +#include "logging/Manager.h" %%} type Filter: record; diff --git a/src/LogMgr.cc b/src/logging/Manager.cc similarity index 69% rename from src/LogMgr.cc rename to src/logging/Manager.cc index bebce97352..9437f0099f 100644 --- a/src/LogMgr.cc +++ b/src/logging/Manager.cc @@ -2,33 +2,42 @@ #include -#include "LogMgr.h" -#include "Event.h" -#include "EventHandler.h" -#include "NetVar.h" -#include "Net.h" +#include "../Event.h" +#include "../EventHandler.h" +#include "../NetVar.h" +#include "../Net.h" -#include "LogWriterAscii.h" -#include "LogWriterNone.h" +#include "threading/SerialTypes.h" + +#include "Manager.h" +#include "WriterFrontend.h" +#include "WriterBackend.h" + +#include "writers/Ascii.h" +#include "writers/None.h" + +using namespace logging; +using threading::Value; +using threading::Field; // Structure describing a log writer type. -struct LogWriterDefinition { +struct WriterDefinition { bro_int_t type; // The type. const char *name; // Descriptive name for error messages. bool (*init)(); // An optional one-time initialization function. - LogWriter* (*factory)(); // A factory function creating instances. + WriterBackend* (*factory)(WriterFrontend* frontend); // A factory function creating instances. }; // Static table defining all availabel log writers. -LogWriterDefinition log_writers[] = { - { BifEnum::Log::WRITER_NONE, "None", 0, LogWriterNone::Instantiate }, - { BifEnum::Log::WRITER_ASCII, "Ascii", 0, LogWriterAscii::Instantiate }, +WriterDefinition log_writers[] = { + { BifEnum::Log::WRITER_NONE, "None", 0, writer::None::Instantiate }, + { BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate }, // End marker, don't touch. - { BifEnum::Log::WRITER_DEFAULT, "None", 0, (LogWriter* (*)())0 } + { BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)(WriterFrontend* frontend))0 } }; -struct LogMgr::Filter { +struct Manager::Filter { string name; EnumVal* id; Func* pred; @@ -42,7 +51,7 @@ struct LogMgr::Filter { Func* postprocessor; int num_fields; - LogField** fields; + Field** fields; // Vector indexed by field number. Each element is a list of record // indices defining a path leading to the value across potential @@ -52,16 +61,16 @@ struct LogMgr::Filter { ~Filter(); }; -struct LogMgr::WriterInfo { +struct Manager::WriterInfo { EnumVal* type; double open_time; Timer* rotation_timer; double interval; Func* postprocessor; - LogWriter* writer; + WriterFrontend* writer; }; -struct LogMgr::Stream { +struct Manager::Stream { EnumVal* id; bool enabled; string name; @@ -78,275 +87,7 @@ struct LogMgr::Stream { ~Stream(); }; -bool LogField::Read(SerializationFormat* fmt) - { - int t; - int st; - - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); - type = (TypeTag) t; - subtype = (TypeTag) st; - - return success; - } - -bool LogField::Write(SerializationFormat* fmt) const - { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); - } - -LogVal::~LogVal() - { - if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC) - && present ) - delete val.string_val; - - if ( type == TYPE_ADDR && present ) - delete val.addr_val; - - if ( type == TYPE_SUBNET && present ) - delete val.subnet_val; - - if ( type == TYPE_TABLE && present ) - { - for ( int i = 0; i < val.set_val.size; i++ ) - delete val.set_val.vals[i]; - - delete [] val.set_val.vals; - } - - if ( type == TYPE_VECTOR && present ) - { - for ( int i = 0; i < val.vector_val.size; i++ ) - delete val.vector_val.vals[i]; - - delete [] val.vector_val.vals; - } - } - -bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) - { - if ( ! t ) - return false; - - switch ( t->Tag() ) { - case TYPE_BOOL: - case TYPE_INT: - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - case TYPE_SUBNET: - case TYPE_ADDR: - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - return true; - - case TYPE_RECORD: - return ! atomic_only; - - case TYPE_TABLE: - { - if ( atomic_only ) - return false; - - if ( ! t->IsSet() ) - return false; - - return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); - } - - case TYPE_VECTOR: - { - if ( atomic_only ) - return false; - - return IsCompatibleType(t->AsVectorType()->YieldType(), true); - } - - default: - return false; - } - - return false; - } - -bool LogVal::Read(SerializationFormat* fmt) - { - int ty; - - if ( ! (fmt->Read(&ty, "type") && fmt->Read(&present, "present")) ) - return false; - - type = (TypeTag)(ty); - - if ( ! present ) - return true; - - switch ( type ) { - case TYPE_BOOL: - case TYPE_INT: - return fmt->Read(&val.int_val, "int"); - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - return fmt->Read(&val.uint_val, "uint"); - - case TYPE_SUBNET: - { - IPPrefix prefix; - if ( ! fmt->Read(&prefix, "subnet") ) - return false; - - val.subnet_val = new IPPrefix(prefix); - return true; - } - - case TYPE_ADDR: - { - IPAddr addr; - if ( ! fmt->Read(&addr, "net") ) - return false; - - val.addr_val = new IPAddr(addr); - return true; - } - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - return fmt->Read(&val.double_val, "double"); - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - { - val.string_val = new string; - return fmt->Read(val.string_val, "string"); - } - - case TYPE_TABLE: - { - if ( ! fmt->Read(&val.set_val.size, "set_size") ) - return false; - - val.set_val.vals = new LogVal* [val.set_val.size]; - - for ( int i = 0; i < val.set_val.size; ++i ) - { - val.set_val.vals[i] = new LogVal; - - if ( ! val.set_val.vals[i]->Read(fmt) ) - return false; - } - - return true; - } - - case TYPE_VECTOR: - { - if ( ! fmt->Read(&val.vector_val.size, "vector_size") ) - return false; - - val.vector_val.vals = new LogVal* [val.vector_val.size]; - - for ( int i = 0; i < val.vector_val.size; ++i ) - { - val.vector_val.vals[i] = new LogVal; - - if ( ! val.vector_val.vals[i]->Read(fmt) ) - return false; - } - - return true; - } - - default: - reporter->InternalError("unsupported type %s in LogVal::Write", type_name(type)); - } - - return false; - } - -bool LogVal::Write(SerializationFormat* fmt) const - { - if ( ! (fmt->Write((int)type, "type") && - fmt->Write(present, "present")) ) - return false; - - if ( ! present ) - return true; - - switch ( type ) { - case TYPE_BOOL: - case TYPE_INT: - return fmt->Write(val.int_val, "int"); - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - return fmt->Write(val.uint_val, "uint"); - - case TYPE_SUBNET: - return fmt->Write(*val.subnet_val, "subnet"); - - - case TYPE_ADDR: - return fmt->Write(*val.addr_val, "addr"); - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - return fmt->Write(val.double_val, "double"); - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - return fmt->Write(*val.string_val, "string"); - - case TYPE_TABLE: - { - if ( ! fmt->Write(val.set_val.size, "set_size") ) - return false; - - for ( int i = 0; i < val.set_val.size; ++i ) - { - if ( ! val.set_val.vals[i]->Write(fmt) ) - return false; - } - - return true; - } - - case TYPE_VECTOR: - { - if ( ! fmt->Write(val.vector_val.size, "vector_size") ) - return false; - - for ( int i = 0; i < val.vector_val.size; ++i ) - { - if ( ! val.vector_val.vals[i]->Write(fmt) ) - return false; - } - - return true; - } - - default: - reporter->InternalError("unsupported type %s in LogVal::REad", type_name(type)); - } - - return false; - } - -LogMgr::Filter::~Filter() +Manager::Filter::~Filter() { for ( int i = 0; i < num_fields; ++i ) delete fields[i]; @@ -356,7 +97,7 @@ LogMgr::Filter::~Filter() Unref(path_val); } -LogMgr::Stream::~Stream() +Manager::Stream::~Stream() { Unref(columns); @@ -364,9 +105,6 @@ LogMgr::Stream::~Stream() { WriterInfo* winfo = i->second; - if ( ! winfo ) - continue; - if ( winfo->rotation_timer ) timer_mgr->Cancel(winfo->rotation_timer); @@ -379,17 +117,73 @@ LogMgr::Stream::~Stream() delete *f; } -LogMgr::LogMgr() +Manager::Manager() { } -LogMgr::~LogMgr() +Manager::~Manager() { for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) delete *s; } -LogMgr::Stream* LogMgr::FindStream(EnumVal* id) +WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) + { + WriterDefinition* ld = log_writers; + + while ( true ) + { + if ( ld->type == BifEnum::Log::WRITER_DEFAULT ) + { + reporter->Error("unknow writer when creating writer"); + return 0; + } + + if ( ld->type != type ) + { + // Not the right one. + ++ld; + continue; + } + + // If the writer has an init function, call it. + if ( ld->init ) + { + if ( (*ld->init)() ) + // Clear the init function so that we won't + // call it again later. + ld->init = 0; + else + { + // Init failed, disable by deleting factory + // function. + ld->factory = 0; + + DBG_LOG(DBG_LOGGING, "failed to init writer class %s", + ld->name); + + return false; + } + } + + if ( ! ld->factory ) + // Oops, we can't instantiate this guy. + return 0; + + // All done. + break; + } + + assert(ld->factory); + + WriterBackend* backend = (*ld->factory)(frontend); + assert(backend); + + frontend->ty_name = ld->name; + return backend; + } + +Manager::Stream* Manager::FindStream(EnumVal* id) { unsigned int idx = id->AsEnum(); @@ -399,7 +193,7 @@ LogMgr::Stream* LogMgr::FindStream(EnumVal* id) return streams[idx]; } -LogMgr::WriterInfo* LogMgr::FindWriter(LogWriter* writer) +Manager::WriterInfo* Manager::FindWriter(WriterFrontend* writer) { for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) { @@ -410,7 +204,7 @@ LogMgr::WriterInfo* LogMgr::FindWriter(LogWriter* writer) { WriterInfo* winfo = i->second; - if ( winfo && winfo->writer == writer ) + if ( winfo->writer == writer ) return winfo; } } @@ -418,14 +212,15 @@ LogMgr::WriterInfo* LogMgr::FindWriter(LogWriter* writer) return 0; } -void LogMgr::RemoveDisabledWriters(Stream* stream) +void Manager::RemoveDisabledWriters(Stream* stream) { list disabled; for ( Stream::WriterMap::iterator j = stream->writers.begin(); j != stream->writers.end(); j++ ) { - if ( j->second && j->second->writer->Disabled() ) + if ( j->second->writer->Disabled() ) { + j->second->writer->Stop(); delete j->second; disabled.push_back(j->first); } @@ -435,7 +230,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) +bool Manager::CreateStream(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); @@ -455,7 +250,7 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) if ( ! (columns->FieldDecl(i)->FindAttr(ATTR_LOG)) ) continue; - if ( ! LogVal::IsCompatibleType(columns->FieldType(i)) ) + if ( ! threading::Value::IsCompatibleType(columns->FieldType(i)) ) { reporter->Error("type of field '%s' is not support for logging output", columns->FieldName(i)); @@ -527,7 +322,7 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) return true; } -bool LogMgr::EnableStream(EnumVal* id) +bool Manager::EnableStream(EnumVal* id) { Stream* stream = FindStream(id); @@ -543,7 +338,7 @@ bool LogMgr::EnableStream(EnumVal* id) return true; } -bool LogMgr::DisableStream(EnumVal* id) +bool Manager::DisableStream(EnumVal* id) { Stream* stream = FindStream(id); @@ -560,7 +355,7 @@ bool LogMgr::DisableStream(EnumVal* id) } // Helper for recursive record field unrolling. -bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, +bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices) { for ( int i = 0; i < rt->NumFields(); ++i ) @@ -654,9 +449,9 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, filter->indices.push_back(new_indices); - filter->fields = (LogField**) + filter->fields = (Field**) realloc(filter->fields, - sizeof(LogField) * ++filter->num_fields); + sizeof(Field) * ++filter->num_fields); if ( ! filter->fields ) { @@ -664,24 +459,23 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, return false; } - LogField* field = new LogField(); + Field* field = new Field(); field->name = new_path; field->type = t->Tag(); - if ( field->type == TYPE_TABLE ) - { - field->subtype = t->AsSetType()->Indices()->PureType()->Tag(); - } - else if ( field->type == TYPE_VECTOR ) - { - field->subtype = t->AsVectorType()->YieldType()->Tag(); - } + + if ( field->type == TYPE_TABLE ) + field->subtype = t->AsSetType()->Indices()->PureType()->Tag(); + + else if ( field->type == TYPE_VECTOR ) + field->subtype = t->AsVectorType()->YieldType()->Tag(); + filter->fields[filter->num_fields - 1] = field; } return true; } -bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) +bool Manager::AddFilter(EnumVal* id, RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); @@ -777,7 +571,7 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) for ( int i = 0; i < filter->num_fields; i++ ) { - LogField* field = filter->fields[i]; + Field* field = filter->fields[i]; DBG_LOG(DBG_LOGGING, " field %10s: %s", field->name.c_str(), type_name(field->type)); } @@ -786,12 +580,12 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) return true; } -bool LogMgr::RemoveFilter(EnumVal* id, StringVal* name) +bool Manager::RemoveFilter(EnumVal* id, StringVal* name) { return RemoveFilter(id, name->AsString()->CheckString()); } -bool LogMgr::RemoveFilter(EnumVal* id, string name) +bool Manager::RemoveFilter(EnumVal* id, string name) { Stream* stream = FindStream(id); if ( ! stream ) @@ -818,7 +612,7 @@ bool LogMgr::RemoveFilter(EnumVal* id, string name) return true; } -bool LogMgr::Write(EnumVal* id, RecordVal* columns) +bool Manager::Write(EnumVal* id, RecordVal* columns) { bool error = false; @@ -882,11 +676,11 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) Val* path_arg; if ( filter->path_val ) - path_arg = filter->path_val; + path_arg = filter->path_val->Ref(); else path_arg = new StringVal(""); - vl.append(path_arg->Ref()); + vl.append(path_arg); Val* rec_arg; BroType* rt = filter->path_func->FType()->Args()->FieldType("rec"); @@ -911,7 +705,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return false; } - if ( v->Type()->Tag() != TYPE_STRING ) + if ( ! v->Type()->Tag() == TYPE_STRING ) { reporter->Error("path_func did not return string"); Unref(v); @@ -920,7 +714,6 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) if ( ! filter->path_val ) { - Unref(path_arg); filter->path = v->AsString()->CheckString(); filter->path_val = v->Ref(); } @@ -938,83 +731,42 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(filter->writer->AsEnum(), path)); - LogWriter* writer = 0; + WriterFrontend* writer = 0; if ( w != stream->writers.end() ) // We know this writer already. - writer = w->second ? w->second->writer : 0; + writer = w->second->writer; else { // No, need to create one. - // Copy the fields for LogWriter::Init() as it will take - // ownership. - LogField** arg_fields = new LogField*[filter->num_fields]; + // Copy the fields for WriterFrontend::Init() as it + // will take ownership. + Field** arg_fields = new Field*[filter->num_fields]; for ( int j = 0; j < filter->num_fields; ++j ) - arg_fields[j] = new LogField(*filter->fields[j]); + arg_fields[j] = new Field(*filter->fields[j]); - if ( filter->remote ) - remote_serializer->SendLogCreateWriter(stream->id, - filter->writer, - path, - filter->num_fields, - arg_fields); + writer = CreateWriter(stream->id, filter->writer, + path, filter->num_fields, + arg_fields, filter->local, filter->remote); - if ( filter->local ) + if ( ! writer ) { - writer = CreateWriter(stream->id, filter->writer, - path, filter->num_fields, - arg_fields); - - if ( ! writer ) - { - Unref(columns); - return false; - } + Unref(columns); + return false; } - else - { - // Insert a null pointer into the map to make - // sure we don't try creating it again. - stream->writers.insert(Stream::WriterMap::value_type( - Stream::WriterPathPair(filter->writer->AsEnum(), path), 0)); - for( int i = 0; i < filter->num_fields; ++i) - delete arg_fields[i]; - - delete [] arg_fields; - } } // Alright, can do the write now. - if ( filter->local || filter->remote ) - { - LogVal** vals = RecordToFilterVals(stream, filter, columns); - - if ( filter->remote ) - remote_serializer->SendLogWrite(stream->id, - filter->writer, - path, - filter->num_fields, - vals); - - if ( filter->local ) - { - assert(writer); - - // Write takes ownership of vals. - if ( ! writer->Write(filter->num_fields, vals) ) - error = true; - } - - else - DeleteVals(filter->num_fields, vals); - - } + threading::Value** vals = RecordToFilterVals(stream, filter, columns); + // Write takes ownership of vals. + assert(writer); + writer->Write(filter->num_fields, vals); #ifdef DEBUG DBG_LOG(DBG_LOGGING, "Wrote record to filter '%s' on stream '%s'", @@ -1030,15 +782,15 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } -LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) +threading::Value* Manager::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) ty = val->Type(); if ( ! val ) - return new LogVal(ty->Tag(), false); + return new threading::Value(ty->Tag(), false); - LogVal* lval = new LogVal(ty->Tag()); + threading::Value* lval = new threading::Value(ty->Tag()); switch ( lval->type ) { case TYPE_BOOL: @@ -1061,18 +813,17 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; case TYPE_PORT: - lval->val.uint_val = val->AsPortVal()->Port(); + lval->val.port_val.port = val->AsPortVal()->Port(); + lval->val.port_val.proto = val->AsPortVal()->PortType(); break; case TYPE_SUBNET: - lval->val.subnet_val = new IPPrefix(val->AsSubNet()); + val->AsSubNet().ConvertToThreadingValue(&lval->val.subnet_val); break; case TYPE_ADDR: - { - lval->val.addr_val = new IPAddr(val->AsAddr()); + val->AsAddr().ConvertToThreadingValue(&lval->val.addr_val); break; - } case TYPE_DOUBLE: case TYPE_TIME: @@ -1113,7 +864,7 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) set = new ListVal(TYPE_INT); lval->val.set_val.size = set->Length(); - lval->val.set_val.vals = new LogVal* [lval->val.set_val.size]; + lval->val.set_val.vals = new threading::Value* [lval->val.set_val.size]; for ( int i = 0; i < lval->val.set_val.size; i++ ) lval->val.set_val.vals[i] = ValToLogVal(set->Index(i)); @@ -1127,7 +878,7 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) VectorVal* vec = val->AsVectorVal(); lval->val.vector_val.size = vec->Size(); lval->val.vector_val.vals = - new LogVal* [lval->val.vector_val.size]; + new threading::Value* [lval->val.vector_val.size]; for ( int i = 0; i < lval->val.vector_val.size; i++ ) { @@ -1146,10 +897,10 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) return lval; } -LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, +Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns) { - LogVal** vals = new LogVal*[filter->num_fields]; + Value** vals = new Value*[filter->num_fields]; for ( int i = 0; i < filter->num_fields; ++i ) { @@ -1168,7 +919,7 @@ LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, if ( ! val ) { // Value, or any of its parents, is not set. - vals[i] = new LogVal(filter->fields[i]->type, false); + vals[i] = new Value(filter->fields[i]->type, false); break; } } @@ -1180,8 +931,8 @@ LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, return vals; } -LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, LogField** fields) +WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, + int num_fields, const Field* const* fields, bool local, bool remote) { Stream* stream = FindStream(id); @@ -1192,61 +943,15 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path)); - if ( w != stream->writers.end() && w->second ) + if ( w != stream->writers.end() ) // If we already have a writer for this. That's fine, we just // return it. return w->second->writer; - // Need to instantiate a new writer. + WriterFrontend* writer_obj = new WriterFrontend(id, writer, local, remote); + assert(writer_obj); - LogWriterDefinition* ld = log_writers; - - while ( true ) - { - if ( ld->type == BifEnum::Log::WRITER_DEFAULT ) - { - reporter->Error("unknow writer when creating writer"); - return 0; - } - - if ( ld->type == writer->AsEnum() ) - break; - - if ( ! ld->factory ) - // Oops, we can't instantiate this guy. - return 0; - - // If the writer has an init function, call it. - if ( ld->init ) - { - if ( (*ld->init)() ) - // Clear the init function so that we won't - // call it again later. - ld->init = 0; - else - // Init failed, disable by deleting factory - // function. - ld->factory = 0; - - DBG_LOG(DBG_LOGGING, "failed to init writer class %s", - ld->name); - - return false; - } - - ++ld; - } - - assert(ld->factory); - LogWriter* writer_obj = (*ld->factory)(); - - if ( ! writer_obj->Init(path, num_fields, fields) ) - { - DBG_LOG(DBG_LOGGING, "failed to init instance of writer %s", - ld->name); - - return 0; - } + writer_obj->Init(path, num_fields, fields); WriterInfo* winfo = new WriterInfo; winfo->type = writer->Ref()->AsEnumVal(); @@ -1291,16 +996,17 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, return writer_obj; } -void LogMgr::DeleteVals(int num_fields, LogVal** vals) +void Manager::DeleteVals(int num_fields, Value** vals) { + // Note this code is duplicated in WriterBackend::DeleteVals(). for ( int i = 0; i < num_fields; i++ ) delete vals[i]; delete [] vals; } -bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, - LogVal** vals) +bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, + Value** vals) { Stream* stream = FindStream(id); @@ -1310,7 +1016,7 @@ bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, #ifdef DEBUG ODesc desc; id->Describe(&desc); - DBG_LOG(DBG_LOGGING, "unknown stream %s in LogMgr::Write()", + DBG_LOG(DBG_LOGGING, "unknown stream %s in Manager::Write()", desc.Description()); #endif DeleteVals(num_fields, vals); @@ -1332,23 +1038,23 @@ bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, #ifdef DEBUG ODesc desc; id->Describe(&desc); - DBG_LOG(DBG_LOGGING, "unknown writer %s in LogMgr::Write()", + DBG_LOG(DBG_LOGGING, "unknown writer %s in Manager::Write()", desc.Description()); #endif DeleteVals(num_fields, vals); return false; } - bool success = (w->second ? w->second->writer->Write(num_fields, vals) : true); + w->second->writer->Write(num_fields, vals); DBG_LOG(DBG_LOGGING, - "Wrote pre-filtered record to path '%s' on stream '%s' [%s]", - path.c_str(), stream->name.c_str(), (success ? "ok" : "error")); + "Wrote pre-filtered record to path '%s' on stream '%s'", + path.c_str(), stream->name.c_str()); - return success; + return true; } -void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) +void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer) { for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) { @@ -1360,10 +1066,7 @@ void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) { - if ( ! i->second ) - continue; - - LogWriter* writer = i->second->writer; + WriterFrontend* writer = i->second->writer; EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer); remote_serializer->SendLogCreateWriter(peer, (*s)->id, @@ -1375,7 +1078,7 @@ void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) } } -bool LogMgr::SetBuf(EnumVal* id, bool enabled) +bool Manager::SetBuf(EnumVal* id, bool enabled) { Stream* stream = FindStream(id); if ( ! stream ) @@ -1383,17 +1086,14 @@ bool LogMgr::SetBuf(EnumVal* id, bool enabled) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - { - if ( i->second ) - i->second->writer->SetBuf(enabled); - } + i->second->writer->SetBuf(enabled); RemoveDisabledWriters(stream); return true; } -bool LogMgr::Flush(EnumVal* id) +bool Manager::Flush(EnumVal* id) { Stream* stream = FindStream(id); if ( ! stream ) @@ -1404,26 +1104,26 @@ bool LogMgr::Flush(EnumVal* id) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - { - if ( i->second ) - i->second->writer->Flush(); - } + i->second->writer->Flush(); RemoveDisabledWriters(stream); return true; } -void LogMgr::Error(LogWriter* writer, const char* msg) +void Manager::Terminate() { - reporter->Error("error with writer for %s: %s", - writer->Path().c_str(), msg); + for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) + { + if ( *s ) + Flush((*s)->id); + } } // Timer which on dispatching rotates the filter. class RotationTimer : public Timer { public: - RotationTimer(double t, LogMgr::WriterInfo* arg_winfo, bool arg_rotate) + RotationTimer(double t, Manager::WriterInfo* arg_winfo, bool arg_rotate) : Timer(t, TIMER_ROTATE) { winfo = arg_winfo; @@ -1435,7 +1135,7 @@ public: void Dispatch(double t, int is_expire); protected: - LogMgr::WriterInfo* winfo; + Manager::WriterInfo* winfo; bool rotate; }; @@ -1459,7 +1159,7 @@ void RotationTimer::Dispatch(double t, int is_expire) } } -void LogMgr::InstallRotationTimer(WriterInfo* winfo) +void Manager::InstallRotationTimer(WriterInfo* winfo) { if ( terminating ) return; @@ -1501,7 +1201,7 @@ void LogMgr::InstallRotationTimer(WriterInfo* winfo) } } -void LogMgr::Rotate(WriterInfo* winfo) +void Manager::Rotate(WriterInfo* winfo) { DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f", winfo->writer->Path().c_str(), network_time); @@ -1521,7 +1221,7 @@ void LogMgr::Rotate(WriterInfo* winfo) winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating); } -bool LogMgr::FinishedRotation(LogWriter* writer, string new_name, string old_name, +bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name, double open, double close, bool terminating) { DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s", diff --git a/src/logging/Manager.h b/src/logging/Manager.h new file mode 100644 index 0000000000..bf097c5e1a --- /dev/null +++ b/src/logging/Manager.h @@ -0,0 +1,205 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// A class managing log writers and filters. + +#ifndef LOGGING_MANAGER_H +#define LOGGING_MANAGER_H + +#include "../Val.h" +#include "../EventHandler.h" +#include "../RemoteSerializer.h" + +class SerializationFormat; +class RemoteSerializer; +class RotationTimer; + +namespace logging { + + +class WriterBackend; +class WriterFrontend; +class RotationFinishedMessage; + +/** + * Singleton class for managing log streams. + */ +class Manager { +public: + /** + * Constructor. + */ + Manager(); + + /** + * Destructor. + */ + ~Manager(); + + /** + * Creates a new log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param stream A record of script type \c Log::Stream. + * + * This method corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool CreateStream(EnumVal* id, RecordVal* stream); + + /** + * Enables a log log stream. + * + * @param id The enum value corresponding the log stream. + * + * This method corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool EnableStream(EnumVal* id); + + /** + * Disables a log stream. + * + * @param id The enum value corresponding the log stream. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool DisableStream(EnumVal* id); + + /** + * Adds a filter to a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param filter A record of script type \c Log::Filter. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool AddFilter(EnumVal* id, RecordVal* filter); + + /** + * Removes a filter from a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param name The name of the filter to remove. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool RemoveFilter(EnumVal* id, StringVal* name); + + /** + * Removes a filter from a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param name The name of the filter to remove. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool RemoveFilter(EnumVal* id, string name); + + /** + * Write a record to a log stream. + * + * @param id The enum value corresponding the log stream. + * + * @param colums A record of the type defined for the stream's + * columns. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool Write(EnumVal* id, RecordVal* columns); + + /** + * Sets log streams buffering state. This adjusts all associated + * writers to the new state. + * + * @param id The enum value corresponding the log stream. + * + * @param enabled False to disable buffering (default is enabled). + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool SetBuf(EnumVal* id, bool enabled); + + /** + * Flushes a log stream. This flushed all associated writers. + * + * @param id The enum value corresponding the log stream. + * + * This methods corresponds directly to the internal BiF defined in + * logging.bif, which just forwards here. + */ + bool Flush(EnumVal* id); + + /** + * Prepares the log manager to terminate. This will flush all log + * stream. + */ + void Terminate(); + +protected: + friend class WriterFrontend; + friend class RotationFinishedMessage; + friend class ::RemoteSerializer; + friend class ::RotationTimer; + + // Instantiates a new WriterBackend of the given type (note that + // doing so creates a new thread!). + WriterBackend* CreateBackend(WriterFrontend* frontend, bro_int_t type); + + //// Function also used by the RemoteSerializer. + + // Takes ownership of fields. + WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path, + int num_fields, const threading::Field* const* fields, + bool local, bool remote); + + // Takes ownership of values.. + bool Write(EnumVal* id, EnumVal* writer, string path, + int num_fields, threading::Value** vals); + + // Announces all instantiated writers to peer. + void SendAllWritersTo(RemoteSerializer::PeerID peer); + + // Signals that a file has been rotated. + bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name, + double open, double close, bool terminating); + + // Deletes the values as passed into Write(). + void DeleteVals(int num_fields, threading::Value** vals); + +private: + struct Filter; + struct Stream; + struct WriterInfo; + + bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, + TableVal* include, TableVal* exclude, string path, list indices); + + threading::Value** RecordToFilterVals(Stream* stream, Filter* filter, + RecordVal* columns); + + threading::Value* ValToLogVal(Val* val, BroType* ty = 0); + Stream* FindStream(EnumVal* id); + void RemoveDisabledWriters(Stream* stream); + void InstallRotationTimer(WriterInfo* winfo); + void Rotate(WriterInfo* info); + Filter* FindFilter(EnumVal* id, StringVal* filter); + WriterInfo* FindWriter(WriterFrontend* writer); + + vector streams; // Indexed by stream enum. +}; + +} + +extern logging::Manager* log_mgr; + +#endif diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc new file mode 100644 index 0000000000..7c71c09604 --- /dev/null +++ b/src/logging/WriterBackend.cc @@ -0,0 +1,281 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "util.h" +#include "threading/SerialTypes.h" + +#include "WriterBackend.h" +#include "WriterFrontend.h" + +// Messages sent from backend to frontend (i.e., "OutputMessages"). + +using threading::Value; +using threading::Field; + +namespace logging { + +class RotationFinishedMessage : public threading::OutputMessage +{ +public: + RotationFinishedMessage(WriterFrontend* writer, string new_name, string old_name, + double open, double close, bool terminating) + : threading::OutputMessage("RotationFinished", writer), + new_name(new_name), old_name(old_name), open(open), + close(close), terminating(terminating) { } + + virtual bool Process() + { + return log_mgr->FinishedRotation(Object(), new_name, old_name, open, close, terminating); + } + +private: + string new_name; + string old_name; + double open; + double close; + bool terminating; +}; + +class FlushWriteBufferMessage : public threading::OutputMessage +{ +public: + FlushWriteBufferMessage(WriterFrontend* writer) + : threading::OutputMessage("FlushWriteBuffer", writer) {} + + virtual bool Process() { Object()->FlushWriteBuffer(); return true; } +}; + +class DisableMessage : public threading::OutputMessage +{ +public: + DisableMessage(WriterFrontend* writer) + : threading::OutputMessage("Disable", writer) {} + + virtual bool Process() { Object()->SetDisable(); return true; } +}; + +} + +// Backend methods. + +using namespace logging; + +WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread() + { + path = ""; + num_fields = 0; + fields = 0; + buffering = true; + frontend = arg_frontend; + + SetName(frontend->Name()); + } + +WriterBackend::~WriterBackend() + { + if ( fields ) + { + for(int i = 0; i < num_fields; ++i) + delete fields[i]; + + delete [] fields; + } + } + +void WriterBackend::DeleteVals(int num_writes, Value*** vals) + { + for ( int j = 0; j < num_writes; ++j ) + { + // Note this code is duplicated in Manager::DeleteVals(). + for ( int i = 0; i < num_fields; i++ ) + delete vals[j][i]; + + delete [] vals[j]; + } + + delete [] vals; + } + +bool WriterBackend::FinishedRotation(string new_name, string old_name, + double open, double close, bool terminating) + { + SendOut(new RotationFinishedMessage(frontend, new_name, old_name, open, close, terminating)); + return true; + } + +void WriterBackend::DisableFrontend() + { + SendOut(new DisableMessage(frontend)); + } + +bool WriterBackend::Init(string arg_path, int arg_num_fields, const Field* const* arg_fields) + { + path = arg_path; + num_fields = arg_num_fields; + fields = arg_fields; + + string name = Fmt("%s/%s", path.c_str(), frontend->Name().c_str()); + + SetName(name); + + if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) + { + DisableFrontend(); + return false; + } + + return true; + } + +bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals) + { + // Double-check that the arguments match. If we get this from remote, + // something might be mixed up. + if ( num_fields != arg_num_fields ) + { + +#ifdef DEBUG + const char* msg = Fmt("Number of fields don't match in WriterBackend::Write() (%d vs. %d)", + arg_num_fields, num_fields); + Debug(DBG_LOGGING, msg); +#endif + + DeleteVals(num_writes, vals); + DisableFrontend(); + return false; + } + +#ifdef DEBUG + // Double-check all the types match. + for ( int j = 0; j < num_writes; j++ ) + { + for ( int i = 0; i < num_fields; ++i ) + { + if ( vals[j][i]->type != fields[i]->type ) + { + const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)", + vals[j][i]->type, fields[i]->type); + Debug(DBG_LOGGING, msg); + + DisableFrontend(); + DeleteVals(num_writes, vals); + return false; + } + } + } +#endif + + bool success = true; + + for ( int j = 0; j < num_writes; j++ ) + { + success = DoWrite(num_fields, fields, vals[j]); + + if ( ! success ) + break; + } + + DeleteVals(num_writes, vals); + + if ( ! success ) + DisableFrontend(); + + return success; + } + +bool WriterBackend::SetBuf(bool enabled) + { + if ( enabled == buffering ) + // No change. + return true; + + buffering = enabled; + + if ( ! DoSetBuf(enabled) ) + { + DisableFrontend(); + return false; + } + + return true; + } + +bool WriterBackend::Rotate(string rotated_path, double open, + double close, bool terminating) + { + if ( ! DoRotate(rotated_path, open, close, terminating) ) + { + DisableFrontend(); + return false; + } + + return true; + } + +bool WriterBackend::Flush() + { + if ( ! DoFlush() ) + { + DisableFrontend(); + return false; + } + + return true; + } + +bool WriterBackend::Finish() + { + if ( ! DoFlush() ) + { + DisableFrontend(); + return false; + } + + return true; + } + +bool WriterBackend::DoHeartbeat(double network_time, double current_time) + { + MsgThread::DoHeartbeat(network_time, current_time); + + SendOut(new FlushWriteBufferMessage(frontend)); + + return true; + } + +string WriterBackend::Render(const threading::Value::addr_t& addr) const + { + if ( addr.family == IPv4 ) + { + char s[INET_ADDRSTRLEN]; + + if ( inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) == NULL ) + return ""; + else + return s; + } + else + { + char s[INET6_ADDRSTRLEN]; + + if ( inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) == NULL ) + return ""; + else + return s; + } + } + +string WriterBackend::Render(const threading::Value::subnet_t& subnet) const + { + char l[16]; + + if ( subnet.prefix.family == IPv4 ) + modp_uitoa10(subnet.length - 96, l); + else + modp_uitoa10(subnet.length, l); + + string s = Render(subnet.prefix) + "/" + l; + + return s; + } + + diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h new file mode 100644 index 0000000000..eea0927fd2 --- /dev/null +++ b/src/logging/WriterBackend.h @@ -0,0 +1,319 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Bridge class between main process and writer threads. + +#ifndef LOGGING_WRITERBACKEND_H +#define LOGGING_WRITERBACKEND_H + +#include "Manager.h" + +#include "threading/MsgThread.h" + +namespace logging { + +/** + * Base class for writer implementation. When the logging::Manager creates a + * new logging filter, it instantiates a WriterFrontend. That then in turn + * creates a WriterBackend of the right type. The frontend then forwards + * messages over the backend as its methods are called. + * + * All of this methods must be called only from the corresponding child + * thread (the constructor and destructor are the exceptions.) + */ +class WriterBackend : public threading::MsgThread +{ +public: + /** + * Constructor. + * + * @param frontend The frontend writer that created this backend. The + * *only* purpose of this value is to be passed back via messages as + * a argument to callbacks. One must not otherwise access the + * frontend, it's running in a different thread. + * + * @param name A descriptive name for writer's type (e.g., \c Ascii). + * + */ + WriterBackend(WriterFrontend* frontend); + + /** + * Destructor. + */ + virtual ~WriterBackend(); + + /** + * One-time initialization of the writer to define the logged fields. + * + * @param path A string left to the interpretation of the writer + * implementation; it corresponds to the value configured on the + * script-level for the logging filter. + * + * @param num_fields The number of log fields for the stream. + * + * @param fields An array of size \a num_fields with the log fields. + * The methods takes ownership of the array. + * + * @return False if an error occured. + */ + bool Init(string path, int num_fields, const threading::Field* const* fields); + + /** + * Writes one log entry. + * + * @param num_fields: The number of log fields for this stream. The + * value must match what was passed to Init(). + * + * @param An array of size \a num_fields with the log values. Their + * types musst match with the field passed to Init(). The method + * takes ownership of \a vals.. + * + * Returns false if an error occured, in which case the writer must + * not be used any further. + * + * @return False if an error occured. + */ + bool Write(int num_fields, int num_writes, threading::Value*** vals); + + /** + * Sets the buffering status for the writer, assuming the writer + * supports that. (If not, it will be ignored). + * + * @param enabled False if buffering is to be disabled (by default + * it's on). + * + * @return False if an error occured. + */ + bool SetBuf(bool enabled); + + /** + * Flushes any currently buffered output, assuming the writer + * supports that. (If not, it will be ignored). + * + * @return False if an error occured. + */ + bool Flush(); + + /** + * Triggers rotation, if the writer supports that. (If not, it will + * be ignored). + * + * @return False if an error occured. + */ + bool Rotate(string rotated_path, double open, double close, bool terminating); + + /** + * Finishes writing to this logger in a regularl fashion. Must not be + * called if an error has been indicated earlier. After calling this, + * no further writing must be performed. + * + * @return False if an error occured. + */ + bool Finish(); + + /** + * Disables the frontend that has instantiated this backend. Once + * disabled,the frontend will not send any further message over. + */ + void DisableFrontend(); + + /** + * Returns the log path as passed into the constructor. + */ + const string Path() const { return path; } + + /** + * Returns the number of log fields as passed into the constructor. + */ + int NumFields() const { return num_fields; } + + /** + * Returns the log fields as passed into the constructor. + */ + const threading::Field* const * Fields() const { return fields; } + + /** + * Returns the current buffering state. + * + * @return True if buffering is enabled. + */ + bool IsBuf() { return buffering; } + + /** + * Signals that a file has been rotated. This must be called by a + * writer's implementation of DoRotate() once rotation has finished. + * + * Most of the parameters should be passed through from DoRotate(). + * + * @param new_name The filename of the rotated file. + * + * @param old_name The filename of the original file. + * + * @param open: The timestamp when the original file was opened. + * + * @param close: The timestamp when the origina file was closed. + * + * @param terminating: True if the original rotation request occured + * due to the main Bro process shutting down. + */ + bool FinishedRotation(string new_name, string old_name, + double open, double close, bool terminating); + + /** Helper method to render an IP address as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + string Render(const threading::Value::addr_t& addr) const; + + /** Helper method to render an subnet value as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + string Render(const threading::Value::subnet_t& subnet) const; + +protected: + /** + * Writer-specific intialization method. + * + * A writer implementation must override this method. If it returns + * false, it will be assumed that a fatal error has occured that + * prevents the writer from further operation; it will then be + * disabled and eventually deleted. When returning false, an + * implementation should also call Error() to indicate what happened. + */ + virtual bool DoInit(string path, int num_fields, + const threading::Field* const* fields) = 0; + + /** + * Writer-specific output method implementing recording of fone log + * entry. + * + * A writer implementation must override this method. If it returns + * false, it will be assumed that a fatal error has occured that + * prevents the writer from further operation; it will then be + * disabled and eventually deleted. When returning false, an + * implementation should also call Error() to indicate what happened. + */ + virtual bool DoWrite(int num_fields, const threading::Field* const* fields, + threading::Value** vals) = 0; + + /** + * Writer-specific method implementing a change of fthe buffering + * state. If buffering is disabled, the writer should attempt to + * write out information as quickly as possible even if doing so may + * have a performance impact. If enabled (which is the default), it + * may buffer data as helpful and write it out later in a way + * optimized for performance. The current buffering state can be + * queried via IsBuf(). + * + * A writer implementation must override this method but it can just + * ignore calls if buffering doesn't align with its semantics. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + */ + virtual bool DoSetBuf(bool enabled) = 0; + + /** + * Writer-specific method implementing flushing of its output. + * + * A writer implementation must override this method but it can just + * ignore calls if flushing doesn't align with its semantics. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + */ + virtual bool DoFlush() = 0; + + /** + * Writer-specific method implementing log rotation. Most directly + * this only applies to writers writing into files, which should then + * close the current file and open a new one. However, a writer may + * also trigger other apppropiate actions if semantics are similar. * + * Once rotation has finished, the implementation must call + * FinishedRotation() to signal the log manager that potential + * postprocessors can now run. + * + * A writer implementation must override this method but it can just + * ignore calls if flushing doesn't align with its semantics. It + * still needs to call FinishedRotation() though. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + * + * @param rotate_path Reflects the path to where the rotated output + * is to be moved, with specifics depending on the writer. It should + * generally be interpreted in a way consistent with that of \c path + * as passed into DoInit(). As an example, for file-based output, \c + * rotate_path could be the original filename extended with a + * timestamp indicating the time of the rotation. + * + * @param open The network time when the *current* file was opened. + * + * @param close The network time when the *current* file was closed. + * + * @param terminating Indicates whether the rotation request occurs + * due the main Bro prcoess terminating (and not because we've + * reached a regularly scheduled time for rotation). + */ + virtual bool DoRotate(string rotated_path, double open, double close, + bool terminating) = 0; + + /** + * Writer-specific method implementing log output finalization at + * termination. Not called when any of the other methods has + * previously signaled an error, i.e., executing this method signals + * a regular shutdown of the writer. + * + * A writer implementation must override this method but it can just + * ignore calls if flushing doesn't align with its semantics. + * + * If the method returns false, it will be assumed that a fatal error + * has occured that prevents the writer from further operation; it + * will then be disabled and eventually deleted. When returning + * false, an implementation should also call Error() to indicate what + * happened. + */ + virtual bool DoFinish() = 0; + + /** + * Triggered by regular heartbeat messages from the main thread. + * + * This method can be overridden but once must call + * WriterBackend::DoHeartbeat(). + */ + virtual bool DoHeartbeat(double network_time, double current_time); + +private: + /** + * Deletes the values as passed into Write(). + */ + void DeleteVals(int num_writes, threading::Value*** vals); + + // Frontend that instantiated us. This object must not be access from + // this class, it's running in a different thread! + WriterFrontend* frontend; + + string path; // Log path. + int num_fields; // Number of log fields. + const threading::Field* const* fields; // Log fields. + bool buffering; // True if buffering is enabled. +}; + + +} + +#endif + diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc new file mode 100644 index 0000000000..c100e99f90 --- /dev/null +++ b/src/logging/WriterFrontend.cc @@ -0,0 +1,279 @@ + +#include "Net.h" +#include "threading/SerialTypes.h" + +#include "WriterFrontend.h" +#include "WriterBackend.h" + +using threading::Value; +using threading::Field; + +namespace logging { + +// Messages sent from frontend to backend (i.e., "InputMessages"). + +class InitMessage : public threading::InputMessage +{ +public: + InitMessage(WriterBackend* backend, const string path, const int num_fields, const Field* const* fields) + : threading::InputMessage("Init", backend), + path(path), num_fields(num_fields), fields(fields) { } + + virtual bool Process() { return Object()->Init(path, num_fields, fields); } + +private: + const string path; + const int num_fields; + const Field * const* fields; +}; + +class RotateMessage : public threading::InputMessage +{ +public: + RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const string rotated_path, const double open, + const double close, const bool terminating) + : threading::InputMessage("Rotate", backend), + frontend(frontend), + rotated_path(rotated_path), open(open), + close(close), terminating(terminating) { } + + virtual bool Process() { return Object()->Rotate(rotated_path, open, close, terminating); } + +private: + WriterFrontend* frontend; + const string rotated_path; + const double open; + const double close; + const bool terminating; +}; + +class WriteMessage : public threading::InputMessage +{ +public: + WriteMessage(WriterBackend* backend, int num_fields, int num_writes, Value*** vals) + : threading::InputMessage("Write", backend), + num_fields(num_fields), num_writes(num_writes), vals(vals) {} + + virtual bool Process() { return Object()->Write(num_fields, num_writes, vals); } + +private: + int num_fields; + int num_writes; + Value ***vals; +}; + +class SetBufMessage : public threading::InputMessage +{ +public: + SetBufMessage(WriterBackend* backend, const bool enabled) + : threading::InputMessage("SetBuf", backend), + enabled(enabled) { } + + virtual bool Process() { return Object()->SetBuf(enabled); } + +private: + const bool enabled; +}; + +class FlushMessage : public threading::InputMessage +{ +public: + FlushMessage(WriterBackend* backend) + : threading::InputMessage("Flush", backend) {} + + virtual bool Process() { return Object()->Flush(); } +}; + +class FinishMessage : public threading::InputMessage +{ +public: + FinishMessage(WriterBackend* backend) + : threading::InputMessage("Finish", backend) {} + + virtual bool Process() { return Object()->Finish(); } +}; + +} + +// Frontend methods. + +using namespace logging; + +WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote) + { + stream = arg_stream; + writer = arg_writer; + Ref(stream); + Ref(writer); + + disabled = initialized = false; + buf = true; + local = arg_local; + remote = arg_remote; + write_buffer = 0; + write_buffer_pos = 0; + ty_name = ""; + + if ( local ) + { + backend = log_mgr->CreateBackend(this, writer->AsEnum()); + assert(backend); + backend->Start(); + } + + else + backend = 0; + } + +WriterFrontend::~WriterFrontend() + { + Unref(stream); + Unref(writer); + } + +string WriterFrontend::Name() const + { + if ( path.size() ) + return ty_name; + + return ty_name + "/" + path; + } + +void WriterFrontend::Stop() + { + FlushWriteBuffer(); + SetDisable(); + + if ( backend ) + backend->Stop(); + } + +void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* const * arg_fields) + { + if ( disabled ) + return; + + if ( initialized ) + reporter->InternalError("writer initialize twice"); + + path = arg_path; + num_fields = arg_num_fields; + fields = arg_fields; + + initialized = true; + + if ( backend ) + backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + + if ( remote ) + remote_serializer->SendLogCreateWriter(stream, + writer, + arg_path, + arg_num_fields, + arg_fields); + + } + +void WriterFrontend::Write(int num_fields, Value** vals) + { + if ( disabled ) + return; + + if ( remote ) + remote_serializer->SendLogWrite(stream, + writer, + path, + num_fields, + vals); + + if ( ! backend ) + { + DeleteVals(vals); + return; + } + + if ( ! write_buffer ) + { + // Need new buffer. + write_buffer = new Value**[WRITER_BUFFER_SIZE]; + write_buffer_pos = 0; + } + + write_buffer[write_buffer_pos++] = vals; + + if ( write_buffer_pos >= WRITER_BUFFER_SIZE || ! buf || terminating ) + // Buffer full (or no bufferin desired or termiating). + FlushWriteBuffer(); + + } + +void WriterFrontend::FlushWriteBuffer() + { + if ( ! write_buffer_pos ) + // Nothing to do. + return; + + if ( backend ) + backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer)); + + // Clear buffer (no delete, we pass ownership to child thread.) + write_buffer = 0; + write_buffer_pos = 0; + } + +void WriterFrontend::SetBuf(bool enabled) + { + if ( disabled ) + return; + + buf = enabled; + + if ( backend ) + backend->SendIn(new SetBufMessage(backend, enabled)); + + if ( ! buf ) + // Make sure no longer buffer any still queued data. + FlushWriteBuffer(); + } + +void WriterFrontend::Flush() + { + if ( disabled ) + return; + + FlushWriteBuffer(); + + if ( backend ) + backend->SendIn(new FlushMessage(backend)); + } + +void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating) + { + if ( disabled ) + return; + + FlushWriteBuffer(); + + if ( backend ) + backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); + } + +void WriterFrontend::Finish() + { + if ( disabled ) + return; + + FlushWriteBuffer(); + + if ( backend ) + backend->SendIn(new FinishMessage(backend)); + } + +void WriterFrontend::DeleteVals(Value** vals) + { + // Note this code is duplicated in Manager::DeleteVals(). + for ( int i = 0; i < num_fields; i++ ) + delete vals[i]; + + delete [] vals; + } diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h new file mode 100644 index 0000000000..b83250a5b8 --- /dev/null +++ b/src/logging/WriterFrontend.h @@ -0,0 +1,222 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef LOGGING_WRITERFRONTEND_H +#define LOGGING_WRITERFRONTEND_H + +#include "Manager.h" + +#include "threading/MsgThread.h" + +namespace logging { + +class WriterBackend; + +/** + * Bridge class between the logging::Manager and backend writer threads. The + * Manager instantiates one \a WriterFrontend for each open logging filter. + * Each frontend in turns instantiates a WriterBackend-derived class + * internally that's specific to the particular output format. That backend + * runs in a new thread, and it receives messages from the frontend that + * correspond to method called by the manager. + * + */ +class WriterFrontend { +public: + /** + * Constructor. + * + * stream: The logging stream. + * + * writer: The backend writer type, with the value corresponding to the + * script-level \c Log::Writer enum (e.g., \a WRITER_ASCII). The + * frontend will internally instantiate a WriterBackend of the + * corresponding type. + * + * local: If true, the writer will instantiate a local backend. + * + * remote: If true, the writer will forward all data to remote + * clients. + * + * Frontends must only be instantiated by the main thread. + */ + WriterFrontend(EnumVal* stream, EnumVal* writer, bool local, bool remote); + + /** + * Destructor. + * + * Frontends must only be destroyed by the main thread. + */ + virtual ~WriterFrontend(); + + /** + * Stops all output to this writer. Calling this methods disables all + * message forwarding to the backend and stops the backend thread. + * + * This method must only be called from the main thread. + */ + void Stop(); + + /** + * Initializes the writer. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See WriterBackend::Init() for arguments. The method takes + * ownership of \a fields. + * + * This method must only be called from the main thread. + */ + void Init(string path, int num_fields, const threading::Field* const* fields); + + /** + * Write out a record. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * As an optimization, if buffering is enabled (which is the default) + * this method may buffer several writes and send them over to the + * backend in bulk with a single message. An explicit bulk write of + * all currently buffered data can be triggered with + * FlushWriteBuffer(). The backend writer triggers this with a + * message at every heartbeat. + * + * See WriterBackend::Writer() for arguments (except that this method + * takes only a single record, not an array). The method takes + * ownership of \a vals. + * + * This method must only be called from the main thread. + */ + void Write(int num_fields, threading::Value** vals); + + /** + * Sets the buffering state. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See WriterBackend::SetBuf() for arguments. + * + * This method must only be called from the main thread. + */ + void SetBuf(bool enabled); + + /** + * Flushes the output. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. In addition, it also triggers + * FlushWriteBuffer(). If the backend method fails, it sends a + * message back that will asynchronously call Disable(). + * + * This method must only be called from the main thread. + */ + void Flush(); + + /** + * Triggers log rotation. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See WriterBackend::Rotate() for arguments. + * + * This method must only be called from the main thread. + */ + void Rotate(string rotated_path, double open, double close, bool terminating); + + /** + * Finalizes writing to this tream. + * + * This method generates a message to the backend writer and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * This method must only be called from the main thread. + */ + void Finish(); + + /** + * Explicitly triggers a transfer of all potentially buffered Write() + * operations over to the backend. + * + * This method must only be called from the main thread. + */ + void FlushWriteBuffer(); + + /** + * Disables the writer frontend. From now on, all method calls that + * would normally send message over to the backend, turn into no-ops. + * Note though that it does not stop the backend itself, use Stop() + * to do thast as well (this method is primarily for use as callback + * when the backend wants to disable the frontend). + * + * Disabled frontend will eventually be discarded by the + * logging::Manager. + * + * This method must only be called from the main thread. + */ + void SetDisable() { disabled = true; } + + /** + * Returns true if the writer frontend has been disabled with SetDisable(). + */ + bool Disabled() { return disabled; } + + /** + * Returns the log path as passed into the constructor. + */ + const string Path() const { return path; } + + /** + * Returns the number of log fields as passed into the constructor. + */ + int NumFields() const { return num_fields; } + + /** + * Returns a descriptive name for the writer, including the type of + * the backend and the path used. + * + * This method is safe to call from any thread. + */ + string Name() const; + + /** + * Returns the log fields as passed into the constructor. + */ + const threading::Field* const * Fields() const { return fields; } + +protected: + friend class Manager; + + void DeleteVals(threading::Value** vals); + + EnumVal* stream; + EnumVal* writer; + + WriterBackend* backend; // The backend we have instanatiated. + bool disabled; // True if disabled. + bool initialized; // True if initialized. + bool buf; // True if buffering is enabled (default). + bool local; // True if logging locally. + bool remote; // True if loggin remotely. + + string ty_name; // Name of the backend type. Set by the manager. + string path; // The log path. + int num_fields; // The number of log fields. + const threading::Field* const* fields; // The log fields. + + // Buffer for bulk writes. + static const int WRITER_BUFFER_SIZE = 1000; + int write_buffer_pos; // Position of next write in buffer. + threading::Value*** write_buffer; // Buffer of size WRITER_BUFFER_SIZE. +}; + +} + +#endif diff --git a/src/LogWriterAscii.cc b/src/logging/writers/Ascii.cc similarity index 88% rename from src/LogWriterAscii.cc rename to src/logging/writers/Ascii.cc index 8dd36e1991..0759e60a82 100644 --- a/src/LogWriterAscii.cc +++ b/src/logging/writers/Ascii.cc @@ -3,10 +3,17 @@ #include #include -#include "LogWriterAscii.h" #include "NetVar.h" +#include "threading/SerialTypes.h" -LogWriterAscii::LogWriterAscii() +#include "Ascii.h" + +using namespace logging; +using namespace writer; +using threading::Value; +using threading::Field; + +Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) { file = 0; @@ -42,7 +49,7 @@ LogWriterAscii::LogWriterAscii() desc.AddEscapeSequence(separator, separator_len); } -LogWriterAscii::~LogWriterAscii() +Ascii::~Ascii() { if ( file ) fclose(file); @@ -54,7 +61,7 @@ LogWriterAscii::~LogWriterAscii() delete [] header_prefix; } -bool LogWriterAscii::WriteHeaderField(const string& key, const string& val) +bool Ascii::WriteHeaderField(const string& key, const string& val) { string str = string(header_prefix, header_prefix_len) + key + string(separator, separator_len) + val + "\n"; @@ -62,8 +69,8 @@ bool LogWriterAscii::WriteHeaderField(const string& key, const string& val) return (fwrite(str.c_str(), str.length(), 1, file) == 1); } -bool LogWriterAscii::DoInit(string path, int num_fields, - const LogField* const * fields) +bool Ascii::DoInit(string path, int num_fields, + const Field* const * fields) { if ( output_to_stdout ) path = "/dev/stdout"; @@ -108,7 +115,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, types += string(separator, separator_len); } - const LogField* field = fields[i]; + const Field* field = fields[i]; names += field->name; types += type_name(field->type); if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) ) @@ -131,17 +138,18 @@ write_error: return false; } -bool LogWriterAscii::DoFlush() +bool Ascii::DoFlush() { fflush(file); return true; } -void LogWriterAscii::DoFinish() +bool Ascii::DoFinish() { + return true; } -bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) +bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) { if ( ! val->present ) { @@ -161,16 +169,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: desc->Add(val->val.uint_val); break; + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + case TYPE_SUBNET: - desc->Add(*val->val.subnet_val); + desc->Add(Render(val->val.subnet_val)); break; case TYPE_ADDR: - desc->Add(*val->val.addr_val); + desc->Add(Render(val->val.addr_val)); break; case TYPE_TIME: @@ -182,7 +193,7 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_DOUBLE: desc->Add(val->val.double_val); - break; + break; case TYPE_ENUM: case TYPE_STRING: @@ -279,8 +290,8 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) return true; } -bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals) +bool Ascii::DoWrite(int num_fields, const Field* const * fields, + Value** vals) { if ( ! file ) DoInit(Path(), NumFields(), Fields()); @@ -310,8 +321,7 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, return true; } -bool LogWriterAscii::DoRotate(string rotated_path, double open, - double close, bool terminating) +bool Ascii::DoRotate(string rotated_path, double open, double close, bool terminating) { // Don't rotate special files or if there's not one currently open. if ( ! file || IsSpecial(Path()) ) @@ -332,13 +342,13 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open, return true; } -bool LogWriterAscii::DoSetBuf(bool enabled) +bool Ascii::DoSetBuf(bool enabled) { // Nothing to do. return true; } -string LogWriterAscii::LogExt() +string Ascii::LogExt() { const char* ext = getenv("BRO_LOG_SUFFIX"); if ( ! ext ) ext = "log"; diff --git a/src/LogWriterAscii.h b/src/logging/writers/Ascii.h similarity index 53% rename from src/LogWriterAscii.h rename to src/logging/writers/Ascii.h index 72127c8b1f..6f507aff01 100644 --- a/src/LogWriterAscii.h +++ b/src/logging/writers/Ascii.h @@ -2,33 +2,36 @@ // // Log writer for delimiter-separated ASCII logs. -#ifndef LOGWRITERASCII_H -#define LOGWRITERASCII_H +#ifndef LOGGING_WRITER_ASCII_H +#define LOGGING_WRITER_ASCII_H -#include "LogWriter.h" +#include "../WriterBackend.h" -class LogWriterAscii : public LogWriter { +namespace logging { namespace writer { + +class Ascii : public WriterBackend { public: - LogWriterAscii(); - ~LogWriterAscii(); + Ascii(WriterFrontend* frontend); + ~Ascii(); - static LogWriter* Instantiate() { return new LogWriterAscii; } + static WriterBackend* Instantiate(WriterFrontend* frontend) + { return new Ascii(frontend); } static string LogExt(); protected: virtual bool DoInit(string path, int num_fields, - const LogField* const * fields); - virtual bool DoWrite(int num_fields, const LogField* const * fields, - LogVal** vals); + const threading::Field* const* fields); + virtual bool DoWrite(int num_fields, const threading::Field* const* fields, + threading::Value** vals); virtual bool DoSetBuf(bool enabled); - virtual bool DoRotate(string rotated_path, double open, double close, - bool terminating); + virtual bool DoRotate(string rotated_path, double open, + double close, bool terminating); virtual bool DoFlush(); - virtual void DoFinish(); + virtual bool DoFinish(); private: bool IsSpecial(string path) { return path.find("/dev/") == 0; } - bool DoWriteOne(ODesc* desc, LogVal* val, const LogField* field); + bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field); bool WriteHeaderField(const string& key, const string& value); FILE* file; @@ -55,4 +58,8 @@ private: int header_prefix_len; }; +} +} + + #endif diff --git a/src/LogWriterNone.cc b/src/logging/writers/None.cc similarity index 54% rename from src/LogWriterNone.cc rename to src/logging/writers/None.cc index 592772afdb..a9a7872f85 100644 --- a/src/LogWriterNone.cc +++ b/src/logging/writers/None.cc @@ -1,8 +1,10 @@ -#include "LogWriterNone.h" +#include "None.h" -bool LogWriterNone::DoRotate(string rotated_path, double open, - double close, bool terminating) +using namespace logging; +using namespace writer; + +bool None::DoRotate(string rotated_path, double open, double close, bool terminating) { if ( ! FinishedRotation(string("/dev/null"), Path(), open, close, terminating)) { diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h new file mode 100644 index 0000000000..cce48953d1 --- /dev/null +++ b/src/logging/writers/None.h @@ -0,0 +1,36 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Dummy log writer that just discards everything (but still pretends to rotate). + +#ifndef LOGGING_WRITER_NONE_H +#define LOGGING_WRITER_NONE_H + +#include "../WriterBackend.h" + +namespace logging { namespace writer { + +class None : public WriterBackend { +public: + None(WriterFrontend* frontend) : WriterBackend(frontend) {} + ~None() {}; + + static WriterBackend* Instantiate(WriterFrontend* frontend) + { return new None(frontend); } + +protected: + virtual bool DoInit(string path, int num_fields, + const threading::Field* const * fields) { return true; } + + virtual bool DoWrite(int num_fields, const threading::Field* const* fields, + threading::Value** vals) { return true; } + virtual bool DoSetBuf(bool enabled) { return true; } + virtual bool DoRotate(string rotated_path, double open, + double close, bool terminating); + virtual bool DoFlush() { return true; } + virtual bool DoFinish() { return true; } +}; + +} +} + +#endif diff --git a/src/main.cc b/src/main.cc index f5a5b5282f..ff33a3859d 100644 --- a/src/main.cc +++ b/src/main.cc @@ -29,7 +29,6 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "Event.h" #include "File.h" #include "Reporter.h" -#include "LogMgr.h" #include "Net.h" #include "NetVar.h" #include "Var.h" @@ -47,7 +46,10 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "DPM.h" #include "BroDoc.h" #include "Brofiler.h" -#include "LogWriterAscii.h" + +#include "threading/Manager.h" +#include "logging/Manager.h" +#include "logging/writers/Ascii.h" #include "binpac_bro.h" @@ -63,7 +65,7 @@ extern "C" { #include "setsignal.h" }; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker* heap_checker = 0; int perftools_leaks = 0; int perftools_profile = 0; @@ -74,7 +76,8 @@ char* writefile = 0; name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; -LogMgr* log_mgr; +logging::Manager* log_mgr = 0; +threading::Manager* thread_mgr = 0; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -174,7 +177,7 @@ void usage() fprintf(stderr, " -W|--watchdog | activate watchdog timer\n"); fprintf(stderr, " -Z|--doc-scripts | generate documentation for all loaded scripts\n"); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG fprintf(stderr, " -m|--mem-leaks | show leaks [perftools]\n"); fprintf(stderr, " -M|--mem-profile | record heap [perftools]\n"); #endif @@ -195,7 +198,7 @@ void usage() fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes()); fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake()); fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); - fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", LogWriterAscii::LogExt().c_str()); + fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str()); fprintf(stderr, " $BRO_PROFILER_FILE | Output file for script execution statistics (not set)\n"); exit(1); @@ -241,7 +244,7 @@ void done_with_network() net_finish(1); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG if ( perftools_profile ) { @@ -285,6 +288,9 @@ void terminate_bro() if ( remote_serializer ) remote_serializer->LogStats(); + log_mgr->Terminate(); + thread_mgr->Terminate(); + delete timer_mgr; delete dns_mgr; delete persistence_serializer; @@ -296,6 +302,7 @@ void terminate_bro() delete remote_serializer; delete dpm; delete log_mgr; + delete thread_mgr; delete reporter; } @@ -324,6 +331,13 @@ RETSIGTYPE sig_handler(int signo) { set_processing_status("TERMINATING", "sig_handler"); signal_val = signo; + + if ( thread_mgr->Terminating() && (signal_val == SIGTERM || signal_val == SIGINT) ) + // If the thread manager is already terminating (i.e., + // waiting for child threads to exit), another term signal + // will send the threads a kill. + thread_mgr->KillThreads(); + return RETSIGVAL; } @@ -410,7 +424,7 @@ int main(int argc, char** argv) #ifdef USE_IDMEF {"idmef-dtd", required_argument, 0, 'n'}, #endif -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG {"mem-leaks", no_argument, 0, 'm'}, {"mem-profile", no_argument, 0, 'M'}, #endif @@ -452,7 +466,7 @@ int main(int argc, char** argv) safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLOPSWbdghvZ", sizeof(opts)); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG strncat(opts, "mM", 2); #endif @@ -608,7 +622,7 @@ int main(int argc, char** argv) exit(0); break; -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG case 'm': perftools_leaks = 1; break; @@ -658,7 +672,9 @@ int main(int argc, char** argv) set_processing_status("INITIALIZING", "main"); bro_start_time = current_time(true); + reporter = new Reporter(); + thread_mgr = new threading::Manager(); #ifdef DEBUG if ( debug_streams ) @@ -724,7 +740,7 @@ int main(int argc, char** argv) persistence_serializer = new PersistenceSerializer(); remote_serializer = new RemoteSerializer(); event_registry = new EventRegistry(); - log_mgr = new LogMgr(); + log_mgr = new logging::Manager(); if ( events_file ) event_player = new EventPlayer(events_file); @@ -742,14 +758,14 @@ int main(int argc, char** argv) // nevertheless reported; see perftools docs), thus // we suppress some messages here. -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG { HeapLeakChecker::Disabler disabler; #endif yyparse(); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG } #endif @@ -996,12 +1012,14 @@ int main(int argc, char** argv) have_pending_timers = ! reading_traces && timer_mgr->Size() > 0; + io_sources.Register(thread_mgr, true); + if ( io_sources.Size() > 0 || have_pending_timers ) { if ( profiling_logger ) profiling_logger->Log(); -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG if ( perftools_leaks ) heap_checker = new HeapLeakChecker("net_run"); diff --git a/src/net_util.cc b/src/net_util.cc index 856e351138..57fcc800f9 100644 --- a/src/net_util.cc +++ b/src/net_util.cc @@ -38,33 +38,6 @@ int ones_complement_checksum(const IPAddr& a, uint32 sum) return ones_complement_checksum(bytes, len*4, sum); } -int tcp_checksum(const struct ip* ip, const struct tcphdr* tp, int len) - { - // ### Note, this is only correct for IPv4. This routine is only - // used by the connection compressor (which we turn off for IPv6 - // traffic). - - int tcp_len = tp->th_off * 4 + len; - uint32 sum; - - if ( len % 2 == 1 ) - // Add in pad byte. - sum = htons(((const u_char*) tp)[tcp_len - 1] << 8); - else - sum = 0; - - sum = ones_complement_checksum((void*) &ip->ip_src.s_addr, 4, sum); - sum = ones_complement_checksum((void*) &ip->ip_dst.s_addr, 4, sum); - - uint32 addl_pseudo = - (htons(IPPROTO_TCP) << 16) | htons((unsigned short) tcp_len); - - sum = ones_complement_checksum((void*) &addl_pseudo, 4, sum); - sum = ones_complement_checksum((void*) tp, tcp_len, sum); - - return sum; - } - int udp_checksum(const struct ip* ip, const struct udphdr* up, int len) { uint32 sum; @@ -191,12 +164,11 @@ const char* fmt_conn_id(const IPAddr& src_addr, uint32 src_port, const char* fmt_conn_id(const uint32* src_addr, uint32 src_port, const uint32* dst_addr, uint32 dst_port) { - IPAddr src(IPAddr::IPv6, src_addr, IPAddr::Network); - IPAddr dst(IPAddr::IPv6, dst_addr, IPAddr::Network); + IPAddr src(IPv6, src_addr, IPAddr::Network); + IPAddr dst(IPv6, dst_addr, IPAddr::Network); return fmt_conn_id(src, src_port, dst, dst_port); } - uint32 extract_uint32(const u_char* data) { uint32 val; diff --git a/src/net_util.h b/src/net_util.h index 17ca2b9cc1..8f05a41a2a 100644 --- a/src/net_util.h +++ b/src/net_util.h @@ -5,6 +5,13 @@ #include "config.h" +// Define first. +typedef enum { + TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP, +} TransportProto; + +typedef enum { IPv4, IPv6 } IPFamily; + #include #include @@ -21,7 +28,6 @@ #include #include "util.h" -#include "IPAddr.h" #ifdef HAVE_NETINET_IP6_H #include @@ -58,11 +64,12 @@ inline int seq_delta(uint32 a, uint32 b) return int(a-b); } +class IPAddr; + // Returns the ones-complement checksum of a chunk of b short-aligned bytes. extern int ones_complement_checksum(const void* p, int b, uint32 sum); extern int ones_complement_checksum(const IPAddr& a, uint32 sum); -extern int tcp_checksum(const struct ip* ip, const struct tcphdr* tp, int len); extern int udp_checksum(const struct ip* ip, const struct udphdr* up, int len); extern int udp6_checksum(const struct ip6_hdr* ip, const struct udphdr* up, int len); diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc new file mode 100644 index 0000000000..e590b13434 --- /dev/null +++ b/src/threading/BasicThread.cc @@ -0,0 +1,174 @@ + +#include +#include + +#include "config.h" +#include "BasicThread.h" +#include "Manager.h" + +#ifdef HAVE_LINUX +#include +#endif + +using namespace threading; + +uint64_t BasicThread::thread_counter = 0; + +BasicThread::BasicThread() + { + started = false; + terminating = false; + pthread = 0; + + buf_len = 2048; + buf = (char*) malloc(buf_len); + + name = Fmt("thread-%d", ++thread_counter); + + thread_mgr->AddThread(this); + } + +BasicThread::~BasicThread() + { + if ( buf ) + free(buf); + } + +void BasicThread::SetName(const string& arg_name) + { + // Slight race condition here with reader threads, but shouldn't matter. + name = arg_name; + } + +void BasicThread::SetOSName(const string& name) + { +#ifdef HAVE_LINUX + prctl(PR_SET_NAME, name.c_str(), 0, 0, 0); +#endif + +#ifdef __APPLE__ + pthread_setname_np(name.c_str()); +#endif + +#ifdef FREEBSD + pthread_set_name_np(pthread_self(), name, name.c_str()); +#endif + } + +const char* BasicThread::Fmt(const char* format, ...) + { + va_list al; + va_start(al, format); + int n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + + if ( (unsigned int) n >= buf_len ) + { // Not enough room, grow the buffer. + int tmp_len = n + 32; + char* tmp = (char*) malloc(tmp_len); + + // Is it portable to restart? + va_start(al, format); + n = safe_vsnprintf(tmp, tmp_len, format, al); + va_end(al); + + free(tmp); + } + + return buf; + } + +void BasicThread::Start() + { + if ( started ) + return; + + if ( pthread_mutex_init(&terminate, 0) != 0 ) + reporter->FatalError("Cannot create terminate mutex for thread %s", name.c_str()); + + // We use this like a binary semaphore and acquire it immediately. + if ( pthread_mutex_lock(&terminate) != 0 ) + reporter->FatalError("Cannot aquire terminate mutex for thread %s", name.c_str()); + + if ( pthread_create(&pthread, 0, BasicThread::launcher, this) != 0 ) + reporter->FatalError("Cannot create thread %s", name.c_str()); + + DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str()); + + started = true; + + OnStart(); + } + +void BasicThread::Stop() + { + if ( ! started ) + return; + + if ( terminating ) + return; + + DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str()); + + // Signal that it's ok for the thread to exit now by unlocking the + // mutex. + if ( pthread_mutex_unlock(&terminate) != 0 ) + reporter->FatalError("Failure flagging terminate condition for thread %s", name.c_str()); + + terminating = true; + + OnStop(); + } + +void BasicThread::Join() + { + if ( ! started ) + return; + + if ( ! terminating ) + Stop(); + + DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str()); + + if ( pthread_join(pthread, 0) != 0 ) + reporter->FatalError("Failure joining thread %s", name.c_str()); + + pthread_mutex_destroy(&terminate); + + DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str()); + + pthread = 0; + } + +void BasicThread::Kill() + { + if ( ! (started && pthread) ) + return; + + // I believe this is safe to call from a signal handler ... Not error + // checking so that killing doesn't bail out if we have already + // terminated. + pthread_kill(pthread, SIGKILL); + } + +void* BasicThread::launcher(void *arg) + { + BasicThread* thread = (BasicThread *)arg; + + // Block signals in thread. We handle signals only in the main + // process. + sigset_t mask_set; + sigfillset(&mask_set); + int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0); + assert(res == 0); // + + // Run thread's main function. + thread->Run(); + + // Wait until somebody actually wants us to terminate. + if ( pthread_mutex_lock(&thread->terminate) != 0 ) + reporter->FatalError("Failure acquiring terminate mutex at end of thread %s", thread->Name().c_str()); + + return 0; + } + diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h new file mode 100644 index 0000000000..cc87ae03bc --- /dev/null +++ b/src/threading/BasicThread.h @@ -0,0 +1,170 @@ + +#ifndef THREADING_BASICTHREAD_H +#define THREADING_BASICTHREAD_H + +#include +#include + +#include "Queue.h" +#include "util.h" + +using namespace std; + +namespace threading { + +class Manager; + +/** + * Base class for all threads. + * + * This class encapsulates all the OS-level thread handling. All thread + * instances are automatically added to the threading::Manager for management. The + * manager also takes care of deleting them (which must not be done + * manually). + */ +class BasicThread +{ +public: + /** + * Creates a new thread object. Instantiating the object does however + * not yet start the actual OS thread, that requires calling Start(). + * + * Only Bro's main thread may create new thread instances. + * + * @param name A descriptive name for thread the thread. This may + * show up in messages to the user. + */ + BasicThread(); + + /** + * Returns a descriptive name for the thread. If not set via + * SetName(). If not set, a default name is choosen automatically. + * + * This method is safe to call from any thread. + */ + const string& Name() const { return name; } + + /** + * Sets a descriptive name for the thread. This should be a string + * that's useful in output presented to the user and uniquely + * identifies the thread. + * + * This method must be called only from the thread itself. + */ + void SetName(const string& name); + + /** + * Set the name shown by the OS as the thread's description. Not + * supported on all OSs. + */ + void SetOSName(const string& name); + + /** + * Starts the thread. Calling this methods will spawn a new OS thread + * executing Run(). Note that one can't restart a thread after a + * Stop(), doing so will be ignored. + * + * Only Bro's main thread must call this method. + */ + void Start(); + + /** + * Signals the thread to stop. The method lets Terminating() now + * return true. It does however not force the thread to terminate. + * It's up to the Run() method to to query Terminating() and exit + * eventually. + * + * Calling this method has no effect if Start() hasn't been executed + * yet. + * + * Only Bro's main thread must call this method. + */ + void Stop(); + + /** + * Returns true if Stop() has been called. + * + * This method is safe to call from any thread. + */ + bool Terminating() const { return terminating; } + + /** + * A version of fmt() that the thread can safely use. + * + * This is safe to call from Run() but must not be used from any + * other thread than the current one. + */ + const char* Fmt(const char* format, ...); + +protected: + friend class Manager; + + /** + * Entry point for the thread. This must be overridden by derived + * classes and will execute in a separate thread once Start() is + * called. The thread will not terminate before this method finishes. + * An implementation should regularly check Terminating() to see if + * exiting has been requested. + */ + virtual void Run() = 0; + + /** + * Executed with Start(). This is a hook into starting the thread. It + * will be called from Bro's main thread after the OS thread has been + * started. + */ + virtual void OnStart() {} + + /** + * Executed with Stop(). This is a hook into stopping the thread. It + * will be called from Bro's main thread after the thread has been + * signaled to stop. + */ + virtual void OnStop() {} + + /** + * Destructor. This will be called by the manager. + * + * Only Bro's main thread may delete thread instances. + * + */ + virtual ~BasicThread(); + + /** + * Waits until the thread's Run() method has finished and then joins + * it. This is called from the threading::Manager. + */ + void Join(); + + /** + * Kills the thread immediately. One still needs to call Join() + * afterwards. + * + * This is called from the threading::Manager and safe to execute + * during a signal handler. + */ + void Kill(); + +private: + // pthread entry function. + static void* launcher(void *arg); + + string name; + pthread_t pthread; + bool started; // Set to to true once running. + bool terminating; // Set to to true to signal termination. + + // Used as a semaphore to tell the pthread thread when it may + // terminate. + pthread_mutex_t terminate; + + // For implementing Fmt(). + char* buf; + unsigned int buf_len; + + static uint64_t thread_counter; +}; + +} + +#endif diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc new file mode 100644 index 0000000000..abdbf849b5 --- /dev/null +++ b/src/threading/Manager.cc @@ -0,0 +1,147 @@ + +#include "Manager.h" + +using namespace threading; + +Manager::Manager() + { + DBG_LOG(DBG_THREADING, "Creating thread manager ..."); + + did_process = true; + next_beat = 0; + terminating = false; + idle = true; + } + +Manager::~Manager() + { + if ( all_threads.size() ) + Terminate(); + } + +void Manager::Terminate() + { + DBG_LOG(DBG_THREADING, "Terminating thread manager ..."); + + terminating = true; + + // First process remaining thread output for the message threads. + do Process(); while ( did_process ); + + // Signal all to stop. + for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ ) + (*i)->Stop(); + + // Then join them all. + for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ ) + { + (*i)->Join(); + delete *i; + } + + all_threads.clear(); + msg_threads.clear(); + + idle = true; + closed = true; + terminating = false; + } + +void Manager::KillThreads() + { + DBG_LOG(DBG_THREADING, "Killing threads ..."); + + for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ ) + (*i)->Kill(); + } + +void Manager::AddThread(BasicThread* thread) + { + DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str()); + all_threads.push_back(thread); + idle = false; + } + +void Manager::AddMsgThread(MsgThread* thread) + { + DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name().c_str()); + msg_threads.push_back(thread); + } + +void Manager::GetFds(int* read, int* write, int* except) + { + } + +double Manager::NextTimestamp(double* network_time) + { +// fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat); + + if ( ::network_time && (did_process || ::network_time > next_beat || ! next_beat) ) + // If we had something to process last time (or out heartbeat + // is due or not set yet), we want to check for more asap. + return timer_mgr->Time(); + + return -1.0; + } + +void Manager::Process() + { + bool do_beat = false; + + if ( network_time && (network_time > next_beat || ! next_beat) ) + { + do_beat = true; + next_beat = ::network_time + HEART_BEAT_INTERVAL; + } + + did_process = false; + + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + { + MsgThread* t = *i; + + if ( do_beat ) + t->Heartbeat(); + + while ( t->HasOut() ) + { + Message* msg = t->RetrieveOut(); + + if ( msg->Process() ) + { + if ( network_time ) + did_process = true; + } + + else + { + string s = msg->Name() + " failed, terminating thread"; + reporter->Error("%s", s.c_str()); + t->Stop(); + } + + delete msg; + } + } + +// fprintf(stderr, "P %.6f %.6f do_beat=%d did_process=%d next_next=%.6f\n", network_time, timer_mgr->Time(), do_beat, (int)did_process, next_beat); + } + +const threading::Manager::msg_stats_list& threading::Manager::GetMsgThreadStats() + { + stats.clear(); + + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + { + MsgThread* t = *i; + + MsgThread::Stats s; + t->GetStats(&s); + + stats.push_back(std::make_pair(t->Name(),s)); + } + + return stats; + } + + diff --git a/src/threading/Manager.h b/src/threading/Manager.h new file mode 100644 index 0000000000..7d9ba766d4 --- /dev/null +++ b/src/threading/Manager.h @@ -0,0 +1,146 @@ + +#ifndef THREADING_MANAGER_H +#define THREADING_MANAGER_H + +#include + +#include "IOSource.h" + +#include "BasicThread.h" +#include "MsgThread.h" + +namespace threading { + +/** + * The thread manager coordinates all child threads. Once a BasicThread is + * instantitated, it gets addedd to the manager, which will delete it later + * once it has terminated. + * + * In addition to basic threads, the manager also provides additional + * functionality specific to MsgThread instances. In particular, it polls + * their outgoing message queue on a regular basis and feeds data sent into + * the rest of Bro. It also triggers the regular heartbeats. + */ +class Manager : public IOSource +{ +public: + /** + * Constructor. Only a single instance of the manager must be + * created. + */ + Manager(); + + /** + * Destructir. + */ + ~Manager(); + + /** + * Terminates the manager's processor. The method signals all threads + * to terminates and wait for them to do so. It then joins them and + * returns to the caller. Afterwards, no more thread instances may be + * created. + */ + void Terminate(); + + /** + * Returns True if we are currently in Terminate() waiting for + * threads to exit. + */ + bool Terminating() const { return terminating; } + + /** + * Immediately kills all child threads. It does however not yet join + * them, one still needs to call Terminate() for that. + * + * This method is safe to call from a signal handler, and can in fact + * be called while Terminate() is already in progress. + */ + void KillThreads(); + + typedef std::list > msg_stats_list; + + /** + * Returns statistics from all current MsgThread instances. + * + * @return A list of statistics, with one entry for each MsgThread. + * Each entry is a tuple of thread name and statistics. The list + * reference remains valid until the next call to this method (or + * termination of the manager). + */ + const msg_stats_list& GetMsgThreadStats(); + + /** + * Returns the number of currently active threads. This counts all + * threads that are not yet joined, includingt any potentially in + * Terminating() state. + */ + int NumThreads() const { return all_threads.size(); } + +protected: + friend class BasicThread; + friend class MsgThread; + + /** + * Registers a new basic thread with the manager. This is + * automatically called by the thread's constructor. + * + * @param thread The thread. + */ + void AddThread(BasicThread* thread); + + /** + * Registers a new message thread with the manager. This is + * automatically called by the thread's constructor. This must be + * called \a in \a addition to AddThread(BasicThread* thread). The + * MsgThread constructor makes sure to do so. + * + * @param thread The thread. + */ + void AddMsgThread(MsgThread* thread); + + /** + * Part of the IOSource interface. + */ + virtual void GetFds(int* read, int* write, int* except); + + /** + * Part of the IOSource interface. + */ + virtual double NextTimestamp(double* network_time); + + /** + * Part of the IOSource interface. + */ + virtual void Process(); + + /** + * Part of the IOSource interface. + */ + virtual const char* Tag() { return "threading::Manager"; } + +private: + static const int HEART_BEAT_INTERVAL = 1; + + typedef std::list all_thread_list; + all_thread_list all_threads; + + typedef std::list msg_thread_list; + msg_thread_list msg_threads; + + bool did_process; // True if the last Process() found some work to do. + double next_beat; // Timestamp when the next heartbeat will be sent. + bool terminating; // True if we are in Terminate(). + + msg_stats_list stats; +}; + +} + +/** + * A singleton instance of the thread manager. All methods must only be + * called from Bro's main thread. + */ +extern threading::Manager* thread_mgr; + +#endif diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc new file mode 100644 index 0000000000..ddcd3df1dd --- /dev/null +++ b/src/threading/MsgThread.cc @@ -0,0 +1,289 @@ + +#include "DebugLogger.h" + +#include "MsgThread.h" +#include "Manager.h" + +using namespace threading; + +namespace threading { + +////// Messages. + +// Signals child thread to terminate. This is actually a no-op; its only +// purpose is unblock the current read operation so that the child's Run() +// methods can check the termination status. +class TerminateMessage : public InputMessage +{ +public: + TerminateMessage(MsgThread* thread) : InputMessage("Terminate", thread) { } + + virtual bool Process() { return true; } +}; + +/// Sends a heartbeat to the child thread. +class HeartbeatMessage : public InputMessage +{ +public: + HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time) + : InputMessage("Heartbeat", thread) + { network_time = arg_network_time; current_time = arg_current_time; } + + virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); } + +private: + double network_time; + double current_time; +}; + +// A message from the child to be passed on to the Reporter. +class ReporterMessage : public OutputMessage +{ +public: + enum Type { + INFO, WARNING, ERROR, FATAL_ERROR, FATAL_ERROR_WITH_CORE, + INTERNAL_WARNING, INTERNAL_ERROR + }; + + ReporterMessage(Type arg_type, MsgThread* thread, const string& arg_msg) + : OutputMessage("ReporterMessage", thread) + { type = arg_type; msg = arg_msg; } + + virtual bool Process(); + +private: + string msg; + Type type; +}; + +#ifdef DEBUG +// A debug message from the child to be passed on to the DebugLogger. +class DebugMessage : public OutputMessage +{ +public: + DebugMessage(DebugStream arg_stream, MsgThread* thread, const string& arg_msg) + : OutputMessage("DebugMessage", thread) + { stream = arg_stream; msg = arg_msg; } + + virtual bool Process() + { + string s = Object()->Name() + ": " + msg; + debug_logger.Log(stream, "%s", s.c_str()); + return true; + } +private: + string msg; + DebugStream stream; +}; +#endif + +} + +////// Methods. + +Message::~Message() + { + } + +bool ReporterMessage::Process() + { + string s = Object()->Name() + ": " + msg; + const char* cmsg = s.c_str(); + + switch ( type ) { + + case INFO: + reporter->Info("%s", cmsg); + break; + + case WARNING: + reporter->Warning("%s", cmsg); + break; + + case ERROR: + reporter->Error("%s", cmsg); + break; + + case FATAL_ERROR: + reporter->FatalError("%s", cmsg); + break; + + case FATAL_ERROR_WITH_CORE: + reporter->FatalErrorWithCore("%s", cmsg); + break; + + case INTERNAL_WARNING: + reporter->InternalWarning("%s", cmsg); + break; + + case INTERNAL_ERROR : + reporter->InternalError("%s", cmsg); + break; + + default: + reporter->InternalError("unknown ReporterMessage type %d", type); + } + + return true; + } + +MsgThread::MsgThread() : BasicThread() + { + cnt_sent_in = cnt_sent_out = 0; + thread_mgr->AddMsgThread(this); + } + +void MsgThread::OnStop() + { + // This is to unblock the current queue read operation. + SendIn(new TerminateMessage(this), true); + } + +void MsgThread::Heartbeat() + { + SendIn(new HeartbeatMessage(this, network_time, current_time())); + } + +bool MsgThread::DoHeartbeat(double network_time, double current_time) + { + string n = Name(); + + n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", n.c_str(), + cnt_sent_in - queue_in.Size(), + cnt_sent_out - queue_out.Size()); + + SetOSName(n.c_str()); + + return true; + } + +void MsgThread::Info(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::INFO, this, msg)); + } + +void MsgThread::Warning(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::WARNING, this, msg)); + } + +void MsgThread::Error(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::ERROR, this, msg)); + } + +void MsgThread::FatalError(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::FATAL_ERROR, this, msg)); + } + +void MsgThread::FatalErrorWithCore(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::FATAL_ERROR_WITH_CORE, this, msg)); + } + +void MsgThread::InternalWarning(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::INTERNAL_WARNING, this, msg)); + } + +void MsgThread::InternalError(const char* msg) + { + SendOut(new ReporterMessage(ReporterMessage::INTERNAL_ERROR, this, msg)); + } + +#ifdef DEBUG + +void MsgThread::Debug(DebugStream stream, const char* msg) + { + SendOut(new DebugMessage(stream, this, msg)); + } + +#endif + +void MsgThread::SendIn(BasicInputMessage* msg, bool force) + { + if ( Terminating() && ! force ) + { + delete msg; + return; + } + + DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str()); + + queue_in.Put(msg); + ++cnt_sent_in; + } + + +void MsgThread::SendOut(BasicOutputMessage* msg, bool force) + { + if ( Terminating() && ! force ) + { + delete msg; + return; + } + + queue_out.Put(msg); + + ++cnt_sent_out; + } + +BasicOutputMessage* MsgThread::RetrieveOut() + { + BasicOutputMessage* msg = queue_out.Get(); + assert(msg); + + DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str()); + + return msg; + } + +BasicInputMessage* MsgThread::RetrieveIn() + { + BasicInputMessage* msg = queue_in.Get(); + assert(msg); + +#ifdef DEBUG + string s = Fmt("Retrieved '%s' in %s", msg->Name().c_str(), Name().c_str()); + Debug(DBG_THREADING, s.c_str()); +#endif + + return msg; + } + +void MsgThread::Run() + { + while ( true ) + { + // When requested to terminate, we only do so when + // all input has been processed. + if ( Terminating() && ! queue_in.Ready() ) + break; + + BasicInputMessage* msg = RetrieveIn(); + + bool result = msg->Process(); + + if ( ! result ) + { + string s = msg->Name() + " failed, terminating thread"; + Error(s.c_str()); + Stop(); + break; + } + + delete msg; + } + } + +void MsgThread::GetStats(Stats* stats) + { + stats->sent_in = cnt_sent_in; + stats->sent_out = cnt_sent_out; + stats->pending_in = queue_in.Size(); + stats->pending_out = queue_out.Size(); + queue_in.GetStats(&stats->queue_in_stats); + queue_out.GetStats(&stats->queue_out_stats); + } + diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h new file mode 100644 index 0000000000..5ac1c0f780 --- /dev/null +++ b/src/threading/MsgThread.h @@ -0,0 +1,402 @@ + +#ifndef THREADING_MSGTHREAD_H +#define THREADING_MSGTHREAD_H + +#include + +#include "DebugLogger.h" + +#include "BasicThread.h" +#include "Queue.h" + +namespace threading { + +class BasicInputMessage; +class BasicOutputMessage; +class HeartbeatMessage; + +/** + * A specialized thread that provides bi-directional message passing between + * Bro's main thread and the child thread. Messages are instances of + * BasicInputMessage and BasicOutputMessage for message sent \a to the child + * thread and received \a from the child thread, respectively. + * + * The thread's Run() method implements main loop that processes incoming + * messages until Terminating() indicates that execution should stop. Once + * that happens, the thread stops accepting any new messages, finishes + * processes all remaining ones still in the queue, and then exits. + */ +class MsgThread : public BasicThread +{ +public: + /** + * Constructor. It automatically registers the thread with the + * threading::Manager. + * + * Only Bro's main thread may instantiate a new thread. + */ + MsgThread(); + + /** + * Sends a message to the child thread. The message will be proceesed + * once the thread has retrieved it from its incoming queue. + * + * Only the main thread may call this method. + * + * @param msg The message. + */ + void SendIn(BasicInputMessage* msg) { return SendIn(msg, false); } + + /** + * Sends a message from the child thread to the main thread. + * + * Only the child thread may call this method. + * + * @param msg The mesasge. + */ + void SendOut(BasicOutputMessage* msg) { return SendOut(msg, false); } + + /** + * Reports an informational message from the child thread. The main + * thread will pass this to the Reporter once received. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ + void Info(const char* msg); + + /** + * Reports a warning from the child thread that may indicate a + * problem. The main thread will pass this to the Reporter once + * received. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ + void Warning(const char* msg); + + /** + * Reports a non-fatal error from the child thread. The main thread + * will pass this to the Reporter once received. Processing proceeds + * normally after the error has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ + void Error(const char* msg); + + /** + * Reports a fatal error from the child thread. The main thread will + * pass this to the Reporter once received. Bro will terminate after + * the message has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ + void FatalError(const char* msg); + + /** + * Reports a fatal error from the child thread. The main thread will + * pass this to the Reporter once received. Bro will terminate with a + * core dump after the message has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ + void FatalErrorWithCore(const char* msg); + + /** + * Reports a potential internal problem from the child thread. The + * main thread will pass this to the Reporter once received. Bro will + * continue normally. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ + void InternalWarning(const char* msg); + + /** + * Reports an internal program error from the child thread. The main + * thread will pass this to the Reporter once received. Bro will + * terminate with a core dump after the message has been reported. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ + void InternalError(const char* msg); + +#ifdef DEBUG + /** + * Records a debug message for the given stream from the child + * thread. The main thread will pass this to the DebugLogger once + * received. + * + * Only the child thread may call this method. + * + * @param msg The message. It will be prefixed with the thread's name. + */ + void Debug(DebugStream stream, const char* msg); +#endif + + /** + * Statistics about inter-thread communication. + */ + struct Stats + { + uint64_t sent_in; //! Number of messages sent to the child thread. + uint64_t sent_out; //! Number of messages sent from the child thread to the main thread + uint64_t pending_in; //! Number of messages sent to the child but not yet processed. + uint64_t pending_out; //! Number of messages sent from the child but not yet processed by the main thread. + + /// Statistics from our queues. + Queue::Stats queue_in_stats; + Queue::Stats queue_out_stats; + }; + + /** + * Returns statistics about the inter-thread communication. + * + * @param stats A pointer to a structure that will be filled with + * current numbers. + */ + void GetStats(Stats* stats); + +protected: + friend class Manager; + friend class HeartbeatMessage; + + /** + * Pops a message sent by the child from the child-to-main queue. + * + * This is method is called regularly by the threading::Manager. + * + * @return The message, wth ownership passed to caller. Returns null + * if the queue is empty. + */ + BasicOutputMessage* RetrieveOut(); + + /** + * Triggers a heartbeat message being sent to the client thread. + * + * This is method is called regularly by the threading::Manager. + * + * Can be overriden in derived classed to hook into the heart beat, + * but must call the parent implementation. Note that this method is + * always called by the main thread and must not access data of the + * child thread directly. See DoHeartbeat() if you want to do + * something on the child-side. + */ + virtual void Heartbeat(); + + /** + * Overriden from BasicThread. + * + */ + virtual void Run(); + virtual void OnStop(); + + /** + * Regulatly triggered for execution in the child thread. + * + * When overriding, one must call the parent class' implementation. + * + * network_time: The network_time when the heartbeat was trigger by + * the main thread. + * + * current_time: Wall clock when the heartbeat was trigger by the + * main thread. + */ + virtual bool DoHeartbeat(double network_time, double current_time); + +private: + /** + * Pops a message sent by the main thread from the main-to-chold + * queue. + * + * Must only be called by the child thread. + * + * @return The message, wth ownership passed to caller. Returns null + * if the queue is empty. + */ + BasicInputMessage* RetrieveIn(); + + /** + * Queues a message for the child. + * + * Must only be called by the main thread. + * + * @param msg The message. + * + * @param force: If true, the message will be queued even when we're already + * Terminating(). Normally, the message would be discarded in that + * case. + */ + void SendIn(BasicInputMessage* msg, bool force); + + /** + * Queues a message for the main thread. + * + * Must only be called by the child thread. + * + * @param msg The message. + * + * @param force: If true, the message will be queued even when we're already + * Terminating(). Normally, the message would be discarded in that + * case. + */ + void SendOut(BasicOutputMessage* msg, bool force); + + /** + * Returns true if there's at least one message pending for the child + * thread. + */ + bool HasIn() { return queue_in.Ready(); } + + /** + * Returns true if there's at least one message pending for the main + * thread. + */ + bool HasOut() { return queue_out.Ready(); } + + Queue queue_in; + Queue queue_out; + + uint64_t cnt_sent_in; // Counts message sent to child. + uint64_t cnt_sent_out; // Counts message sent by child. +}; + +/** + * Base class for all message between Bro's main process and a MsgThread. + */ +class Message +{ +public: + /** + * Destructor. + */ + virtual ~Message(); + + /** + * Returns a descriptive name for the message's general type. This is + * what's passed into the constructor and used mainly for debugging + * purposes. + */ + const string& Name() const { return name; } + + /** + * Callback that must be overriden for processing a message. + */ + virtual bool Process() = 0; // Thread will be terminated if returngin false. + +protected: + /** + * Constructor. + * + * @param arg_name A descriptive name for the type of message. Used + * mainly for debugging purposes. + */ + Message(const string& arg_name) { name = arg_name; } + +private: + string name; +}; + +/** + * Base class for messages sent from Bro's main thread to a child MsgThread. + */ +class BasicInputMessage : public Message +{ +protected: + /** + * Constructor. + * + * @param name A descriptive name for the type of message. Used + * mainly for debugging purposes. + */ + BasicInputMessage(const string& name) : Message(name) {} +}; + +/** + * Base class for messages sent from a child MsgThread to Bro's main thread. + */ +class BasicOutputMessage : public Message +{ +protected: + /** + * Constructor. + * + * @param name A descriptive name for the type of message. Used + * mainly for debugging purposes. + */ + BasicOutputMessage(const string& name) : Message(name) {} +}; + +/** + * A paremeterized InputMessage that stores a pointer to an argument object. + * Normally, the objects will be used from the Process() callback. + */ +template +class InputMessage : public BasicInputMessage +{ +public: + /** + * Returns the objects passed to the constructor. + */ + O* Object() const { return object; } + +protected: + /** + * Constructor. + * + * @param name: A descriptive name for the type of message. Used + * mainly for debugging purposes. + * + * @param arg_object: An object to store with the message. + */ + InputMessage(const string& name, O* arg_object) : BasicInputMessage(name) + { object = arg_object; } + +private: + O* object; +}; + +/** + * A paremeterized OututMessage that stores a pointer to an argument object. + * Normally, the objects will be used from the Process() callback. + */ +template +class OutputMessage : public BasicOutputMessage +{ +public: + /** + * Returns the objects passed to the constructor. + */ + O* Object() const { return object; } + +protected: + /** + * Constructor. + * + * @param name A descriptive name for the type of message. Used + * mainly for debugging purposes. + * + * @param arg_object An object to store with the message. + */ + OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name) + { object = arg_object; } + +private: + O* object; +}; + +} + + +#endif diff --git a/src/threading/Queue.h b/src/threading/Queue.h new file mode 100644 index 0000000000..985ba31714 --- /dev/null +++ b/src/threading/Queue.h @@ -0,0 +1,222 @@ + +#ifndef THREADING_QUEUE_H +#define THREADING_QUEUE_H + +#include +#include +#include +#include + +#include "Reporter.h" + +#undef Queue // Defined elsewhere unfortunately. + +namespace threading { + +/** + * A thread-safe single-reader single-writer queue. + * + * The implementation uses multiple queues and reads/writes in rotary fashion + * in an attempt to limit contention. + * + * All Queue instances must be instantiated by Bro's main thread. + * + * TODO: Unclear how critical performance is for this qeueue. We could like;y + * optimize it further if helpful. + */ +template +class Queue +{ +public: + /** + * Constructor. + */ + Queue(); + + /** + * Destructor. + */ + ~Queue(); + + /** + * Retrieves one elment. + */ + T Get(); + + /** + * Queues one element. + */ + void Put(T data); + + /** + * Returns true if the next Get() operation will succeed. + */ + bool Ready(); + + /** + * Returns the number of queued items not yet retrieved. + */ + uint64_t Size(); + + /** + * Statistics about inter-thread communication. + */ + struct Stats + { + uint64_t num_reads; //! Number of messages read from the queue. + uint64_t num_writes; //! Number of messages written to the queue. + }; + + /** + * Returns statistics about the queue's usage. + * + * @param stats A pointer to a structure that will be filled with + * current numbers. */ + void GetStats(Stats* stats); + +private: + static const int NUM_QUEUES = 8; + + pthread_mutex_t mutex[NUM_QUEUES]; // Mutex protected shared accesses. + pthread_cond_t has_data[NUM_QUEUES]; // Signals when data becomes available + std::queue messages[NUM_QUEUES]; // Actually holds the queued messages + + int read_ptr; // Where the next operation will read from + int write_ptr; // Where the next operation will write to + + // Statistics. + uint64_t num_reads; + uint64_t num_writes; +}; + +inline static void safe_lock(pthread_mutex_t* mutex) + { + if ( pthread_mutex_lock(mutex) != 0 ) + reporter->FatalErrorWithCore("cannot lock mutex"); + } + +inline static void safe_unlock(pthread_mutex_t* mutex) + { + if ( pthread_mutex_unlock(mutex) != 0 ) + reporter->FatalErrorWithCore("cannot unlock mutex"); + } + +template +inline Queue::Queue() + { + read_ptr = 0; + write_ptr = 0; + num_reads = num_writes = 0; + + for( int i = 0; i < NUM_QUEUES; ++i ) + { + if ( pthread_cond_init(&has_data[i], NULL) != 0 ) + reporter->FatalError("cannot init queue condition variable"); + + if ( pthread_mutex_init(&mutex[i], NULL) != 0 ) + reporter->FatalError("cannot init queue mutex"); + } + } + +template +inline Queue::~Queue() + { + for( int i = 0; i < NUM_QUEUES; ++i ) + { + pthread_cond_destroy(&has_data[i]); + pthread_mutex_destroy(&mutex[i]); + } + } + +template +inline T Queue::Get() + { + safe_lock(&mutex[read_ptr]); + + int old_read_ptr = read_ptr; + + if ( messages[read_ptr].empty() ) + pthread_cond_wait(&has_data[read_ptr], &mutex[read_ptr]); + + T data = messages[read_ptr].front(); + messages[read_ptr].pop(); + + read_ptr = (read_ptr + 1) % NUM_QUEUES; + ++num_reads; + + safe_unlock(&mutex[old_read_ptr]); + + return data; + } + +template +inline void Queue::Put(T data) + { + safe_lock(&mutex[write_ptr]); + + int old_write_ptr = write_ptr; + + bool need_signal = messages[write_ptr].empty(); + + messages[write_ptr].push(data); + + if ( need_signal ) + pthread_cond_signal(&has_data[write_ptr]); + + write_ptr = (write_ptr + 1) % NUM_QUEUES; + ++num_writes; + + safe_unlock(&mutex[old_write_ptr]); + } + + +template +inline bool Queue::Ready() + { + safe_lock(&mutex[read_ptr]); + + bool ret = (messages[read_ptr].size()); + + safe_unlock(&mutex[read_ptr]); + + return ret; + } + +template +inline uint64_t Queue::Size() + { + // Need to lock all queues. + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_lock(&mutex[i]); + + uint64_t size = 0; + + for ( int i = 0; i < NUM_QUEUES; i++ ) + size += messages[i].size(); + + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_unlock(&mutex[i]); + + return size; + } + +template +inline void Queue::GetStats(Stats* stats) + { + // To be safe, we look all queues. That's probably unneccessary, but + // doesn't really hurt. + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_lock(&mutex[i]); + + stats->num_reads = num_reads; + stats->num_writes = num_writes; + + for ( int i = 0; i < NUM_QUEUES; i++ ) + safe_unlock(&mutex[i]); + } + +} + + +#endif + diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc new file mode 100644 index 0000000000..a5692b2ffd --- /dev/null +++ b/src/threading/SerialTypes.cc @@ -0,0 +1,355 @@ +// See the file "COPYING" in the main distribution directory for copyright. + + +#include "SerialTypes.h" +#include "../RemoteSerializer.h" + + +using namespace threading; + +bool Field::Read(SerializationFormat* fmt) + { + int t; + int st; + + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); + type = (TypeTag) t; + subtype = (TypeTag) st; + + return success; + } + +bool Field::Write(SerializationFormat* fmt) const + { + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); + } + +Value::~Value() + { + if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC) + && present ) + delete val.string_val; + + if ( type == TYPE_TABLE && present ) + { + for ( int i = 0; i < val.set_val.size; i++ ) + delete val.set_val.vals[i]; + + delete [] val.set_val.vals; + } + + if ( type == TYPE_VECTOR && present ) + { + for ( int i = 0; i < val.vector_val.size; i++ ) + delete val.vector_val.vals[i]; + + delete [] val.vector_val.vals; + } + } + +bool Value::IsCompatibleType(BroType* t, bool atomic_only) + { + if ( ! t ) + return false; + + switch ( t->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_SUBNET: + case TYPE_ADDR: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + return true; + + case TYPE_RECORD: + return ! atomic_only; + + case TYPE_TABLE: + { + if ( atomic_only ) + return false; + + if ( ! t->IsSet() ) + return false; + + return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); + } + + case TYPE_VECTOR: + { + if ( atomic_only ) + return false; + + return IsCompatibleType(t->AsVectorType()->YieldType(), true); + } + + default: + return false; + } + + return false; + } + +bool Value::Read(SerializationFormat* fmt) + { + int ty; + + if ( ! (fmt->Read(&ty, "type") && fmt->Read(&present, "present")) ) + return false; + + type = (TypeTag)(ty); + + if ( ! present ) + return true; + + switch ( type ) { + case TYPE_BOOL: + case TYPE_INT: + return fmt->Read(&val.int_val, "int"); + + case TYPE_COUNT: + case TYPE_COUNTER: + return fmt->Read(&val.uint_val, "uint"); + + case TYPE_PORT: { + int proto; + if ( ! (fmt->Read(&val.port_val.port, "port") && fmt->Read(&proto, "proto") ) ) { + return false; + } + + switch ( proto ) { + case 0: + val.port_val.proto = TRANSPORT_UNKNOWN; + break; + case 1: + val.port_val.proto = TRANSPORT_TCP; + break; + case 2: + val.port_val.proto = TRANSPORT_UDP; + break; + case 3: + val.port_val.proto = TRANSPORT_ICMP; + break; + default: + return false; + } + + return true; + } + + case TYPE_ADDR: + { + char family; + + if ( ! fmt->Read(&family, "addr-family") ) + return false; + + switch ( family ) { + case 4: + val.addr_val.family = IPv4; + return fmt->Read(&val.addr_val.in.in4, "addr-in4"); + + case 6: + val.addr_val.family = IPv6; + return fmt->Read(&val.addr_val.in.in6, "addr-in6"); + + } + + // Can't be reached. + abort(); + } + + case TYPE_SUBNET: + { + char length; + char family; + + if ( ! (fmt->Read(&length, "subnet-len") && fmt->Read(&family, "subnet-family")) ) + return false; + + switch ( family ) { + case 4: + val.subnet_val.length = (uint8_t)length; + val.subnet_val.prefix.family = IPv4; + return fmt->Read(&val.subnet_val.prefix.in.in4, "subnet-in4"); + + case 6: + val.subnet_val.length = (uint8_t)length; + val.subnet_val.prefix.family = IPv6; + return fmt->Read(&val.subnet_val.prefix.in.in6, "subnet-in6"); + + } + + // Can't be reached. + abort(); + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return fmt->Read(&val.double_val, "double"); + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + { + val.string_val = new string; + return fmt->Read(val.string_val, "string"); + } + + case TYPE_TABLE: + { + if ( ! fmt->Read(&val.set_val.size, "set_size") ) + return false; + + val.set_val.vals = new Value* [val.set_val.size]; + + for ( int i = 0; i < val.set_val.size; ++i ) + { + val.set_val.vals[i] = new Value; + + if ( ! val.set_val.vals[i]->Read(fmt) ) + return false; + } + + return true; + } + + case TYPE_VECTOR: + { + if ( ! fmt->Read(&val.vector_val.size, "vector_size") ) + return false; + + val.vector_val.vals = new Value* [val.vector_val.size]; + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + val.vector_val.vals[i] = new Value; + + if ( ! val.vector_val.vals[i]->Read(fmt) ) + return false; + } + + return true; + } + + default: + reporter->InternalError("unsupported type %s in Value::Write", type_name(type)); + } + + return false; + } + +bool Value::Write(SerializationFormat* fmt) const + { + if ( ! (fmt->Write((int)type, "type") && + fmt->Write(present, "present")) ) + return false; + + if ( ! present ) + return true; + + switch ( type ) { + case TYPE_BOOL: + case TYPE_INT: + return fmt->Write(val.int_val, "int"); + + case TYPE_COUNT: + case TYPE_COUNTER: + return fmt->Write(val.uint_val, "uint"); + + case TYPE_PORT: + return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); + + case TYPE_ADDR: + { + switch ( val.addr_val.family ) { + case IPv4: + return fmt->Write((char)4, "addr-family") + && fmt->Write(val.addr_val.in.in4, "addr-in4"); + + case IPv6: + return fmt->Write((char)6, "addr-family") + && fmt->Write(val.addr_val.in.in6, "addr-in6"); + break; + } + + // Can't be reached. + abort(); + } + + case TYPE_SUBNET: + { + if ( ! fmt->Write((char)val.subnet_val.length, "subnet-length") ) + return false; + + switch ( val.subnet_val.prefix.family ) { + case IPv4: + return fmt->Write((char)4, "subnet-family") + && fmt->Write(val.subnet_val.prefix.in.in4, "subnet-in4"); + + case IPv6: + return fmt->Write((char)6, "subnet-family") + && fmt->Write(val.subnet_val.prefix.in.in6, "subnet-in6"); + break; + } + + // Can't be reached. + abort(); + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return fmt->Write(val.double_val, "double"); + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + return fmt->Write(*val.string_val, "string"); + + case TYPE_TABLE: + { + if ( ! fmt->Write(val.set_val.size, "set_size") ) + return false; + + for ( int i = 0; i < val.set_val.size; ++i ) + { + if ( ! val.set_val.vals[i]->Write(fmt) ) + return false; + } + + return true; + } + + case TYPE_VECTOR: + { + if ( ! fmt->Write(val.vector_val.size, "vector_size") ) + return false; + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + if ( ! val.vector_val.vals[i]->Write(fmt) ) + return false; + } + + return true; + } + + default: + reporter->InternalError("unsupported type %s in Value::REad", type_name(type)); + } + + return false; + } + diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h new file mode 100644 index 0000000000..db7dc837bd --- /dev/null +++ b/src/threading/SerialTypes.h @@ -0,0 +1,146 @@ + +#ifndef THREADING_SERIALIZATIONTYPES_H +#define THREADING_SERIALIZATIONTYPES_H + +using namespace std; + +#include +#include +#include + +#include "Type.h" +#include "net_util.h" + +class SerializationFormat; + +namespace threading { + +/** + * Definition of a log file, i.e., one column of a log stream. + */ +struct Field { + string name; //! Name of the field. + TypeTag type; //! Type of the field. + TypeTag subtype; //! Inner type for sets. + + /** + * Constructor. + */ + Field() { subtype = TYPE_VOID; } + + /** + * Copy constructor. + */ + Field(const Field& other) + : name(other.name), type(other.type), subtype(other.subtype) { } + + /** + * Unserializes a field. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ + bool Read(SerializationFormat* fmt); + + /** + * Serializes a field. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ + bool Write(SerializationFormat* fmt) const; +}; + +/** + * Definition of a log value, i.e., a entry logged by a stream. + * + * This struct essentialy represents a serialization of a Val instance (for + * those Vals supported). + */ +struct Value { + TypeTag type; //! The type of the value. + bool present; //! False for optional record fields that are not set. + + struct set_t { bro_int_t size; Value** vals; }; + typedef set_t vec_t; + struct port_t { bro_uint_t port; TransportProto proto; }; + + struct addr_t { + IPFamily family; + union { + struct in_addr in4; + struct in6_addr in6; + } in; + }; + + struct subnet_t { addr_t prefix; uint8_t length; }; + + /** + * This union is a subset of BroValUnion, including only the types we + * can log directly. See IsCompatibleType(). + */ + union _val { + bro_int_t int_val; + bro_uint_t uint_val; + port_t port_val; + double double_val; + set_t set_val; + vec_t vector_val; + addr_t addr_val; + subnet_t subnet_val; + string* string_val; + } val; + + /** + * Constructor. + * + * arg_type: The type of the value. + * + * arg_present: False if the value represents an optional record field + * that is not set. + */ + Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) + : type(arg_type), present(arg_present) {} + + /** + * Destructor. + */ + ~Value(); + + /** + * Unserializes a value. + * + * @param fmt The serialization format to use. The format handles low-level I/O. + * + * @return False if an error occured. + */ + bool Read(SerializationFormat* fmt); + + /** + * Serializes a value. + * + * @param fmt The serialization format to use. The format handles + * low-level I/O. + * + * @return False if an error occured. + */ + bool Write(SerializationFormat* fmt) const; + + /** + * Returns true if the type can be represented by a Value. If + * `atomic_only` is true, will not permit composite types. + */ + static bool IsCompatibleType(BroType* t, bool atomic_only=false); + +private: +friend class ::IPAddr; + Value(const Value& other) { } // Disabled. +}; + +} + +#endif /* THREADING_SERIALIZATIONTZPES_H */ diff --git a/src/util.h b/src/util.h index 498bdf00e4..a4e3aa71b8 100644 --- a/src/util.h +++ b/src/util.h @@ -37,7 +37,7 @@ #endif -#ifdef USE_PERFTOOLS +#ifdef USE_PERFTOOLS_DEBUG #include #include extern HeapLeakChecker* heap_checker; diff --git a/testing/btest/Baseline/bifs.routing0_data_to_addrs/output b/testing/btest/Baseline/bifs.routing0_data_to_addrs/output index 7179bf8564..c79aef89d0 100644 --- a/testing/btest/Baseline/bifs.routing0_data_to_addrs/output +++ b/testing/btest/Baseline/bifs.routing0_data_to_addrs/output @@ -1,4 +1 @@ -{ -2001:78:1:32::1, -2001:78:1:32::2 -} +[2001:78:1:32::1, 2001:78:1:32::2] diff --git a/testing/btest/Baseline/core.ipv6-atomic-frag/output b/testing/btest/Baseline/core.ipv6-atomic-frag/output new file mode 100644 index 0000000000..4a628a4bdc --- /dev/null +++ b/testing/btest/Baseline/core.ipv6-atomic-frag/output @@ -0,0 +1,4 @@ +[orig_h=2001:db8:1::2, orig_p=36951/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp] +[orig_h=2001:db8:1::2, orig_p=59694/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp] +[orig_h=2001:db8:1::2, orig_p=27393/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp] +[orig_h=2001:db8:1::2, orig_p=45805/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp] diff --git a/testing/btest/Baseline/core.ipv6_ext_headers/output b/testing/btest/Baseline/core.ipv6_ext_headers/output index a5a0caf7c6..58332ca900 100644 --- a/testing/btest/Baseline/core.ipv6_ext_headers/output +++ b/testing/btest/Baseline/core.ipv6_ext_headers/output @@ -1 +1 @@ -[ip=, ip6=[class=0, flow=0, len=59, nxt=0, hlim=64, src=2001:4f8:4:7:2e0:81ff:fe52:ffff, dst=2001:4f8:4:7:2e0:81ff:fe52:9a6b, exts=[[id=0, hopopts=[nxt=43, len=0, options=[[otype=1, len=4, data=\0\0\0\0]]], dstopts=, routing=, fragment=, ah=, esp=], [id=43, hopopts=, dstopts=, routing=[nxt=17, len=4, rtype=0, segleft=2, data=\0\0\0\0 ^A\0x\0^A\02\0\0\0\0\0\0\0^A ^A\0x\0^A\02\0\0\0\0\0\0\0^B], fragment=, ah=, esp=]]], tcp=, udp=[sport=53/udp, dport=53/udp, ulen=11], icmp=] +[ip=, ip6=[class=0, flow=0, len=68, nxt=0, hlim=64, src=2001:4f8:4:7:2e0:81ff:fe52:ffff, dst=2001:4f8:4:7:2e0:81ff:fe52:9a6b, exts=[[id=0, hopopts=[nxt=43, len=0, options=[[otype=1, len=4, data=\0\0\0\0]]], dstopts=, routing=, fragment=, ah=, esp=], [id=43, hopopts=, dstopts=, routing=[nxt=6, len=4, rtype=0, segleft=0, data=\0\0\0\0 ^A\0x\0^A\02\0\0\0\0\0\0\0^A ^A\0x\0^A\02\0\0\0\0\0\0\0^B], fragment=, ah=, esp=]]], tcp=[sport=30000/tcp, dport=80/tcp, seq=0, ack=0, hl=20, dl=0, flags=2, win=8192], udp=, icmp=] diff --git a/testing/btest/Baseline/core.ipv6_rh0/segleft.out b/testing/btest/Baseline/core.ipv6_rh0/segleft.out new file mode 100644 index 0000000000..3c722ee3b4 --- /dev/null +++ b/testing/btest/Baseline/core.ipv6_rh0/segleft.out @@ -0,0 +1,2 @@ +flow_weird routing0_segleft from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:78:1:32::2 +rh0 w/ segments left from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:4f8:4:7:2e0:81ff:fe52:9a6b diff --git a/testing/btest/Baseline/core.ipv6_rh0/segleft0.out b/testing/btest/Baseline/core.ipv6_rh0/segleft0.out new file mode 100644 index 0000000000..ae57c7cc8d --- /dev/null +++ b/testing/btest/Baseline/core.ipv6_rh0/segleft0.out @@ -0,0 +1,2 @@ +flow_weird routing0_header from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:4f8:4:7:2e0:81ff:fe52:9a6b +new_connection: [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/tcp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=80/tcp] diff --git a/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log new file mode 100644 index 0000000000..42fcd6a526 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path metrics +#fields ts metric_id filter_name index.host index.str index.network value +#types time enum string addr string subnet count +1331256494.591966 TEST_METRIC foo-bar 6.5.4.3 - - 4 +1331256494.591966 TEST_METRIC foo-bar 7.2.1.5 - - 2 +1331256494.591966 TEST_METRIC foo-bar 1.2.3.4 - - 6 diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log new file mode 100644 index 0000000000..5a26f322f4 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test.failure +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.log b/testing/btest/Baseline/core.leaks.remote/sender.test.log new file mode 100644 index 0000000000..9d2ba26f48 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.success.log b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log new file mode 100644 index 0000000000..1b2ed452a0 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log @@ -0,0 +1,9 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test.success +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/Traces/ext_hdr_hbh_routing.trace b/testing/btest/Traces/ipv6-hbh-rh0-segleft.trace similarity index 100% rename from testing/btest/Traces/ext_hdr_hbh_routing.trace rename to testing/btest/Traces/ipv6-hbh-rh0-segleft.trace diff --git a/testing/btest/Traces/ipv6-hbh-rh0-segleft0.trace b/testing/btest/Traces/ipv6-hbh-rh0-segleft0.trace new file mode 100644 index 0000000000..35f5b3afe6 Binary files /dev/null and b/testing/btest/Traces/ipv6-hbh-rh0-segleft0.trace differ diff --git a/testing/btest/Traces/ipv6-http-atomic-frag.trace b/testing/btest/Traces/ipv6-http-atomic-frag.trace new file mode 100644 index 0000000000..d5d9db276c Binary files /dev/null and b/testing/btest/Traces/ipv6-http-atomic-frag.trace differ diff --git a/testing/btest/bifs/routing0_data_to_addrs.test b/testing/btest/bifs/routing0_data_to_addrs.test index 4bf15cae87..de10dd80e0 100644 --- a/testing/btest/bifs/routing0_data_to_addrs.test +++ b/testing/btest/bifs/routing0_data_to_addrs.test @@ -1,7 +1,7 @@ -# @TEST-EXEC: bro -C -b -r $TRACES/ext_hdr_hbh_routing.trace %INPUT >output +# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft.trace %INPUT >output # @TEST-EXEC: btest-diff output -event ipv6_ext_headers(c: connection, p: pkt_hdr) +event rh0_segleft(p: pkt_hdr) { for ( h in p$ip6$exts ) if ( p$ip6$exts[h]$id == IPPROTO_ROUTING ) diff --git a/testing/btest/core/ipv6-atomic-frag.test b/testing/btest/core/ipv6-atomic-frag.test new file mode 100644 index 0000000000..0829d8e973 --- /dev/null +++ b/testing/btest/core/ipv6-atomic-frag.test @@ -0,0 +1,7 @@ +# @TEST-EXEC: bro -r $TRACES/ipv6-http-atomic-frag.trace %INPUT >output +# @TEST-EXEC: btest-diff output + +event new_connection(c: connection) + { + print c$id; + } diff --git a/testing/btest/core/ipv6_ext_headers.test b/testing/btest/core/ipv6_ext_headers.test index 170a67bc72..0cf3f2f3fb 100644 --- a/testing/btest/core/ipv6_ext_headers.test +++ b/testing/btest/core/ipv6_ext_headers.test @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -C -b -r $TRACES/ext_hdr_hbh_routing.trace %INPUT >output +# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft0.trace %INPUT >output # @TEST-EXEC: btest-diff output # Just check that the event is raised correctly for a packet containing diff --git a/testing/btest/core/ipv6_rh0.test b/testing/btest/core/ipv6_rh0.test new file mode 100644 index 0000000000..18c23ed3b7 --- /dev/null +++ b/testing/btest/core/ipv6_rh0.test @@ -0,0 +1,22 @@ +# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft0.trace %INPUT >segleft0.out +# @TEST-EXEC: btest-diff segleft0.out +# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft.trace %INPUT >segleft.out +# @TEST-EXEC: btest-diff segleft.out + +# This will be raised only by the packet with RH0 and segments left. +event rh0_segleft(p: pkt_hdr) + { + print fmt("rh0 w/ segments left from %s to %s", p$ip6$src, p$ip6$dst); + } + +# This will be raised only by the packet with RH0 and no segments left. +event new_connection(c: connection) + { + print fmt("new_connection: %s", c$id); + } + +# This will be raised by any packet with RH0 regardless of segments left. +event flow_weird(name: string, src: addr, dst: addr) + { + print fmt("flow_weird %s from %s to %s", name, src, dst); + } diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro new file mode 100644 index 0000000000..a82f52c8b2 --- /dev/null +++ b/testing/btest/core/leaks/basic-cluster.bro @@ -0,0 +1,39 @@ +# Needs perftools support. +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT +# @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run worker-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT +# @TEST-EXEC: btest-bg-run worker-2 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT +# @TEST-EXEC: btest-bg-wait -k 30 +# @TEST-EXEC: btest-diff manager-1/metrics.log + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")], + ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"], +}; +@TEST-END-FILE + +redef Log::default_rotation_interval = 0secs; + +redef enum Metrics::ID += { + TEST_METRIC, +}; + +event bro_init() &priority=5 + { + Metrics::add_filter(TEST_METRIC, + [$name="foo-bar", + $break_interval=3secs]); + + if ( Cluster::local_node_type() == Cluster::WORKER ) + { + Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); + Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); + Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); + } + } diff --git a/testing/btest/core/leaks/remote.bro b/testing/btest/core/leaks/remote.bro new file mode 100644 index 0000000000..fa72ce6024 --- /dev/null +++ b/testing/btest/core/leaks/remote.bro @@ -0,0 +1,79 @@ +# Needs perftools support. +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-wait -k 10 +# @TEST-EXEC: btest-diff sender/test.log +# @TEST-EXEC: btest-diff sender/test.failure.log +# @TEST-EXEC: btest-diff sender/test.success.log +# @TEST-EXEC: cmp receiver/test.log sender/test.log +# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log +# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log + +# This is the common part loaded by both sender and receiver. +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { LOG }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); + Log::add_filter(Test::LOG, [$name="f1", $path="test.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); +} + +##### + +@TEST-START-FILE sender.bro + +module Test; + +@load frameworks/communication/listen + +function fail(rec: Log): bool + { + return rec$status != "success"; + } + +event remote_connection_handshake_done(p: event_peer) + { + Log::add_filter(Test::LOG, [$name="f2", $path="test.failure", $pred=fail]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + + # Log something. + Log::write(Test::LOG, r); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + disconnect(p); + } +@TEST-END-FILE + +@TEST-START-FILE receiver.bro + +##### + +redef Communication::nodes += { + ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T] +}; + +@TEST-END-FILE diff --git a/testing/external/scripts/perftools-adapt-paths b/testing/external/scripts/perftools-adapt-paths index 2eda2477c7..cfecd39993 100755 --- a/testing/external/scripts/perftools-adapt-paths +++ b/testing/external/scripts/perftools-adapt-paths @@ -7,4 +7,4 @@ cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1 -grep -q "No leaks found" $1 +grep -qv "detected leaks of" $1 diff --git a/testing/scripts/diff-canon-notice-policy b/testing/scripts/diff-canon-notice-policy new file mode 100755 index 0000000000..f05abaa103 --- /dev/null +++ b/testing/scripts/diff-canon-notice-policy @@ -0,0 +1,10 @@ +#! /usr/bin/awk -f +# +# A diff canonifier that removes the priorities in notice_policy.log. + +/^#/ && $2 == "notice_policy" { filter = 1; } + +filter == 1 && /^[^#]/ { sub("^[0-9]*", "X"); } + +{ print; } + diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external index 6796614362..e788a4a1bb 100755 --- a/testing/scripts/diff-canonifier-external +++ b/testing/scripts/diff-canonifier-external @@ -6,4 +6,6 @@ | `dirname $0`/diff-remove-uids \ | `dirname $0`/diff-remove-mime-types \ | `dirname $0`/diff-remove-x509-names \ + | `dirname $0`/diff-canon-notice-policy \ | `dirname $0`/diff-sort +