From 574bcb0a51b18d1e209f75f89bbe8ee4b9e6306a Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 21 Jul 2015 11:57:16 -0700 Subject: [PATCH 01/53] Add simple XMPP StartTLS analyzer. This is a very simple XMPP analyzer that basically only can parse the protocol until the client and server start negotiating a TLS session. At that point, the TLS analyzer is attached. While the basic case seems to be working, I fully expect that I missed something and that this might break in a lot of cases. --- scripts/base/init-default.bro | 1 + scripts/base/protocols/xmpp/README | 5 + scripts/base/protocols/xmpp/__load__.bro | 1 + scripts/base/protocols/xmpp/main.bro | 11 +++ src/analyzer/protocol/CMakeLists.txt | 1 + src/analyzer/protocol/xmpp/CMakeLists.txt | 11 +++ src/analyzer/protocol/xmpp/Plugin.cc | 26 ++++++ src/analyzer/protocol/xmpp/XMPP.cc | 87 ++++++++++++++++++ src/analyzer/protocol/xmpp/XMPP.h | 38 ++++++++ src/analyzer/protocol/xmpp/xmpp-analyzer.pac | 41 +++++++++ src/analyzer/protocol/xmpp/xmpp-protocol.pac | 17 ++++ src/analyzer/protocol/xmpp/xmpp.pac | 35 +++++++ .../conn.log | 10 ++ .../ssl.log | 10 ++ .../x509.log | 11 +++ testing/btest/Traces/tls/xmpp-starttls.pcap | Bin 0 -> 8174 bytes .../scripts/base/protocols/xmpp/starttls.test | 9 ++ 17 files changed, 314 insertions(+) create mode 100644 scripts/base/protocols/xmpp/README create mode 100644 scripts/base/protocols/xmpp/__load__.bro create mode 100644 scripts/base/protocols/xmpp/main.bro create mode 100644 src/analyzer/protocol/xmpp/CMakeLists.txt create mode 100644 src/analyzer/protocol/xmpp/Plugin.cc create mode 100644 src/analyzer/protocol/xmpp/XMPP.cc create mode 100644 src/analyzer/protocol/xmpp/XMPP.h create mode 100644 src/analyzer/protocol/xmpp/xmpp-analyzer.pac create mode 100644 src/analyzer/protocol/xmpp/xmpp-protocol.pac create mode 100644 src/analyzer/protocol/xmpp/xmpp.pac create mode 100644 testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/conn.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/ssl.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/x509.log create mode 100644 testing/btest/Traces/tls/xmpp-starttls.pcap create mode 100644 testing/btest/scripts/base/protocols/xmpp/starttls.test diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 473d94fc84..7e921a6831 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -59,6 +59,7 @@ @load base/protocols/ssl @load base/protocols/syslog @load base/protocols/tunnels +@load base/protocols/xmpp @load base/files/pe @load base/files/hash diff --git a/scripts/base/protocols/xmpp/README b/scripts/base/protocols/xmpp/README new file mode 100644 index 0000000000..3d2194ef3d --- /dev/null +++ b/scripts/base/protocols/xmpp/README @@ -0,0 +1,5 @@ +Support for the Extensible Messaging and Presence Protocol (XMPP). + +Note that currently the XMPP analyzer only supports analyzing XMPP sessions +until they do or do not switch to TLS using StartTLS. Hence, we do not get +actual chat information from XMPP sessions, only X509 certificates. diff --git a/scripts/base/protocols/xmpp/__load__.bro b/scripts/base/protocols/xmpp/__load__.bro new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/protocols/xmpp/__load__.bro @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/protocols/xmpp/main.bro b/scripts/base/protocols/xmpp/main.bro new file mode 100644 index 0000000000..3d7a4cbc37 --- /dev/null +++ b/scripts/base/protocols/xmpp/main.bro @@ -0,0 +1,11 @@ + +module XMPP; + +const ports = { 5222/tcp, 5269/tcp }; +redef likely_server_ports += { ports }; + +event bro_init() &priority=5 + { + Analyzer::register_for_ports(Analyzer::ANALYZER_XMPP, ports); + } + diff --git a/src/analyzer/protocol/CMakeLists.txt b/src/analyzer/protocol/CMakeLists.txt index 467fce83ee..d19b2ac042 100644 --- a/src/analyzer/protocol/CMakeLists.txt +++ b/src/analyzer/protocol/CMakeLists.txt @@ -43,4 +43,5 @@ add_subdirectory(syslog) add_subdirectory(tcp) add_subdirectory(teredo) add_subdirectory(udp) +add_subdirectory(xmpp) add_subdirectory(zip) diff --git a/src/analyzer/protocol/xmpp/CMakeLists.txt b/src/analyzer/protocol/xmpp/CMakeLists.txt new file mode 100644 index 0000000000..408f01d47c --- /dev/null +++ b/src/analyzer/protocol/xmpp/CMakeLists.txt @@ -0,0 +1,11 @@ + +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro XMPP) +bro_plugin_cc(Plugin.cc) +bro_plugin_cc(XMPP.cc) +bro_plugin_pac(xmpp.pac xmpp-analyzer.pac xmpp-protocol.pac) +bro_plugin_end() + diff --git a/src/analyzer/protocol/xmpp/Plugin.cc b/src/analyzer/protocol/xmpp/Plugin.cc new file mode 100644 index 0000000000..b4332b447b --- /dev/null +++ b/src/analyzer/protocol/xmpp/Plugin.cc @@ -0,0 +1,26 @@ +// See the file in the main distribution directory for copyright. + + +#include "plugin/Plugin.h" + +#include "XMPP.h" + +namespace plugin { +namespace Bro_XMPP { + +class Plugin : public plugin::Plugin { +public: + plugin::Configuration Configure() + { + AddComponent(new ::analyzer::Component("XMPP", ::analyzer::xmpp::XMPP_Analyzer::Instantiate)); + + + plugin::Configuration config; + config.name = "Bro::XMPP"; + config.description = "XMPP analyzer StartTLS only"; + return config; + } +} plugin; + +} +} diff --git a/src/analyzer/protocol/xmpp/XMPP.cc b/src/analyzer/protocol/xmpp/XMPP.cc new file mode 100644 index 0000000000..c84c372c4d --- /dev/null +++ b/src/analyzer/protocol/xmpp/XMPP.cc @@ -0,0 +1,87 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "XMPP.h" +#include "analyzer/protocol/tcp/TCP_Reassembler.h" +#include "analyzer/Manager.h" + +using namespace analyzer::xmpp; + +XMPP_Analyzer::XMPP_Analyzer(Connection* conn) + : tcp::TCP_ApplicationAnalyzer("XMPP", conn) + { + interp = new binpac::XMPP::XMPP_Conn(this); + had_gap = false; + tls_active = false; + } + +XMPP_Analyzer::~XMPP_Analyzer() + { + delete interp; + } + +void XMPP_Analyzer::Done() + { + tcp::TCP_ApplicationAnalyzer::Done(); + + interp->FlowEOF(true); + interp->FlowEOF(false); + } + +void XMPP_Analyzer::EndpointEOF(bool is_orig) + { + tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig); + interp->FlowEOF(is_orig); + } + +void XMPP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) + { + tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig); + + if ( tls_active ) + { + // If TLS has been initiated, forward to child and abort further + // processing + ForwardStream(len, data, orig); + return; + } + + assert(TCP()); + if ( TCP()->IsPartial() ) + return; + + if ( had_gap ) + // If only one side had a content gap, we could still try to + // deliver data to the other side if the script layer can + // handle this. + return; + + try + { + interp->NewData(orig, data, data + len); + } + catch ( const binpac::Exception& e ) + { + printf("BinPAC Exception: %s\n", e.c_msg()); + ProtocolViolation(fmt("Binpac exception: %s", e.c_msg())); + } + } + +void XMPP_Analyzer::Undelivered(uint64 seq, int len, bool orig) + { + tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig); + had_gap = true; + interp->NewGap(orig, len); + } + +void XMPP_Analyzer::StartTLS() + { + // StartTLS was called. This means we saw a client starttls followed + // by a server proceed. From here on, everything should be a binary + // TLS datastream. + + tls_active = true; + + Analyzer* ssl = analyzer_mgr->InstantiateAnalyzer("SSL", Conn()); + if ( ssl ) + AddChildAnalyzer(ssl); + } diff --git a/src/analyzer/protocol/xmpp/XMPP.h b/src/analyzer/protocol/xmpp/XMPP.h new file mode 100644 index 0000000000..628be7bb2d --- /dev/null +++ b/src/analyzer/protocol/xmpp/XMPP.h @@ -0,0 +1,38 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef ANALYZER_PROTOCOL_XMPP_XMPP_H +#define ANALYZER_PROTOCOL_XMPP_XMPP_H + +#include "analyzer/protocol/tcp/TCP.h" + +#include "xmpp_pac.h" + +namespace analyzer { namespace xmpp { + +class XMPP_Analyzer : public tcp::TCP_ApplicationAnalyzer { +public: + XMPP_Analyzer(Connection* conn); + virtual ~XMPP_Analyzer(); + + virtual void Done(); + virtual void DeliverStream(int len, const u_char* data, bool orig); + virtual void Undelivered(uint64 seq, int len, bool orig); + + // Overriden from tcp::TCP_ApplicationAnalyzer. + virtual void EndpointEOF(bool is_orig); + + void StartTLS(); + + static analyzer::Analyzer* Instantiate(Connection* conn) + { return new XMPP_Analyzer(conn); } + +protected: + binpac::XMPP::XMPP_Conn* interp; + bool had_gap; + + bool tls_active; +}; + +} } // namespace analyzer::* + +#endif /* ANALYZER_PROTOCOL_XMPP_XMPP_H */ diff --git a/src/analyzer/protocol/xmpp/xmpp-analyzer.pac b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac new file mode 100644 index 0000000000..a4417e1601 --- /dev/null +++ b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac @@ -0,0 +1,41 @@ +refine connection XMPP_Conn += { + + %member{ + bool client_starttls; + %} + + %init{ + client_starttls = false; + %} + + function proc_xmpp_token(is_orig: bool, name: bytestring, rest: bytestring): bool + %{ + string token = std_str(name); + + if ( is_orig && token == "stream:stream" ) + // Yup, looks like xmpp... + bro_analyzer()->ProtocolConfirmation(); + + if ( token == "success" || token == "message" ) + // Handshake has passed the phase where we should see StartTLS. Simply skip from hereon... + bro_analyzer()->SetSkip(true); + + if ( is_orig && token == "starttls" ) + client_starttls = true; + + if ( !is_orig && token == "proceed" && client_starttls ) + { + bro_analyzer()->StartTLS(); + } + + //printf("Processed: %d %s %s \n", is_orig, c_str(name), c_str(rest)); + + return true; + %} + +}; + +refine typeattr XMPP_TOKEN += &let { + proc: bool = $context.connection.proc_xmpp_token(is_orig, name, rest); +}; + diff --git a/src/analyzer/protocol/xmpp/xmpp-protocol.pac b/src/analyzer/protocol/xmpp/xmpp-protocol.pac new file mode 100644 index 0000000000..e05268fe32 --- /dev/null +++ b/src/analyzer/protocol/xmpp/xmpp-protocol.pac @@ -0,0 +1,17 @@ +type XML_START = RE//; +type XML_NAME = RE/\/?[?:[:alnum:]]+/; +type XML_REST = RE/[^<>]*/; +type SPACING = RE/[ \r\n]*/; + +type XMPP_PDU(is_orig: bool) = XMPP_TOKEN(is_orig)[] &until($input.length() == 0); + +type XMPP_TOKEN(is_orig: bool) = record { + : SPACING; + : XML_START; + name: XML_NAME; + rest: XML_REST; + : XML_END; + : SPACING; +}; + diff --git a/src/analyzer/protocol/xmpp/xmpp.pac b/src/analyzer/protocol/xmpp/xmpp.pac new file mode 100644 index 0000000000..42ec85f0cc --- /dev/null +++ b/src/analyzer/protocol/xmpp/xmpp.pac @@ -0,0 +1,35 @@ +# binpac file for the XMPP analyzer. +# Note that we currently do not even try to parse the protocol +# completely -- this is only supposed to be able to parse xmpp +# till StartTLS does (or does not) kick in. + +%include binpac.pac +%include bro.pac + +%extern{ +namespace analyzer { namespace xmpp { class XMPP_Analyzer; } } +namespace binpac { namespace XMPP { class XMPP_Conn; } } +typedef analyzer::xmpp::XMPP_Analyzer* XMPPAnalyzer; + +#include "XMPP.h" +%} + +extern type XMPPAnalyzer; + +analyzer XMPP withcontext { + connection: XMPP_Conn; + flow: XMPP_Flow; +}; + +connection XMPP_Conn(bro_analyzer: XMPPAnalyzer) { + upflow = XMPP_Flow(true); + downflow = XMPP_Flow(false); +}; + +%include xmpp-protocol.pac + +flow XMPP_Flow(is_orig: bool) { + datagram = XMPP_PDU(is_orig) withcontext(connection, this); +}; + +%include xmpp-analyzer.pac diff --git a/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/conn.log b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/conn.log new file mode 100644 index 0000000000..2f5bd2f66d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/conn.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2015-07-21-18-55-16 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] +1437091701.732171 CXWv6p3arKYeMETxOg 198.128.203.95 56048 146.255.57.229 5222 tcp ssl,xmpp 2.213218 676 4678 SF - - 0 ShADadfFr 19 1676 15 5442 (empty) +#close 2015-07-21-18-55-16 diff --git a/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/ssl.log b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/ssl.log new file mode 100644 index 0000000000..f67ea92631 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/ssl.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-07-21-18-55-16 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1437091702.232293 CXWv6p3arKYeMETxOg 198.128.203.95 56048 146.255.57.229 5222 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T F5Nz2G1vSZQ0QXM2s8,FUw8omi2keRxShDUa (empty) CN=jabber.ccc.de,O=Chaos Computer Club e.V.,L=Hamburg,ST=Hamburg,C=DE emailAddress=support@cacert.org,CN=CA Cert Signing Authority,OU=http://www.cacert.org,O=Root CA - - +#close 2015-07-21-18-55-16 diff --git a/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/x509.log b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/x509.log new file mode 100644 index 0000000000..4a49298e8a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/x509.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path x509 +#open 2015-07-21-18-55-16 +#fields ts id certificate.version certificate.serial certificate.subject certificate.issuer certificate.not_valid_before certificate.not_valid_after certificate.key_alg certificate.sig_alg certificate.key_type certificate.key_length certificate.exponent certificate.curve san.dns san.uri san.email san.ip basic_constraints.ca basic_constraints.path_len +#types time string count string string string time time string string string count string string vector[string] vector[string] vector[string] vector[addr] bool count +1437091702.407347 F5Nz2G1vSZQ0QXM2s8 3 0DF4F2 CN=jabber.ccc.de,O=Chaos Computer Club e.V.,L=Hamburg,ST=Hamburg,C=DE emailAddress=support@cacert.org,CN=CA Cert Signing Authority,OU=http://www.cacert.org,O=Root CA 1382043019.000000 1445115019.000000 rsaEncryption sha1WithRSAEncryption rsa 2048 65537 - jabber.ccc.de,conference.jabber.ccc.de,jabberd.jabber.ccc.de,pubsub.jabber.ccc.de,vjud.jabber.ccc.de - - - F - +1437091702.407347 FUw8omi2keRxShDUa 3 00 emailAddress=support@cacert.org,CN=CA Cert Signing Authority,OU=http://www.cacert.org,O=Root CA emailAddress=support@cacert.org,CN=CA Cert Signing Authority,OU=http://www.cacert.org,O=Root CA 1049027389.000000 1995712189.000000 rsaEncryption md5WithRSAEncryption rsa 4096 65537 - - - - - T - +#close 2015-07-21-18-55-16 diff --git a/testing/btest/Traces/tls/xmpp-starttls.pcap b/testing/btest/Traces/tls/xmpp-starttls.pcap new file mode 100644 index 0000000000000000000000000000000000000000..b4a7ee61e10d771d4783cdc9f369b5d410c60f52 GIT binary patch literal 8174 zcmc&(c|4Tc|39-B3}fGlGDt%*^Vs)2kw`*myOuE++c1kFW%))$Dr=O=5|vb_v|de{ z7L}WP%ayKMT@o#p(53P_&sc8g-tX`G*VpTHUgkN^dEV#qd7t;^^LfsBns3$=V*wsa z{*8?RfPoi0r{msYi^aeS_#16eNkSPX!DhYd^)@#|fFl65HXgRXmQYwypV;7=HD=b) z+A9~t7%jMiB2{6>#*BLbz~Y6@VQ@Gh3`T^gYr1_1y@$t+qX*ZJfD61v+c@s{90GJp zI02v$-VLM3=nQf}t-T%P@0Y_F?iR_I?%N`ZqT{RZSP|lL)i*R09nK_-+T8|`?I9}K zX4!4HcspwMnJqKyz6;S^6Q!h-emMgf5ZIe?07XU-W6zSnMTmyBS%?)0VkqK2x|ZkU=5GFxVU>E7FRhM@1;sF{mHI4=s#kc{be+gSgIu1Q~~-2ZP$WCmJM3=9p;j7&@oO-(7}AT}#v67F{xfWfZN#=qW{#+iL!9l6?XLGsXoJkS!*pcQ;1~pSPV2p6M;7-}5Bdx2Bvvx;Hhkpv!O%z+wm{`b? zRqGd$4x_B{1)!FX@E7y7_zU=Qd?F|VrJxub0bw8%FhK|~1E#0QP)6zAj&zugRC-<3S;?1T>Hb?0_&|nlH>3=VO5o!1HDJB0vGi0x3Y^llVA5 z1oA)zNCI&n3dXRIDQ_SK@Hl|M!dog}E>Hw>;D2SH1b-_42|$AP(ZNKZ0o6hRWH6tA zAz%rxjtgOhaCi(Ji^t(GI4ll_#bB{m90bQ;(DlG;6F|2R5VV;p6KW$Wp(=YZqZ)i% z%NRX)2G*CjYA6o0^1{_U1y$CmfWrZ6ECA*>ocD33m|g$Qo}XJTf9&AN22b3w4epPM zetrE&xr?VoISMc>~1%078e5WT65i8JAQfFdRqv)Q^w| zqKi_UQ-?_yjF1pOV)djD2?EX=Pm~p2#bR;E4t9F;5f!vYOjdp(+o73@?oVg1xoBn9 zQ%A_?eVptZ2RpI@>?M0LgCm)d!DKrgH-yDza%1(>5Y?%PL@^3799~oui_NwD3kI35 zNYXRVL-b7a5hFy;Y#qEb`tA}5*AqiT(TNDM*d>mWc8bUfJNrij@Sq(NmJ9#YgU%}O zMadz=pT!|Nup**(Xs&b!=LL`%)YViy1w?M#c+x};AAfs5k}>MvbAu5AxMYkZfc->l zG6nS1I^*z1O)rV&k_}vT=Qw#Pc7ms4iUB4{O`&uZ|)=Z%SHyd2X}2>K|Hn zPqw)(Ut=cIFEzwzx*y+v_NgirtHq-aEo+_YWAvoL`Yv70Poe(KT)DUBSoZqQMJ`%~ z?`vCOOX+sE*J559blA3boxjWIg~zHNl%4_n9{;jqfrUZ%{T2F-DLPJwzOGI8PRX8N*`{AP~?zN%RU2gXB13h~Cx2>moV?5gh{5 znJ^AJ^$SA~^-Fceq<*%?V+cfe8;?mgA>wd|!Q>DO22VjW5sk@f1e2;bp+pv)6ZM@G zX^0*=l`4J_qJ=ChT2Q2(N}d==XNP}3kj;o;g)ykG7vZ3fNaMqi(B$bFNLHe=B7+!g zMkJjv`6fE9j%4|XB0Zy9E{Yex;RVd-m5B-E1y1k#URp5haLF()*=!cgXjP1pJFZ&y zawzP2SN3D8Dw7?E&ebq!za{M79&(~Bws-2rJ_-%*j7gE`c&~`f7BO+-@akCB$iC0p zwNs?>G<=6vHlKD)ao$>~690Hh=>ts3#nd`}a7oGB+8pytx3`UcFWmFaJ{cyz_cftlzNqYu+w@8D+1fM1pDifX9dB9ha~N zBiDu41#djdbSf&Rd?*Xwt&j6_wq7=D@5LRZouys%k2sr>Ql95=aMSrL>Q;mGef{fJ z>i+Wa9;MhhQ+DX0rBs7u^!7W_SJpTuonAw_9kgVdv9@%UZE)}V`qvwOS|ohDl&QCg z@2JLex;S>ppPGrv3zpXQ}X zIp@_YpGWvS(iFZ*>RHTs{B2o-PMi5#BU;>Z6Y7{g$1cfc%ps0Cwq!JH_=&f3#hvQP z?hkW30$0UGK2`ccx|0ip?T}<)E12~)$1^?#d@nrw|2*F#28e+HqNlHKW{Bo{_}f6= zY~u2NS&EXevy(p-P5xMz{9&j*Qs=)~y7is0mhj5F{vre0-OKwdf^76U!m!5ApT-aM z?cBR!ld+Ip?w_K)GL1`9Ier%+w{`J-0#84zI_0P+P9F&-c0FynX8k7rRQ z8^hIpJ52ud^1~o~6ZXik$l#raKKJ&#k@%_G`kP9z98o?zb^H3jpha;fmV3qy?kw5z zfkyP#$*9@pwz^FAU3Y8N;mRd;^L_jnLP4MTOMY(18MfahHp~dg_YmqG5X!{QQ&voV zcX!Q|n_+P~`>NOvG}n_e-zi{a_UgY?RC501jt~`#*k9Bv6biGcFhADg={(%kR4bg{ zrd)5BkZ?~WIZnt&M`ZngX~{{&jY0SFpBX;#)|9<9g!&?V`AMJ4w2bcDV?{T; zc9o4jPxnwiRCBXVx^8i1YI@%0Q{*RUw7E9*zj!cqs8!FwFcKV9b_~Y3Or9%7>(z^9J)TFbSFJFTlzNNqUts!dpyRL)LfdG;h1-d#bC4?K*(z zSWdonpnT2!On^x*=Bb&Sr;~!Mm%2xhgCE^~y7oEt_ui(4hnE8`?6d27LHOC3e1xeIMK?N5=;67js(MLN} z@j8e$LMhTL(nwW9D=7z_%}js&qkUIOk*Qja!Q(X){vS^m{`G_io}!@n!#7MCGcu5| znHm)mWdqMG^blnAUr&+n%M_fvSma;^mmJIDvB|7Wk!1AMlo`aN`*RuOAQqdvbd{qc zIfUUKNM^x9A$V3a`BpI9l^MxVP?fdrd?-xDUfz%>bKi(KFNw7)wBSL1h-AXCml-~} zXy$iB2>-CFyOEFX!R7={--8`I(@{QhB2h}R>6*pl9&ASz`reNs8ki73A`b5AV4Jy% zz_E#Q(OS}eYhxM-{c}Iz=Du~6=lzBX95AIjqY4iN96Gw=Ot23L zF}CPD<@}_9i%pHTex-v8=`#$lU25Ukt@WvLG$_|I&28gXi(mbo-H&BjR5P;5>ywf< z6h7>6!=#n9OM6$y-&Zx^C+A1`KM#%1E8Ro&vUXQHvF~GB^0l@u$5U?mI3kugj4FIV zOv04* z&40ONoi)b9B3-1aWFU9N+=hb>4^%&|iR#sLdEQd>n&s$hAT##OQh$-x@U7vr-klmZ zoW2FdMe`reFQ-=tZ#znRg4x@4M5nf1uSe|L25wmot*LtUg87)R$&k3b z>3-3eO8xG83U#+0=$nLad__h*hL?F73lVd%uKlrdrG{F}d4&V>`btPq!Kun;6$WV? zM|)yZ|Fjaj!4WGfsAPQ!zSuz%D_2^OvE{L|k+za;WwzhTywsKF;?}n8RTDeXn4DQ} zaV@M(IP=o})fyGG;W_or1j|1jt*H28h$hUDVk zf`*ZUN1~jlzsaRH*N{c?N&@>9sI(Ou70TB%U-hPLB@7lycz;^=G`ltClqSdZwgcSd zVcf(4W*H7B+Tp*+3>zf5EqyvcGf&S0^HrK|M_8YM$t1H(*9$Jqi`wb+{PB4+8i6H zubrBko7>vUHgGqKimS6_=K zfJdtN$#PNc#oob-_r$RTudmr@5Avf*FY7z#E2h}|sR|q-_VF#K-O5M*C1=(Whrhw~ zzWJoePp!2#{Dotj5|?FMxfeMvT~>SQvGegVbnA_mfV+P*9^+v=!mYWW%{*lRhQGc- zBOE(#c7%U@10%fiO4n3`XHQ1>@0J9>hlpsKg;`yrx|6sVJ#Ld-!CQDc`% zLHq3R`x_!M-T$rLdnG?-C3Eje-*55fTid;qzrJohxJYF&7GJDIS{VCkWC^vB)bJ;_ zH&F9)rCi(Kt6Q~=l4-piBZ_hmN^oibv{HrA{{4kQ%nk~;_Uz;O74J6d(mEDIUCuqE zvFv^Bu)1o&gLlojNzbZKhCJ_zOdavf%vd0psElT;SJl%q*4HM1GqKiF8AWyp}XcF#cH#`6;t?meM<^ub+f{z);J-rSm;jY{*c zg&8Ks*GPEJJD{F^w<`Lhjb6%@ethD`ty>ZX2AzV$O|dQ|r#F~i8VX6JX(@-rhGZ<| zy{%P!;~BkZ@cJ4tRc*VBxc8!dxn2Z~IRxwF%{_k9qO9FDMVDmrDcicvo-{kw8?>9; zQW#j~crHg~+n;Qyh1a`BKG5+B={)YsgwB`RH&=DoRa>4uHr6Pq6FnIB#qjcwnq$_B zhLMLT7n>m$@Pty(W^r+wI8VTZ#|$o-C%H)6Gn;vMKB3%ZE5SuXk$=L=jsH{HcC#-xk&J+ocZns71G@vRQl zZQatCUp3-me44I4&aqa$)8!KRp05%ipF(|QjZXgwPXG52BS}N`{ot1w`gVg6(+neK zq~^h-zBBedSA9!G5vd{ngjhX*A}$i2jX1`Dh=Vn@|1IKMxUse+^b0A%k)Jxf5kG>E&QH$^)9=(@Wb^_u-1;;cMj=vai> znCJTgue0I7KB$2Q#^WDcGxk9qX(-|?i0BVRfwozQo|(odqD#dO5n<->+&mpo9_E3$ z6PYKs+35E*+fVl$GF@EibFnGthGr2u;roAd!5)4YwIvkv%$P77(wv Date: Tue, 21 Jul 2015 13:20:35 -0700 Subject: [PATCH 02/53] Add xmpp dpd sig and fix a few parsing problems for connections that do not upgrade to TLS. --- scripts/base/protocols/xmpp/__load__.bro | 2 ++ scripts/base/protocols/xmpp/dpd.sig | 5 +++++ src/analyzer/protocol/xmpp/XMPP.cc | 1 - src/analyzer/protocol/xmpp/xmpp-analyzer.pac | 7 ++++--- src/analyzer/protocol/xmpp/xmpp-protocol.pac | 3 ++- .../ssl.log | 10 ++++++++++ .../ssl.log | 10 ++++++++++ .../Traces/tls/xmpp-dialback-starttls.pcap | Bin 0 -> 14673 bytes .../scripts/base/protocols/xmpp/client-dpd.test | 8 ++++++++ .../protocols/xmpp/server-dialback-dpd.test | 8 ++++++++ 10 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 scripts/base/protocols/xmpp/dpd.sig create mode 100644 testing/btest/Baseline/scripts.base.protocols.xmpp.client-dpd/ssl.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.xmpp.server-dialback-dpd/ssl.log create mode 100644 testing/btest/Traces/tls/xmpp-dialback-starttls.pcap create mode 100644 testing/btest/scripts/base/protocols/xmpp/client-dpd.test create mode 100644 testing/btest/scripts/base/protocols/xmpp/server-dialback-dpd.test diff --git a/scripts/base/protocols/xmpp/__load__.bro b/scripts/base/protocols/xmpp/__load__.bro index a10fe855df..0f41578f8a 100644 --- a/scripts/base/protocols/xmpp/__load__.bro +++ b/scripts/base/protocols/xmpp/__load__.bro @@ -1 +1,3 @@ @load ./main + +@load-sigs ./dpd.sig diff --git a/scripts/base/protocols/xmpp/dpd.sig b/scripts/base/protocols/xmpp/dpd.sig new file mode 100644 index 0000000000..50ae57a669 --- /dev/null +++ b/scripts/base/protocols/xmpp/dpd.sig @@ -0,0 +1,5 @@ +signature dpd_xmpp { + ip-proto == tcp + payload /^(<\?xml[^?>]*\?>)?[\n\r ]*]*xmlns='jabber:/ + enable "xmpp" +} diff --git a/src/analyzer/protocol/xmpp/XMPP.cc b/src/analyzer/protocol/xmpp/XMPP.cc index c84c372c4d..ee2667a276 100644 --- a/src/analyzer/protocol/xmpp/XMPP.cc +++ b/src/analyzer/protocol/xmpp/XMPP.cc @@ -61,7 +61,6 @@ void XMPP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) } catch ( const binpac::Exception& e ) { - printf("BinPAC Exception: %s\n", e.c_msg()); ProtocolViolation(fmt("Binpac exception: %s", e.c_msg())); } } diff --git a/src/analyzer/protocol/xmpp/xmpp-analyzer.pac b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac index a4417e1601..90b51ec183 100644 --- a/src/analyzer/protocol/xmpp/xmpp-analyzer.pac +++ b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac @@ -16,7 +16,8 @@ refine connection XMPP_Conn += { // Yup, looks like xmpp... bro_analyzer()->ProtocolConfirmation(); - if ( token == "success" || token == "message" ) + if ( token == "success" || token == "message" || token == "db:result" + || token == "db:verify" || token == "presence" ) // Handshake has passed the phase where we should see StartTLS. Simply skip from hereon... bro_analyzer()->SetSkip(true); @@ -24,9 +25,9 @@ refine connection XMPP_Conn += { client_starttls = true; if ( !is_orig && token == "proceed" && client_starttls ) - { bro_analyzer()->StartTLS(); - } + else if ( !is_orig && token == "proceed" ) + reporter->Weird(bro_analyzer()->Conn(), "XMPP: proceed without starttls"); //printf("Processed: %d %s %s \n", is_orig, c_str(name), c_str(rest)); diff --git a/src/analyzer/protocol/xmpp/xmpp-protocol.pac b/src/analyzer/protocol/xmpp/xmpp-protocol.pac index e05268fe32..9b21679c30 100644 --- a/src/analyzer/protocol/xmpp/xmpp-protocol.pac +++ b/src/analyzer/protocol/xmpp/xmpp-protocol.pac @@ -3,6 +3,7 @@ type XML_END = RE/>/; type XML_NAME = RE/\/?[?:[:alnum:]]+/; type XML_REST = RE/[^<>]*/; type SPACING = RE/[ \r\n]*/; +type CONTENT = RE/[^<>]*/; type XMPP_PDU(is_orig: bool) = XMPP_TOKEN(is_orig)[] &until($input.length() == 0); @@ -12,6 +13,6 @@ type XMPP_TOKEN(is_orig: bool) = record { name: XML_NAME; rest: XML_REST; : XML_END; - : SPACING; + tagcontent: CONTENT; }; diff --git a/testing/btest/Baseline/scripts.base.protocols.xmpp.client-dpd/ssl.log b/testing/btest/Baseline/scripts.base.protocols.xmpp.client-dpd/ssl.log new file mode 100644 index 0000000000..0ce11b2e6f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.xmpp.client-dpd/ssl.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-07-21-20-08-11 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1437091702.232293 CXWv6p3arKYeMETxOg 198.128.203.95 56048 146.255.57.229 5222 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T F5Nz2G1vSZQ0QXM2s8,FUw8omi2keRxShDUa (empty) CN=jabber.ccc.de,O=Chaos Computer Club e.V.,L=Hamburg,ST=Hamburg,C=DE emailAddress=support@cacert.org,CN=CA Cert Signing Authority,OU=http://www.cacert.org,O=Root CA - - +#close 2015-07-21-20-08-11 diff --git a/testing/btest/Baseline/scripts.base.protocols.xmpp.server-dialback-dpd/ssl.log b/testing/btest/Baseline/scripts.base.protocols.xmpp.server-dialback-dpd/ssl.log new file mode 100644 index 0000000000..15641ba5b0 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.xmpp.server-dialback-dpd/ssl.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-07-21-20-18-36 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1437506779.381295 CXWv6p3arKYeMETxOg 184.73.173.246 1193 104.236.167.107 5269 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp384r1 - F - - T FLFr7Z1TXmFDv9FwC2,FydVem3ToAkEIAHD29,FK07OA1VxtQi69Irde F3D2e62Vxl7iTnwbA4,FUCD5w4ABMG5N0YvSi,FxWUEd3mgvThYO2uod,FGOrVE2laVCPsCLMF6 CN=www.0xxon.net,OU=Free SSL,OU=Domain Control Validated CN=COMODO RSA Domain Validation Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB CN=*.hosted.im,OU=Domain Control Validated CN=Go Daddy Secure Certificate Authority - G2,OU=http://certs.godaddy.com/repository/,O=GoDaddy.com\\, Inc.,L=Scottsdale,ST=Arizona,C=US +#close 2015-07-21-20-18-36 diff --git a/testing/btest/Traces/tls/xmpp-dialback-starttls.pcap b/testing/btest/Traces/tls/xmpp-dialback-starttls.pcap new file mode 100644 index 0000000000000000000000000000000000000000..ad55c6eceba70f34ebabaf3f8aa8f25f83fa0fe8 GIT binary patch literal 14673 zcmdUW2|QG7`~Mj;_I;Oi%9{1qvWx7Ih*C<~8QW-#wX%gulAWmRdv;0LrtoOB6d{yK zi#;iX@;hhHgQvIWUH#}5PZWR-0DyL%H2^U(Z}nP(k~!yzd{u zCkc*W*c|{!P^8re1d0TfR-FB(GypgbAe>iau1W&{4M2me1%xHR5?K;ag+?PN4oISi zClijr=gwwnV}l4l73v8te;J#I-7IlzI$Opj0!iFv!hmC_7l20zf8dcF8A1TBpli7N zMOK&H7g!cP?U%ARa11lt&?t8Mxx1l#9B^J3tcRMgv?M6JTV2%)k8`kbS6=->Lp40S z)^5~9y00z2-%U^uvsNEb<7X2?}LFyNRTKyZ-f;+BiLjKNr(}~u5WwYIpMx4qK zLT$8O>tqz*3=IM>30O`v{#%>aN6i9m2600cV{d9d{Az_knUA^T=^Xs@w<`S1em zoYs%5&R=1c)b5wm2r(kO5MGD^Lv3CP)(` z2s;Ua1Udo=xC+^p6p#dH5*P{O1R4SfKnlf`6%uHst^hTFLIMa9fC`XJ4BN(x-3 z28d;=jZnvgWWiDtB>$0=P)=A^NI;zg0F;r)7sgTDvS}1`>5c4;BlZqU`ILfuD;7PX`jg&`%98^Z*P`0a6_>lvEQ8MeWQWPMQ%&7EI(A zt4tDht+Y#-FE4u!(vne%M(9V((;^VmqyQKaBTWrbk|7VG$e2m=_DFMq+u$1tW+pu- zHsKv`=zTUGcFqo7(4{m3NC*E&&P;A*I5s4aqFA_S*q7O|&-F-3H@&N>ya zs2|KjBWr1|Ip(gg-SDx%k?*s91ssX2L)B4xwZj8DQhz9rotfzDix{Kz=b@jdm`ME^ zBfH=199Mkr%&URhIoJJIOW)MRG+FF?f(yOuvsY?&lda)#FL%gu07P~uq6b=$F5=4b z@S`|DwqTZ+G8g3pOX|LuF{VRaue7R_MP7UWd7wq+p+m8uSY%%-u$NyUw2c3tJaEYD zRx_ETNEgw9c<2#`6%-0dLYe{6z}1=IdlC>>Z7hg_RYf62h9D&+MaOZkG5(FC$%$~d;`s774E=fDAJJucxe@SVXn1K{zWKJP8 z2oeNn4h!T%?E?)#gABb4od~Tp-ere#TWe)!v)Ybm-;fEfn%tiTylTIz=BuSC2{i;m z!q>+Faa(`T7aI54Y(SC?J>dV{9`K;RPl1RCjT`CZ2}uOn}*_}hh6Pe(MxWkGZPu!%=DBiXF$GLu*&I@uSo~7 z$$C_$!!PbgzEl-9oMh4I$zh#zc}jvj{#xdphh0xjjkbuUifoVFhmG4WnEN?Een@Cg zH@EY~5&5VU<5(?|-ACei-AuPn9cC&1;ceolZdA@4!My02Qa`Mcz#D#G5E(DP;qge= zROh|tOF^w1+Shw?Cu;)lKXIRu(s$vZx$E%j z0Ku=#Citx@3nWon%$rdSi!PZ`OquAHk?Gz5zrRFhIDr3zUql4R1m6?kmjWb%P#9uk ze+R-`a1%6&1!Mxr$WYKFumMnw;2-%=I-nM)nXx-VEkfmQ0dvZ%Yj*go-Y@uiQ(>Ax=MT63;p2%yRpTYux2J+=QXS>JWUP(f4Mp0BdmLc&bxwN5;cdza zaN*Any>3%PX^D)oeeWmjw$c-}e!irr;)csgSlGk)Y7$EprE`xAC>r@V*nji2m*2!3 z|6tdUTQuk8+%fahNw=DR9E`qXaD3Umu0y>cX&m#UX;^>8Hk9EVjjqLK*%zJn?I!== zOCU|!%Ng4>Ki+ZqKq)Onzj{H2>FC{a_?`#z_+m-3rMiHn%$v*aGOHJjp4yP$oB~a*Gj+B3z#Of$5C`Gn|wlQoH~DGl5Xh6Q{Paz zL9#@WzL;+kV+uW-)}2o1-o8E5I%T2GFNYij6kyIz`ivB4M*+mLm93))AvrKxS6kl9 z))~=6&d=fRd1@~J#OzGirn^nBHn{gucZ7(GwxIac?JVefqtpycf$Eic(2D8z{y`EXiVJ|na zP&lYyH^dY(9cjqnAm@U(Hubt;qc!(P3qf3f$c1@+KGpGq$hwD(8`|951a0hq_r>B| zrFVd6xE_*u+h6NJr2POEi-)5J6gT8$L1|D^=7YH7DbsZ8wN2PYDo9Q-LeUK zU`WrNulBYaWiz$HnB4XyLY-<+{n)qo_i_RKEUq7}R)uGT2KbbnjOm^DPP=&3ZShrk z=ogm2Z;dT^Qu~9h1l?SA(!w>+0M9RsyNW-Xyqv<#QXKg1t;cd-!AP9I3oTdM+JoQ6 z?$*22T%4u5lsb~;hPkrfrw{7%l)R=@)v;cdNir3#cWQnpS>rmU+t!B8sNi#WsZVhy z$GGtGSzU#&@sE!~^tGIfaiVF*8jQF^B9HAjRb8CYKQ(N6wwY4b*j@d~Vj}yC)3O$t zs+UYBscVjviQ93$vbevye$=WZtxV_2n`}#g>N|yp)H<+5D+Y%fKNYNesVx~*Sh`dD zgw8zpk+1`E$sst+f}JLrbg3X?MR8&v1ZZqn{BA~1$b z98#Hkmi6Z7txD961Bt=$n2*s$4l}Vx2~Rx9*hMGSotOh8b-k}lTn_3sWn_!k1bKEg zXBCNWFVbrlxk{$v(-g`fT-{Sz{mA6K>ldSlli+CHE4GN?z7>t3rOrL)gR*6nC6DoH zGKSc-bDVS)o5+j2C4EfUMQ^y-|HyMRXW3_$lpl#Gvo^xGP5aQmNcXU$W;r?FSZDoPa_+K1s<7mZU~sajHg=zCcLd& z4Va`Llu}p$8Q}+L6cq>s(6w-=4L{;Tsevk>Qiei?e1z=JSmh?^4P|Qaa9AR4E7qNH z-DkBlVNWCmKl$}w2L->KpfD)7eh(rdwr%A6FuI+gCjVXPZh@F?)7YFh5*9ETVXZY` zNYIZkC8oKcKd`JiNY*lxm4M4uq$HUN$%T<}_mj;?$-0^yfsoSQ4*)jJaSK-_?lA0| z&d!`j4exwXo4M11_M_!|*QokuAC$+;0na*K)s=m;c$dc^p@PqvGeoV7zTUZ*>E2>a z>D~5Ln9eH9UqOS}_iObH?oO3rcBbdkavXtTqyEo|Ep%wgL>4;--kBtxuX`14fjJY` zci^E{NE7$$`&ti2M$#(Y1c{*7#JD3fM(x?LCyh?)(Wp>wo6D?li%>sS)FK)B=o@*` z9sPYVpA+sEdS#81o#MatGHZ6tX%@EZW zAAH1eSW|$f@-nOSVZDKA58dyqU;9s_Rt@ykj%ZXpNmeQiuj7fXA3T@Vl{Y=3f@M&| zLRlMxI}L!qAOQ{=)Z>T#=YvtQyu0JLHRQcH2a5EpRa8+ahm#)H@;*(uA#SSkpt{hL zevmxD%QRLkU%gU1N%85sCjCiYwkh6f(a+8U^A{#{8QR_OEERx0_w{5g{{BbfyK1*t zJK~lK>Ea+_2SkV$@40Am?{w-5ZnuNwOQ8&M5^oQgu_bu;w=bC9p-fI{I_OTubfUbb z#v$MVO*N0d0vF|Nwd^xx%^X4(x`XZ?w4kazSt`@;MYZ)PmlfU17K`BowjE4k?Sp4~ zH5s{v>fW`!9qY}hy;Tw}AX$v*calBg-JtI}ap#^@Gb!N1K(4((mWr=cf^xzU;fpFkFEB z(K1-=}qOKI~km??ZesR@35qb0aS1d$}^i?YxjVc;2*Er*(`bUo%Z1 zx(XZ1TVAp*U<@4LRGTC%00ZDVim52K7Y+Q7q3 zQd$BOgDC(tv%oqWK}h5!>4ddkuPB9c@Wgsy@K~I`lr%&G;OasA{XbH5Q%v$nJPDWM+l;0&M4JYrQJfO@jP)SZ+R^Ad+`e%}NYfJ}8LoF)ljD=E< z_L3O)zs=JXip0A{#V+~}r^GTn&+G#jFxxA)QPAH|3-V+<889w-!$pc!p)=X0;GTAl zU={8Wx8m2diE9xIf@LH$lM`p69r%t(U4MMo)qVSN_l0YJOqhK?9^Wiw)QYB}xz5h; z+G&ieb<}R#kw>@BD0r|uk0=PXN{;g?Q^=v_p4o|EcNsUODxyAXIr_LUZTrKPx3aBd zdGicXCznO*Z?b=|e;`;R^QviZu8q#d-|@x$gBKSL(sv=AJb$Kqig9e>54Xa~#{A={ zhFA3^MfrPr(&p=spS+)k-CfueQS zU-fJ39=SA=xw(m&*1ybZfxUbcv{5_{N?>%zQJ!4LO zro{1kaAMa}$8$4z9Y2->;vQw9-{kI_iZe5eJ*cW~SJx3WsGv4`T~40eS~@@1tcPKy z8+0n&<9?l5Fk2>k{}sa>uc;c%Yy>~iJzD6~W4d+RkQOv7`d0af(a_w$s_>P?^k`v4 zb~W3lIh?+roHFHm02(<96jfC(A{n$2ziO=LfH6mx=udwz=kS z(b6ILWjCm5rMH~QR^P0Uq-h!r+3SKxV`o(3Cs2`hwp?hzQP0l(tMM^&5l6r1oELAGI!@=G9s zPs7&FWRz3G$r+qPYNqEJNHsmscN1RFzl09r*3KUw2o(Y0l)?IW#FpcOO%N&%i{(Q} zg5scPhDe4`#82MvmneXPHZ2lPGQ=RD3pYcfc7xFVXKh0SQ;D@6A@nB(6Zmw-?sxp- zSKvqy=oZg#U$Hi+Fvf5t^TdACw43_X^JEW9F6Vj8TS#*?){jUHEZli^bh!G%dw*Lx znyg+IJ1^;*UkmheR462}tYUhrPYdp6JOsAXXkR-ee(l}y_tP{sMNK23TvK7!YtNp! ztJSWZ&+JJX%lG9zbIDoj{ytoRmjLh08$0M8vR|-0tj$$?K@d7fe$v8aJ78MWnZda zs7HYa02Km(;~5AX9ag~+$@PJC%R+&6o6&LYu#}3K!`#^c{gcY+;jrGGYs7F3{ivB& z(GSKgEjJr4FSN`C=z`B`f0Dhmg9d3RQ?wDcWT9hPB8-6{|8l|K$igl_L{n!VsOX{&oP{Aqs$s5+IhX zI|qtoyCpb&xM4QO51A-p{NPA04~hVQ9cP*ITG3hvm9Uxj;Rfl#V{>6##u?QEo+)iI zhze45a^tBli~FUHfIB`EZ#(4rCYXGAmd!g@$@J8_Z}K&GU3T!2bY(LF6gI}14OxPJ-bzAK86CXDfH%s19R|`X>5Ixn-h7JZazj$YZ}AM&DrJS!FrOml0uBH+^~qLp`kD zd@G*%K9zaP4J#HW1-FN}QqmL(MIP9w`m{F|Rstbq$;ns4O5AQJ7&*?`#4@5s8o~5{ zoSl^AujNkI`*LWQzKR0#$^aA!!b3Rm`*Dpd0vpkg6iTB1Z=oS52kw$ll2(BJ5?S2F z+y9+Ggd)%zC%^h03oZN}3kC5#mY3Ih$1FGuGLMe;p3sjNYaCh`nRep-;^16maWdn$ zZ{1{?!Ki1j+93`*3hax|k2`k8#uh0YoeYZnO+<>TuRo0L5!_+T>-d$pSJ^o0! z=M7U9vqWEX*$F1K3<^?;nq#>RiYT2cyxG>)_*TtaoGq7aD zlA9gwU<_2mw=S&Q#Kc?m+1XfVsK>SQ|Q zKFlZgDY~eb~weU$J~f)o&=E`>A8Rw^WM34(ua{*kK1hr5_G(7Ht9SN zxY-j*Qo`?#xl~)Tuf4a1AeLUddhPH7xHc6RdYlVlHy`f+2S8ZWVt zXLCkBgigM(VZ`rUvEV#M@2xM{VTxhTAYrme?CH{|WH6d)xxt{7mwTP6C($;Qw&Q$g zs|?MESh~)8+H{et6}Lq#ElL#V%XoqY8q(q1o+ASQ*xI0GBA+Tp@5w%7u}7-zfVzz; zxuK7^xmGdNn<4i78lTAS-bXugeNScaabl>umGJr#-H6-QV{?r-V=p}Srhaqd^2@nd z@*muRIXg>_sD++es-|_OON}h7IVdAkbRe1)lwRW4E&i2Gd;WPus2K~i?!-3`0g+Qf z_fB(vs*PyU!Xm?OwnY=Gx}X|Hg^5_6hI!8hwfcFTN@4((}}UkO9Z=r{Z-pUAIb z|H`jatl^yMI=@=PQ2czsckB^w{;^%K7W%Q7l?(2uP8+IyQ7xx#G@9K&BRW{8X|TT@bK-b zq0BPz1IxOKr2>|rJGg8miNKWI4_Fo+|4UiWBwQT<0D8?AxVyIc+P-z4W(y{l5+3Zw zN2f4}yi*u=VzJ4gd;28tAop$-HirDuN8cBtnyQpK+Z1}Uq<(yV=1V52w%3_cY=7^l z!o_w&J5|2Ul6^K#VlJwuBTGmpEfoXnFYRJF%A+$Hc-xaHC_^dJP9eSS#kpjZ2d6Wd z@BVSFDIr&Dj?+<@qUP$R-`Q#~?4$%&=ACURCe;@Z28@w(3Z~wQhda$t9Y^|>q86>5 zW;P?Or)z2sPU;0Ff2Wk~qwyj5^uTNJ9!~{KK`)=dWvfCF!y$#RLN$7~D74~Kj;fFe zM=^(=3vn$FRcoqz%ecdGW&Fi~s+L5J?R^behw67nT;N z@lOfzs0OZZ9xMIEyIuip@!u^I>gFC^5{M3J_6X}xV9m`IiQ?TpMnl6Tn2IyI==9mZ zm2Ys@R9Pu#z}YO48$LQykx|YVc75dgQPI4Z3e5ZR{tL%FQ1RFD1rHmd=<$5UJ^dCZVeOCIp#Gg#tq{FRX|pG&3K@-XVoVr{ zIS#hNHUocPS-PdvKoN8em%n(`ReONwRV}}i1zD0u=c-p>*Stz+-!Q0DP!e;ZBJZtN zvPX7uR%@`*Ig>8BJg(>CsOm#Py7^LzOU@U+=Jo+U@;t_sPV)FM->Pbq@QGI#f1}+t zgEvi4`s6dqTpxX@dHT5i5eMfl?8lZT_1;WG*f3Vpeac71+Rffvfk!S^K@DU=qlU{? z9{}P*iin`bzu5;kF^~_W!am^3vpRCeH6Iv>BL7ih%5$ti)n!PUCO{tl?y<2eha@#+ z>t5&b;n5zPnGRw9vz#fM(yM^p^nqnnf^+Z#X->zK2qzv3Q}c<8MO)IO9YfLDRpI%l zEDz(im>b-8`I$8G2D5_x=x~7t95Y7q8@1UZh-y>Vvec=N+PX~`i#g1vi1UR0z_O$# zXn=#zHC+BOdA*Qg;^aNHOuieMd>TA?pDqAkg}{(=ZSwC-3d6GX#B5Iy%5!8x_EPYW zeSkahhdTL9@VP Date: Mon, 4 Jan 2016 00:55:52 -0500 Subject: [PATCH 03/53] Cleaned up stats collection. - Removed the gap_report event. It wasn't used anymore and functionally no more capable that scheduling events and using the get_gap_summary bif. - Added functionality to Dictionaries to count cumulative numbers of inserts performed. This is further used to measure the total number of connections of various types. Previously only the number of active connections was available. - The Reassembler base class now tracks active reassembly size for all subclasses (File/TCP/Frag & unknown). - Improvements to the stats.log. Mostly, more information. --- scripts/base/init-bare.bro | 81 +++++++++++--------- scripts/policy/misc/stats.bro | 37 +++++++-- src/Dict.cc | 2 + src/Dict.h | 7 ++ src/Event.cc | 4 +- src/Event.h | 4 +- src/Frag.cc | 2 +- src/Reassem.cc | 34 ++++++-- src/Reassem.h | 26 ++++++- src/Sessions.cc | 3 + src/Sessions.h | 23 +++--- src/analyzer/protocol/tcp/TCP_Reassembler.cc | 43 +---------- src/bro.bif | 38 ++++++--- src/file_analysis/FileReassembler.cc | 2 +- 14 files changed, 189 insertions(+), 117 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 94b6ed33e5..337052178d 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -462,34 +462,51 @@ type NetStats: record { ## .. note:: All process-level values refer to Bro's main process only, not to ## the child process it spawns for doing communication. type bro_resources: record { - version: string; ##< Bro version string. - debug: bool; ##< True if compiled with --enable-debug. - start_time: time; ##< Start time of process. - real_time: interval; ##< Elapsed real time since Bro started running. - user_time: interval; ##< User CPU seconds. - system_time: interval; ##< System CPU seconds. - mem: count; ##< Maximum memory consumed, in KB. - minor_faults: count; ##< Page faults not requiring actual I/O. - major_faults: count; ##< Page faults requiring actual I/O. - num_swap: count; ##< Times swapped out. - blocking_input: count; ##< Blocking input operations. - blocking_output: count; ##< Blocking output operations. - num_context: count; ##< Number of involuntary context switches. + version: string; ##< Bro version string. + debug: bool; ##< True if compiled with --enable-debug. + start_time: time; ##< Start time of process. + real_time: interval; ##< Elapsed real time since Bro started running. + user_time: interval; ##< User CPU seconds. + system_time: interval; ##< System CPU seconds. + mem: count; ##< Maximum memory consumed, in KB. + minor_faults: count; ##< Page faults not requiring actual I/O. + major_faults: count; ##< Page faults requiring actual I/O. + num_swap: count; ##< Times swapped out. + blocking_input: count; ##< Blocking input operations. + blocking_output: count; ##< Blocking output operations. + num_context: count; ##< Number of involuntary context switches. + + num_packets: count; ##< Total number of packets processed to date. + num_fragments: count; ##< Current number of fragments pending reassembly. + max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. + + num_tcp_conns: count; ##< Current number of TCP connections in memory. + max_tcp_conns: count; ##< Maximum number of concurrent TCP connections so far. + cumulative_tcp_conns: count; ##< - num_TCP_conns: count; ##< Current number of TCP connections in memory. - num_UDP_conns: count; ##< Current number of UDP flows in memory. - num_ICMP_conns: count; ##< Current number of ICMP flows in memory. - num_fragments: count; ##< Current number of fragments pending reassembly. - num_packets: count; ##< Total number of packets processed to date. - num_timers: count; ##< Current number of pending timers. - num_events_queued: count; ##< Total number of events queued so far. - num_events_dispatched: count; ##< Total number of events dispatched so far. + num_udp_conns: count; ##< Current number of UDP flows in memory. + max_udp_conns: count; ##< Maximum number of concurrent UDP connections so far. + cumulative_udp_conns: count; ##< - max_TCP_conns: count; ##< Maximum number of concurrent TCP connections so far. - max_UDP_conns: count; ##< Maximum number of concurrent UDP connections so far. - max_ICMP_conns: count; ##< Maximum number of concurrent ICMP connections so far. - max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. - max_timers: count; ##< Maximum number of concurrent timers pending so far. + num_icmp_conns: count; ##< Current number of ICMP flows in memory. + max_icmp_conns: count; ##< Maximum number of concurrent ICMP connections so far. + cumulative_icmp_conns: count; ##< + + num_timers: count; ##< Current number of pending timers. + max_timers: count; ##< Maximum number of concurrent timers pending so far. + + num_events_queued: count; ##< Total number of events queued so far. + num_events_dispatched: count; ##< Total number of events dispatched so far. + + total_conns: count; ##< + current_conns: count; ##< + current_conns_extern: count; ##< + sess_current_conns: count; ##< + + reassem_file_size: count; ##< Size of File reassembly tracking. + reassem_frag_size: count; ##< Size of Fragment reassembly tracking. + reassem_tcp_size: count; ##< Size of TCP reassembly tracking. + reassem_unknown_size: count; ##< Size of reassembly tracking for unknown purposes. }; ## Summary statistics of all regular expression matchers. @@ -507,7 +524,7 @@ type matcher_stats: record { ## Statistics about number of gaps in TCP connections. ## -## .. bro:see:: gap_report get_gap_summary +## .. bro:see:: get_gap_summary type gap_info: record { ack_events: count; ##< How many ack events *could* have had gaps. ack_bytes: count; ##< How many bytes those covered. @@ -3416,23 +3433,17 @@ global pkt_profile_file: file &redef; ## .. bro:see:: load_sample global load_sample_freq = 20 &redef; -## Rate at which to generate :bro:see:`gap_report` events assessing to what -## degree the measurement process appears to exhibit loss. -## -## .. bro:see:: gap_report -const gap_report_freq = 1.0 sec &redef; - ## Whether to attempt to automatically detect SYN/FIN/RST-filtered trace ## and not report missing segments for such connections. ## If this is enabled, then missing data at the end of connections may not ## be reported via :bro:see:`content_gap`. const detect_filtered_trace = F &redef; -## Whether we want :bro:see:`content_gap` and :bro:see:`gap_report` for partial +## Whether we want :bro:see:`content_gap` and :bro:see:`get_gap_summary` for partial ## connections. A connection is partial if it is missing a full handshake. Note ## that gap reports for partial connections might not be reliable. ## -## .. bro:see:: content_gap gap_report partial_connection +## .. bro:see:: content_gap get_gap_summary partial_connection const report_gaps_for_partial = F &redef; ## Flag to prevent Bro from exiting automatically when input is exhausted. diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 215a3bb9de..484267898c 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -10,7 +10,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 1min &redef; + const stats_report_interval = 5min &redef; type Info: record { ## Timestamp for the measurement. @@ -27,6 +27,22 @@ export { ## interval. events_queued: count &log; + ## TCP connections seen since last stats interval. + tcp_conns: count &log; + ## UDP connections seen since last stats interval. + udp_conns: count &log; + ## ICMP connections seen since last stats interval. + icmp_conns: count &log; + + ## Current size of TCP data in reassembly. + reassem_tcp_size: count &log; + ## Current size of File data in reassembly. + reassem_file_size: count &log; + ## Current size of packet fragment data in reassembly. + reassem_frag_size: count &log; + ## Current size of unkown data in reassembly (this is only PIA buffer right now). + reassem_unknown_size: count &log; + ## Lag between the wall clock and packet timestamps if reading ## live traffic. lag: interval &log &optional; @@ -64,16 +80,27 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) # shutting down. return; - local info: Info = [$ts=now, $peer=peer_description, $mem=res$mem/1000000, + local info: Info = [$ts=now, + $peer=peer_description, + $mem=res$mem/1000000, $pkts_proc=res$num_packets - last_res$num_packets, $events_proc=res$num_events_dispatched - last_res$num_events_dispatched, - $events_queued=res$num_events_queued - last_res$num_events_queued]; + $events_queued=res$num_events_queued - last_res$num_events_queued, + $tcp_conns=res$cumulative_tcp_conns - last_res$cumulative_tcp_conns, + $udp_conns=res$cumulative_udp_conns - last_res$cumulative_udp_conns, + $icmp_conns=res$cumulative_icmp_conns - last_res$cumulative_icmp_conns, + $reassem_tcp_size=res$reassem_tcp_size, + $reassem_file_size=res$reassem_file_size, + $reassem_frag_size=res$reassem_frag_size, + $reassem_unknown_size=res$reassem_unknown_size + ]; + + # Someone's going to have to explain what this is and add a field to the Info record. + # info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts); if ( reading_live_traffic() ) { info$lag = now - network_time(); - # Someone's going to have to explain what this is and add a field to the Info record. - # info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts); info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd; info$pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; info$pkts_link = ns$pkts_link - last_ns$pkts_link; diff --git a/src/Dict.cc b/src/Dict.cc index 1d32eccde3..9e68d64089 100644 --- a/src/Dict.cc +++ b/src/Dict.cc @@ -66,6 +66,7 @@ Dictionary::Dictionary(dict_order ordering, int initial_size) delete_func = 0; tbl_next_ind = 0; + cumulative_entries = 0; num_buckets2 = num_entries2 = max_num_entries2 = thresh_entries2 = 0; den_thresh2 = 0; } @@ -444,6 +445,7 @@ void* Dictionary::Insert(DictEntry* new_entry, int copy_key) // on lists than prepending. chain->append(new_entry); + ++cumulative_entries; if ( *max_num_entries_ptr < ++*num_entries_ptr ) *max_num_entries_ptr = *num_entries_ptr; diff --git a/src/Dict.h b/src/Dict.h index 3a2239ef54..2def5ea28f 100644 --- a/src/Dict.h +++ b/src/Dict.h @@ -71,6 +71,12 @@ public: max_num_entries + max_num_entries2 : max_num_entries; } + // Total number of entries ever. + uint64 NumCumulativeInserts() const + { + return cumulative_entries; + } + // True if the dictionary is ordered, false otherwise. int IsOrdered() const { return order != 0; } @@ -166,6 +172,7 @@ private: int num_buckets; int num_entries; int max_num_entries; + uint64 cumulative_entries; double den_thresh; int thresh_entries; diff --git a/src/Event.cc b/src/Event.cc index 89e745361f..5d54752a5a 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -10,8 +10,8 @@ EventMgr mgr; -int num_events_queued = 0; -int num_events_dispatched = 0; +uint64 num_events_queued = 0; +uint64 num_events_dispatched = 0; Event::Event(EventHandlerPtr arg_handler, val_list* arg_args, SourceID arg_src, analyzer::ID arg_aid, TimerMgr* arg_mgr, diff --git a/src/Event.h b/src/Event.h index 6f9c9d10c3..0d004d526c 100644 --- a/src/Event.h +++ b/src/Event.h @@ -72,8 +72,8 @@ protected: Event* next_event; }; -extern int num_events_queued; -extern int num_events_dispatched; +extern uint64 num_events_queued; +extern uint64 num_events_dispatched; class EventMgr : public BroObj { public: diff --git a/src/Frag.cc b/src/Frag.cc index 6a8b901a73..842059e218 100644 --- a/src/Frag.cc +++ b/src/Frag.cc @@ -28,7 +28,7 @@ void FragTimer::Dispatch(double t, int /* is_expire */) FragReassembler::FragReassembler(NetSessions* arg_s, const IP_Hdr* ip, const u_char* pkt, HashKey* k, double t) - : Reassembler(0) + : Reassembler(0, REASSEM_FRAG) { s = arg_s; key = k; diff --git a/src/Reassem.cc b/src/Reassem.cc index 54f27bd895..35f491f8ed 100644 --- a/src/Reassem.cc +++ b/src/Reassem.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include +#include #include "bro-config.h" @@ -10,7 +11,8 @@ static const bool DEBUG_reassem = false; DataBlock::DataBlock(const u_char* data, uint64 size, uint64 arg_seq, - DataBlock* arg_prev, DataBlock* arg_next) + DataBlock* arg_prev, DataBlock* arg_next, + ReassemblerType reassem_type) { seq = arg_seq; upper = seq + size; @@ -26,17 +28,24 @@ DataBlock::DataBlock(const u_char* data, uint64 size, uint64 arg_seq, if ( next ) next->prev = this; + if ( Reassembler::sizes.size() == 0 ) + Reassembler::sizes.resize(REASSEM_TERM, 0); + + rtype = reassem_type; + Reassembler::sizes[rtype] += pad_size(size) + padded_sizeof(DataBlock); Reassembler::total_size += pad_size(size) + padded_sizeof(DataBlock); } uint64 Reassembler::total_size = 0; +std::vector Reassembler::sizes; -Reassembler::Reassembler(uint64 init_seq) +Reassembler::Reassembler(uint64 init_seq, ReassemblerType reassem_type) { blocks = last_block = 0; old_blocks = last_old_block = 0; total_old_blocks = max_old_blocks = 0; trim_seq = last_reassem_seq = init_seq; + rtype = reassem_type; } Reassembler::~Reassembler() @@ -110,7 +119,7 @@ void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data) if ( ! blocks ) blocks = last_block = start_block = - new DataBlock(data, len, seq, 0, 0); + new DataBlock(data, len, seq, 0, 0, rtype); else start_block = AddAndCheck(blocks, seq, upper_seq, data); @@ -275,7 +284,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, if ( last_block && seq == last_block->upper ) { last_block = new DataBlock(data, upper - seq, seq, - last_block, 0); + last_block, 0, rtype); return last_block; } @@ -288,7 +297,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, { // b is the last block, and it comes completely before // the new block. - last_block = new DataBlock(data, upper - seq, seq, b, 0); + last_block = new DataBlock(data, upper - seq, seq, b, 0, rtype); return last_block; } @@ -297,7 +306,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, if ( upper <= b->seq ) { // The new block comes completely before b. - new_b = new DataBlock(data, upper - seq, seq, b->prev, b); + new_b = new DataBlock(data, upper - seq, seq, b->prev, b, rtype); if ( b == blocks ) blocks = new_b; return new_b; @@ -308,7 +317,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, { // The new block has a prefix that comes before b. uint64 prefix_len = b->seq - seq; - new_b = new DataBlock(data, prefix_len, seq, b->prev, b); + new_b = new DataBlock(data, prefix_len, seq, b->prev, b, rtype); if ( b == blocks ) blocks = new_b; @@ -342,6 +351,17 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, return new_b; } +uint64 Reassembler::MemoryAllocation(ReassemblerType rtype) + { + if (Reassembler::sizes.size() == 0 ) + Reassembler::sizes.resize(REASSEM_TERM, 0); + + if ( rtype < REASSEM_TERM ) + return Reassembler::sizes[rtype]; + else + return 0; + } + bool Reassembler::Serialize(SerialInfo* info) const { return SerialObj::Serialize(info); diff --git a/src/Reassem.h b/src/Reassem.h index e55c809990..d371b998bd 100644 --- a/src/Reassem.h +++ b/src/Reassem.h @@ -6,10 +6,23 @@ #include "Obj.h" #include "IPAddr.h" +// Whenever subclassing the Reassembler class +// you should add to this for known subclasses. +enum ReassemblerType { + REASSEM_UNKNOWN, + REASSEM_TCP, + REASSEM_FRAG, + REASSEM_FILE, + + // Terminal value. Add new above. + REASSEM_TERM, +}; + class DataBlock { public: DataBlock(const u_char* data, uint64 size, uint64 seq, - DataBlock* prev, DataBlock* next); + DataBlock* prev, DataBlock* next, + ReassemblerType reassem_type = REASSEM_UNKNOWN); ~DataBlock(); @@ -19,13 +32,12 @@ public: DataBlock* prev; // previous block with lower seq # uint64 seq, upper; u_char* block; + ReassemblerType rtype; }; - - class Reassembler : public BroObj { public: - Reassembler(uint64 init_seq); + Reassembler(uint64 init_seq, ReassemblerType reassem_type = REASSEM_UNKNOWN); virtual ~Reassembler(); void NewBlock(double t, uint64 seq, uint64 len, const u_char* data); @@ -51,6 +63,9 @@ public: // Sum over all data buffered in some reassembler. static uint64 TotalMemoryAllocation() { return total_size; } + // Data buffered by type of reassembler. + static uint64 MemoryAllocation(ReassemblerType rtype); + void SetMaxOldBlocks(uint32 count) { max_old_blocks = count; } protected: @@ -82,12 +97,15 @@ protected: uint32 max_old_blocks; uint32 total_old_blocks; + ReassemblerType rtype; static uint64 total_size; + static std::vector sizes; }; inline DataBlock::~DataBlock() { Reassembler::total_size -= pad_size(upper - seq) + padded_sizeof(DataBlock); + Reassembler::sizes[rtype] -= pad_size(upper - seq) + padded_sizeof(DataBlock); delete [] block; } diff --git a/src/Sessions.cc b/src/Sessions.cc index b8bfe82b34..3194985515 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1156,8 +1156,11 @@ void NetSessions::Drain() void NetSessions::GetStats(SessionStats& s) const { s.num_TCP_conns = tcp_conns.Length(); + s.cumulative_TCP_conns = tcp_conns.NumCumulativeInserts(); s.num_UDP_conns = udp_conns.Length(); + s.cumulative_UDP_conns = udp_conns.NumCumulativeInserts(); s.num_ICMP_conns = icmp_conns.Length(); + s.cumulative_ICMP_conns = icmp_conns.NumCumulativeInserts(); s.num_fragments = fragments.Length(); s.num_packets = num_packets_processed; s.num_timers = timer_mgr->Size(); diff --git a/src/Sessions.h b/src/Sessions.h index 2aca292789..e8c53256ff 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -32,19 +32,24 @@ namespace analyzer { namespace arp { class ARP_Analyzer; } } struct SessionStats { int num_TCP_conns; - int num_UDP_conns; - int num_ICMP_conns; - int num_fragments; - int num_packets; - int num_timers; - int num_events_queued; - int num_events_dispatched; - int max_TCP_conns; + uint64 cumulative_TCP_conns; + + int num_UDP_conns; int max_UDP_conns; + uint64 cumulative_UDP_conns; + + int num_ICMP_conns; int max_ICMP_conns; + uint64 cumulative_ICMP_conns; + + int num_fragments; int max_fragments; + uint64 num_packets; + int num_timers; int max_timers; + uint64 num_events_queued; + uint64 num_events_dispatched; }; // Drains and deletes a timer manager if it hasn't seen any advances @@ -242,7 +247,7 @@ protected: OSFingerprint* SYN_OS_Fingerprinter; int build_backdoor_analyzer; int dump_this_packet; // if true, current packet should be recorded - int num_packets_processed; + uint64 num_packets_processed; PacketProfiler* pkt_profiler; // We may use independent timer managers for different sets of related diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 5b88d2dafb..0095947071 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -5,9 +5,6 @@ #include "analyzer/protocol/tcp/TCP.h" #include "TCP_Endpoint.h" -// Only needed for gap_report events. -#include "Event.h" - #include "events.bif.h" using namespace analyzer::tcp; @@ -18,17 +15,11 @@ const bool DEBUG_tcp_contents = false; const bool DEBUG_tcp_connection_close = false; const bool DEBUG_tcp_match_undelivered = false; -static double last_gap_report = 0.0; -static uint64 last_ack_events = 0; -static uint64 last_ack_bytes = 0; -static uint64 last_gap_events = 0; -static uint64 last_gap_bytes = 0; - TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer, TCP_Analyzer* arg_tcp_analyzer, TCP_Reassembler::Type arg_type, TCP_Endpoint* arg_endp) - : Reassembler(1) + : Reassembler(1, REASSEM_TCP) { dst_analyzer = arg_dst_analyzer; tcp_analyzer = arg_tcp_analyzer; @@ -45,7 +36,7 @@ TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer, if ( tcp_max_old_segments ) SetMaxOldBlocks(tcp_max_old_segments); - if ( tcp_contents ) + if ( ::tcp_contents ) { // Val dst_port_val(ntohs(Conn()->RespPort()), TYPE_PORT); PortVal dst_port_val(ntohs(tcp_analyzer->Conn()->RespPort()), @@ -387,7 +378,6 @@ void TCP_Reassembler::BlockInserted(DataBlock* start_block) { // New stuff. uint64 len = b->Size(); uint64 seq = last_reassem_seq; - last_reassem_seq += len; if ( record_contents_file ) @@ -548,35 +538,6 @@ void TCP_Reassembler::AckReceived(uint64 seq) tot_gap_bytes += num_missing; tcp_analyzer->Event(ack_above_hole); } - - double dt = network_time - last_gap_report; - - if ( gap_report && gap_report_freq > 0.0 && - dt >= gap_report_freq ) - { - uint64 devents = tot_ack_events - last_ack_events; - uint64 dbytes = tot_ack_bytes - last_ack_bytes; - uint64 dgaps = tot_gap_events - last_gap_events; - uint64 dgap_bytes = tot_gap_bytes - last_gap_bytes; - - RecordVal* r = new RecordVal(gap_info); - r->Assign(0, new Val(devents, TYPE_COUNT)); - r->Assign(1, new Val(dbytes, TYPE_COUNT)); - r->Assign(2, new Val(dgaps, TYPE_COUNT)); - r->Assign(3, new Val(dgap_bytes, TYPE_COUNT)); - - val_list* vl = new val_list; - vl->append(new IntervalVal(dt, Seconds)); - vl->append(r); - - mgr.QueueEvent(gap_report, vl); - - last_gap_report = network_time; - last_ack_events = tot_ack_events; - last_ack_bytes = tot_ack_bytes; - last_gap_events = tot_gap_events; - last_gap_bytes = tot_gap_bytes; - } } // Check EOF here because t_reassem->LastReassemSeq() may have diff --git a/src/bro.bif b/src/bro.bif index b0465b9609..89e132ca24 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1763,20 +1763,38 @@ function resource_usage%(%): bro_resources #define ADD_STAT(x) \ res->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); - ADD_STAT(s.num_TCP_conns); - ADD_STAT(s.num_UDP_conns); - ADD_STAT(s.num_ICMP_conns); - ADD_STAT(s.num_fragments); ADD_STAT(s.num_packets); - ADD_STAT(s.num_timers); - ADD_STAT(s.num_events_queued); - ADD_STAT(s.num_events_dispatched); - ADD_STAT(s.max_TCP_conns); - ADD_STAT(s.max_UDP_conns); - ADD_STAT(s.max_ICMP_conns); + ADD_STAT(s.num_fragments); ADD_STAT(s.max_fragments); + + ADD_STAT(s.num_TCP_conns); + ADD_STAT(s.max_TCP_conns); + ADD_STAT(s.cumulative_TCP_conns); + + ADD_STAT(s.num_UDP_conns); + ADD_STAT(s.max_UDP_conns); + ADD_STAT(s.cumulative_UDP_conns); + + ADD_STAT(s.num_ICMP_conns); + ADD_STAT(s.max_ICMP_conns); + ADD_STAT(s.cumulative_ICMP_conns); + + ADD_STAT(s.num_timers); ADD_STAT(s.max_timers); + ADD_STAT(s.mem); + ADD_STAT(s.num_events_dispatched); + + ADD_STAT(Connection::TotalConnections()); + ADD_STAT(Connection::CurrentConnections()); + ADD_STAT(Connection::CurrentExternalConnections()); + ADD_STAT(sessions->CurrentConnections()); + + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FILE)); + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FRAG)); + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_TCP)); + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)); + return res; %} diff --git a/src/file_analysis/FileReassembler.cc b/src/file_analysis/FileReassembler.cc index 8b678e5209..ba15086320 100644 --- a/src/file_analysis/FileReassembler.cc +++ b/src/file_analysis/FileReassembler.cc @@ -8,7 +8,7 @@ namespace file_analysis { class File; FileReassembler::FileReassembler(File *f, uint64 starting_offset) - : Reassembler(starting_offset), the_file(f), flushing(false) + : Reassembler(starting_offset, REASSEM_FILE), the_file(f), flushing(false) { } From 88517230b6e5b8239a476096f290040d335e6dea Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 4 Jan 2016 00:57:11 -0500 Subject: [PATCH 04/53] Fix memory usage collection on Mac OS X. - getrusage is broken on Mac OS X, but there is a Mach API available which can collect the same memory usage information. --- bro-config.h.in | 3 +++ src/util.cc | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/bro-config.h.in b/bro-config.h.in index 755a9eee98..0937950604 100644 --- a/bro-config.h.in +++ b/bro-config.h.in @@ -14,6 +14,9 @@ /* We are on a Linux system */ #cmakedefine HAVE_LINUX +/* We are on a Mac OS X (Darwin) system */ +#cmakedefine HAVE_DARWIN + /* Define if you have the `mallinfo' function. */ #cmakedefine HAVE_MALLINFO diff --git a/src/util.cc b/src/util.cc index 6a03859a3c..facbab295f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -14,6 +14,11 @@ # endif #endif +#ifdef HAVE_DARWIN +#include +#include +#endif + #include #include #include @@ -1662,11 +1667,24 @@ void get_memory_usage(unsigned int* total, unsigned int* malloced) #endif +#ifdef HAVE_DARWIN + struct task_basic_info t_info; + mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; + + if ( KERN_SUCCESS != task_info(mach_task_self(), + TASK_BASIC_INFO, + (task_info_t)&t_info, + &t_info_count) ) + ret_total = 0; + else + ret_total = t_info.resident_size; +#else struct rusage r; getrusage(RUSAGE_SELF, &r); // In KB. ret_total = r.ru_maxrss * 1024; +#endif if ( total ) *total = ret_total; From 5a4859afe1f59321a354dd9b169d8931d8fb4de7 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 4 Jan 2016 00:59:26 -0500 Subject: [PATCH 05/53] Updating the cmake submodule for the stats updates. --- cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake b/cmake index 843cdf6a91..23773d7107 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 843cdf6a91f06e5407bffbc79a343bff3cf4c81f +Subproject commit 23773d7107e8d51e2b1bb0fd2e2d85fda50df743 From 13cf6e61122099c08aa6a156a629e6f8a6384514 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 5 Jan 2016 09:26:56 -0500 Subject: [PATCH 06/53] Fixing some small mistakes. --- scripts/base/init-bare.bro | 4 ++-- src/bro.bif | 25 ++++++++++--------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 337052178d..f49bf89d18 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -475,11 +475,11 @@ type bro_resources: record { blocking_input: count; ##< Blocking input operations. blocking_output: count; ##< Blocking output operations. num_context: count; ##< Number of involuntary context switches. - + num_packets: count; ##< Total number of packets processed to date. num_fragments: count; ##< Current number of fragments pending reassembly. max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. - + num_tcp_conns: count; ##< Current number of TCP connections in memory. max_tcp_conns: count; ##< Maximum number of concurrent TCP connections so far. cumulative_tcp_conns: count; ##< diff --git a/src/bro.bif b/src/bro.bif index 89e132ca24..948fc62684 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1766,34 +1766,29 @@ function resource_usage%(%): bro_resources ADD_STAT(s.num_packets); ADD_STAT(s.num_fragments); ADD_STAT(s.max_fragments); - ADD_STAT(s.num_TCP_conns); ADD_STAT(s.max_TCP_conns); ADD_STAT(s.cumulative_TCP_conns); - ADD_STAT(s.num_UDP_conns); ADD_STAT(s.max_UDP_conns); ADD_STAT(s.cumulative_UDP_conns); - ADD_STAT(s.num_ICMP_conns); ADD_STAT(s.max_ICMP_conns); ADD_STAT(s.cumulative_ICMP_conns); - ADD_STAT(s.num_timers); ADD_STAT(s.max_timers); - - ADD_STAT(s.mem); + ADD_STAT(s.num_events_queued); ADD_STAT(s.num_events_dispatched); - ADD_STAT(Connection::TotalConnections()); - ADD_STAT(Connection::CurrentConnections()); - ADD_STAT(Connection::CurrentExternalConnections()); - ADD_STAT(sessions->CurrentConnections()); - - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FILE)); - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FRAG)); - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_TCP)); - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)); + res->Assign(n++, new Val(unsigned(Connection::TotalConnections()), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Connection::CurrentConnections()), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Connection::CurrentExternalConnections()), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(sessions->CurrentConnections()), TYPE_COUNT)); + + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FILE)), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FRAG)), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_TCP)), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)), TYPE_COUNT)); return res; %} From 6aeeb94d760e9860b29eadcc52548721c9a3c630 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 6 Jan 2016 22:28:57 -0500 Subject: [PATCH 07/53] Slight change to Mach API for collecting memory usage. --- src/util.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/util.cc b/src/util.cc index facbab295f..9a4b4de9f6 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1664,15 +1664,14 @@ void get_memory_usage(unsigned int* total, unsigned int* malloced) if ( malloced ) *malloced = mi.uordblks; - #endif #ifdef HAVE_DARWIN - struct task_basic_info t_info; - mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; + struct mach_task_basic_info t_info; + mach_msg_type_number_t t_info_count = MACH_TASK_BASIC_INFO; if ( KERN_SUCCESS != task_info(mach_task_self(), - TASK_BASIC_INFO, + MACH_TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count) ) ret_total = 0; From 6d836b795648901558df09e3125fa40153f5c670 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 7 Jan 2016 16:20:24 -0500 Subject: [PATCH 08/53] More stats improvements Broke out the stats collection into a bunch of new Bifs in stats.bif. Scripts that use stats collection functions have also been updated. More work to do. --- .../frameworks/packet-filter/netstats.bro | 4 +- scripts/base/init-bare.bro | 123 +++++--- .../base/misc/find-checksum-offloading.bro | 2 +- scripts/policy/misc/capture-loss.bro | 2 +- scripts/policy/misc/stats.bro | 62 ++-- src/CMakeLists.txt | 1 + src/Conn.cc | 6 +- src/Conn.h | 12 +- src/DFA.cc | 17 +- src/DFA.h | 3 +- src/Func.cc | 20 +- src/NFA.cc | 5 - src/NFA.h | 1 - src/NetVar.cc | 1 - src/NetVar.h | 3 - src/Sessions.cc | 4 - src/Sessions.h | 4 - src/Stats.cc | 16 +- src/Stats.h | 8 +- src/analyzer/protocol/tcp/functions.bif | 20 -- src/bro.bif | 177 +---------- src/event.bif | 20 -- src/file_analysis/Manager.h | 9 + src/main.cc | 12 +- src/stats.bif | 293 ++++++++++++++++++ src/util.cc | 4 +- src/util.h | 3 +- 27 files changed, 479 insertions(+), 353 deletions(-) create mode 100644 src/stats.bif diff --git a/scripts/base/frameworks/packet-filter/netstats.bro b/scripts/base/frameworks/packet-filter/netstats.bro index b5ffe24f54..f1757d8d47 100644 --- a/scripts/base/frameworks/packet-filter/netstats.bro +++ b/scripts/base/frameworks/packet-filter/netstats.bro @@ -18,7 +18,7 @@ export { event net_stats_update(last_stat: NetStats) { - local ns = net_stats(); + local ns = get_net_stats(); local new_dropped = ns$pkts_dropped - last_stat$pkts_dropped; if ( new_dropped > 0 ) { @@ -38,5 +38,5 @@ event bro_init() # Since this currently only calculates packet drops, let's skip the stats # collection if reading traces. if ( ! reading_traces() ) - schedule stats_collection_interval { net_stats_update(net_stats()) }; + schedule stats_collection_interval { net_stats_update(get_net_stats()) }; } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index f49bf89d18..fa9149c674 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -455,30 +455,15 @@ type NetStats: record { bytes_recvd: count &default=0; ##< Bytes received by Bro. }; -## Statistics about Bro's resource consumption. -## -## .. bro:see:: resource_usage -## -## .. note:: All process-level values refer to Bro's main process only, not to -## the child process it spawns for doing communication. -type bro_resources: record { - version: string; ##< Bro version string. - debug: bool; ##< True if compiled with --enable-debug. - start_time: time; ##< Start time of process. - real_time: interval; ##< Elapsed real time since Bro started running. - user_time: interval; ##< User CPU seconds. - system_time: interval; ##< System CPU seconds. - mem: count; ##< Maximum memory consumed, in KB. - minor_faults: count; ##< Page faults not requiring actual I/O. - major_faults: count; ##< Page faults requiring actual I/O. - num_swap: count; ##< Times swapped out. - blocking_input: count; ##< Blocking input operations. - blocking_output: count; ##< Blocking output operations. - num_context: count; ##< Number of involuntary context switches. +type ConnStats: record { + total_conns: count; ##< + current_conns: count; ##< + current_conns_extern: count; ##< + sess_current_conns: count; ##< - num_packets: count; ##< Total number of packets processed to date. - num_fragments: count; ##< Current number of fragments pending reassembly. - max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. + num_packets: count; + num_fragments: count; + max_fragments: count; num_tcp_conns: count; ##< Current number of TCP connections in memory. max_tcp_conns: count; ##< Maximum number of concurrent TCP connections so far. @@ -492,46 +477,96 @@ type bro_resources: record { max_icmp_conns: count; ##< Maximum number of concurrent ICMP connections so far. cumulative_icmp_conns: count; ##< - num_timers: count; ##< Current number of pending timers. - max_timers: count; ##< Maximum number of concurrent timers pending so far. + killed_by_inactivity: count; +}; +## Statistics about Bro's process. +## +## .. bro:see:: get_proc_stats +## +## .. note:: All process-level values refer to Bro's main process only, not to +## the child process it spawns for doing communication. +type ProcStats: record { + debug: bool; ##< True if compiled with --enable-debug. + start_time: time; ##< Start time of process. + real_time: interval; ##< Elapsed real time since Bro started running. + user_time: interval; ##< User CPU seconds. + system_time: interval; ##< System CPU seconds. + mem: count; ##< Maximum memory consumed, in KB. + minor_faults: count; ##< Page faults not requiring actual I/O. + major_faults: count; ##< Page faults requiring actual I/O. + num_swap: count; ##< Times swapped out. + blocking_input: count; ##< Blocking input operations. + blocking_output: count; ##< Blocking output operations. + num_context: count; ##< Number of involuntary context switches. +}; + +type EventStats: record { num_events_queued: count; ##< Total number of events queued so far. num_events_dispatched: count; ##< Total number of events dispatched so far. +}; - total_conns: count; ##< - current_conns: count; ##< - current_conns_extern: count; ##< - sess_current_conns: count; ##< - - reassem_file_size: count; ##< Size of File reassembly tracking. - reassem_frag_size: count; ##< Size of Fragment reassembly tracking. - reassem_tcp_size: count; ##< Size of TCP reassembly tracking. - reassem_unknown_size: count; ##< Size of reassembly tracking for unknown purposes. +## Summary statistics of all regular expression matchers. +## +## .. bro:see:: get_reassembler_stats +type ReassemblerStats: record { + file_size: count; ##< Byte size of File reassembly tracking. + frag_size: count; ##< Byte size of Fragment reassembly tracking. + tcp_size: count; ##< Byte size of TCP reassembly tracking. + unknown_size: count; ##< Byte size of reassembly tracking for unknown purposes. }; ## Summary statistics of all regular expression matchers. ## ## .. bro:see:: get_matcher_stats -type matcher_stats: record { - matchers: count; ##< Number of distinct RE matchers. - dfa_states: count; ##< Number of DFA states across all matchers. - computed: count; ##< Number of computed DFA state transitions. - mem: count; ##< Number of bytes used by DFA states. - hits: count; ##< Number of cache hits. - misses: count; ##< Number of cache misses. - avg_nfa_states: count; ##< Average number of NFA states across all matchers. +type MatcherStats: record { + matchers: count; ##< Number of distinct RE matchers. + dfa_states: count; ##< Number of DFA states across all matchers. + computed: count; ##< Number of computed DFA state transitions. + mem: count; ##< Number of bytes used by DFA states. + hits: count; ##< Number of cache hits. + misses: count; ##< Number of cache misses. + avg_nfa_states: count; ##< Average number of NFA states across all matchers. +}; + +type TimerStats: record { + num_timers: count; ##< Current number of pending timers. + max_timers: count; ##< Maximum number of concurrent timers pending so far. +}; + +type FileAnalysisStats: record { + current: count; + max: count; + cumulative: count; +}; + +type DNSStats: record { + requests: count; + successful: count; + failed: count; + pending: count; + cached_hosts: count; + cached_addresses: count; }; ## Statistics about number of gaps in TCP connections. ## -## .. bro:see:: get_gap_summary -type gap_info: record { +## .. bro:see:: get_gap_stats +type GapStats: record { ack_events: count; ##< How many ack events *could* have had gaps. ack_bytes: count; ##< How many bytes those covered. gap_events: count; ##< How many *did* have gaps. gap_bytes: count; ##< How many bytes were missing in the gaps. }; +type PatternStats: record { + +}; + +type ThreadStats: record { + num_threads: count; +}; + ## Deprecated. ## ## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere diff --git a/scripts/base/misc/find-checksum-offloading.bro b/scripts/base/misc/find-checksum-offloading.bro index fae017fff1..334cf4a2db 100644 --- a/scripts/base/misc/find-checksum-offloading.bro +++ b/scripts/base/misc/find-checksum-offloading.bro @@ -26,7 +26,7 @@ event ChecksumOffloading::check() if ( done ) return; - local pkts_recvd = net_stats()$pkts_recvd; + local pkts_recvd = get_net_stats()$pkts_recvd; local bad_ip_checksum_pct = (pkts_recvd != 0) ? (bad_ip_checksums*1.0 / pkts_recvd*1.0) : 0; local bad_tcp_checksum_pct = (pkts_recvd != 0) ? (bad_tcp_checksums*1.0 / pkts_recvd*1.0) : 0; local bad_udp_checksum_pct = (pkts_recvd != 0) ? (bad_udp_checksums*1.0 / pkts_recvd*1.0) : 0; diff --git a/scripts/policy/misc/capture-loss.bro b/scripts/policy/misc/capture-loss.bro index 28f468a1c8..648e3d6717 100644 --- a/scripts/policy/misc/capture-loss.bro +++ b/scripts/policy/misc/capture-loss.bro @@ -56,7 +56,7 @@ event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps: } local now = network_time(); - local g = get_gap_summary(); + local g = get_gap_stats(); local acks = g$ack_events - last_acks; local gaps = g$gap_events - last_gaps; local pct_lost = (acks == 0) ? 0.0 : (100 * (1.0 * gaps) / (1.0 * acks)); diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 484267898c..877d32130b 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -1,6 +1,4 @@ -##! Log memory/packet/lag statistics. Differs from -##! :doc:`/scripts/policy/misc/profiling.bro` in that this -##! is lighter-weight (much less info, and less load to generate). +##! Log memory/packet/lag statistics. @load base/frameworks/notice @@ -10,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 5min &redef; + const stats_report_interval = 1sec &redef; type Info: record { ## Timestamp for the measurement. @@ -27,12 +25,19 @@ export { ## interval. events_queued: count &log; + ## TCP connections currently in memory. + active_tcp_conns: count &log; + ## UDP connections currently in memory. + active_udp_conns: count &log; + ## ICMP connections currently in memory. + active_icmp_conns: count &log; + ## TCP connections seen since last stats interval. - tcp_conns: count &log; + tcp_conns: count &log; ## UDP connections seen since last stats interval. - udp_conns: count &log; + udp_conns: count &log; ## ICMP connections seen since last stats interval. - icmp_conns: count &log; + icmp_conns: count &log; ## Current size of TCP data in reassembly. reassem_tcp_size: count &log; @@ -69,11 +74,14 @@ event bro_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats"]); } -event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) +event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats) { local now = current_time(); - local ns = net_stats(); - local res = resource_usage(); + local ns = get_net_stats(); + local cs = get_conn_stats(); + local ps = get_proc_stats(); + local es = get_event_stats(); + local rs = get_reassembler_stats(); if ( bro_is_terminating() ) # No more stats will be written or scheduled when Bro is @@ -82,21 +90,27 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) local info: Info = [$ts=now, $peer=peer_description, - $mem=res$mem/1000000, - $pkts_proc=res$num_packets - last_res$num_packets, - $events_proc=res$num_events_dispatched - last_res$num_events_dispatched, - $events_queued=res$num_events_queued - last_res$num_events_queued, - $tcp_conns=res$cumulative_tcp_conns - last_res$cumulative_tcp_conns, - $udp_conns=res$cumulative_udp_conns - last_res$cumulative_udp_conns, - $icmp_conns=res$cumulative_icmp_conns - last_res$cumulative_icmp_conns, - $reassem_tcp_size=res$reassem_tcp_size, - $reassem_file_size=res$reassem_file_size, - $reassem_frag_size=res$reassem_frag_size, - $reassem_unknown_size=res$reassem_unknown_size + $mem=ps$mem/1000000, + $pkts_proc=ns$pkts_recvd - last_ns$pkts_recvd, + + $active_tcp_conns=cs$num_tcp_conns, + $tcp_conns=cs$cumulative_tcp_conns - last_cs$cumulative_tcp_conns, + $active_udp_conns=cs$num_udp_conns, + $udp_conns=cs$cumulative_udp_conns - last_cs$cumulative_udp_conns, + $active_icmp_conns=cs$num_icmp_conns, + $icmp_conns=cs$cumulative_icmp_conns - last_cs$cumulative_icmp_conns, + + $reassem_tcp_size=rs$tcp_size, + $reassem_file_size=rs$file_size, + $reassem_frag_size=rs$frag_size, + $reassem_unknown_size=rs$unknown_size, + + $events_proc=es$num_events_dispatched - last_es$num_events_dispatched, + $events_queued=es$num_events_queued - last_es$num_events_queued ]; # Someone's going to have to explain what this is and add a field to the Info record. - # info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts); + # info$util = 100.0*((ps$user_time + ps$system_time) - (last_ps$user_time + last_ps$system_time))/(now-last_ts); if ( reading_live_traffic() ) { @@ -108,10 +122,10 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(now, ns, res) }; + schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs) }; } event bro_init() { - schedule stats_report_interval { check_stats(current_time(), net_stats(), resource_usage()) }; + schedule stats_report_interval { check_stats(current_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats()) }; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9a807b3182..7b521125e4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -118,6 +118,7 @@ include(BifCl) set(BIF_SRCS bro.bif + stats.bif event.bif const.bif types.bif diff --git a/src/Conn.cc b/src/Conn.cc index 3f6757d89c..1082230869 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -108,9 +108,9 @@ bool ConnectionTimer::DoUnserialize(UnserialInfo* info) return true; } -unsigned int Connection::total_connections = 0; -unsigned int Connection::current_connections = 0; -unsigned int Connection::external_connections = 0; +uint64 Connection::total_connections = 0; +uint64 Connection::current_connections = 0; +uint64 Connection::external_connections = 0; IMPLEMENT_SERIAL(Connection, SER_CONNECTION); diff --git a/src/Conn.h b/src/Conn.h index 7a4331f91d..ffbc115e6e 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -220,11 +220,11 @@ public: unsigned int MemoryAllocation() const; unsigned int MemoryAllocationConnVal() const; - static unsigned int TotalConnections() + static uint64 TotalConnections() { return total_connections; } - static unsigned int CurrentConnections() + static uint64 CurrentConnections() { return current_connections; } - static unsigned int CurrentExternalConnections() + static uint64 CurrentExternalConnections() { return external_connections; } // Returns true if the history was already seen, false otherwise. @@ -315,9 +315,9 @@ protected: unsigned int saw_first_orig_packet:1, saw_first_resp_packet:1; // Count number of connections. - static unsigned int total_connections; - static unsigned int current_connections; - static unsigned int external_connections; + static uint64 total_connections; + static uint64 current_connections; + static uint64 external_connections; string history; uint32 hist_seen; diff --git a/src/DFA.cc b/src/DFA.cc index e7b2279ed5..9b8b3e5d31 100644 --- a/src/DFA.cc +++ b/src/DFA.cc @@ -9,6 +9,8 @@ unsigned int DFA_State::transition_counter = 0; +uint64 total_dfa_states = 0; + DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, NFA_state_list* arg_nfa_states, AcceptingSet* arg_accept) @@ -20,6 +22,8 @@ DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, mark = 0; centry = 0; + ++total_dfa_states; + SymPartition(ec); xtions = new DFA_State*[num_sym]; @@ -433,19 +437,6 @@ void DFA_Machine::Dump(FILE* f) start_state->ClearMarks(); } -void DFA_Machine::DumpStats(FILE* f) - { - DFA_State_Cache::Stats stats; - dfa_state_cache->GetStats(&stats); - - fprintf(f, "Computed dfa_states = %d; Classes = %d; Computed trans. = %d; Uncomputed trans. = %d\n", - stats.dfa_states, EC()->NumClasses(), - stats.computed, stats.uncomputed); - - fprintf(f, "DFA cache hits = %d; misses = %d\n", - stats.hits, stats.misses); - } - unsigned int DFA_Machine::MemoryAllocation() const { DFA_State_Cache::Stats s; diff --git a/src/DFA.h b/src/DFA.h index 00cfdc3d39..c329b929d4 100644 --- a/src/DFA.h +++ b/src/DFA.h @@ -19,6 +19,8 @@ class DFA_Machine; class DFA_State; struct CacheEntry; +extern uint64 total_dfa_states; + class DFA_State : public BroObj { public: DFA_State(int state_num, const EquivClass* ec, @@ -132,7 +134,6 @@ public: void Describe(ODesc* d) const; void Dump(FILE* f); - void DumpStats(FILE* f); unsigned int MemoryAllocation() const; diff --git a/src/Func.cc b/src/Func.cc index e1eadb8c9f..ac3cda6dd6 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -628,10 +628,12 @@ void builtin_error(const char* msg, BroObj* arg) } #include "bro.bif.func_h" +#include "stats.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" #include "bro.bif.func_def" +#include "stats.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" @@ -640,13 +642,23 @@ void builtin_error(const char* msg, BroObj* arg) void init_builtin_funcs() { - bro_resources = internal_type("bro_resources")->AsRecordType(); - net_stats = internal_type("NetStats")->AsRecordType(); - matcher_stats = internal_type("matcher_stats")->AsRecordType(); + ProcStats = internal_type("ProcStats")->AsRecordType(); + NetStats = internal_type("NetStats")->AsRecordType(); + MatcherStats = internal_type("MatcherStats")->AsRecordType(); + ConnStats = internal_type("ConnStats")->AsRecordType(); + ReassemblerStats = internal_type("ReassemblerStats")->AsRecordType(); + DNSStats = internal_type("DNSStats")->AsRecordType(); + GapStats = internal_type("GapStats")->AsRecordType(); + EventStats = internal_type("EventStats")->AsRecordType(); + TimerStats = internal_type("TimerStats")->AsRecordType(); + FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType(); + ThreadStats = internal_type("ThreadStats")->AsRecordType(); + PatternStats = internal_type("PatternStats")->AsRecordType(); + var_sizes = internal_type("var_sizes")->AsTableType(); - gap_info = internal_type("gap_info")->AsRecordType(); #include "bro.bif.func_init" +#include "stats.bif.func_init" #include "reporter.bif.func_init" #include "strings.bif.func_init" diff --git a/src/NFA.cc b/src/NFA.cc index def04d79a1..4d18f75226 100644 --- a/src/NFA.cc +++ b/src/NFA.cc @@ -285,11 +285,6 @@ void NFA_Machine::Dump(FILE* f) first_state->ClearMarks(); } -void NFA_Machine::DumpStats(FILE* f) - { - fprintf(f, "highest NFA state ID is %d\n", nfa_state_id); - } - NFA_Machine* make_alternate(NFA_Machine* m1, NFA_Machine* m2) { if ( ! m1 ) diff --git a/src/NFA.h b/src/NFA.h index 9877b8787c..88ce3429c9 100644 --- a/src/NFA.h +++ b/src/NFA.h @@ -105,7 +105,6 @@ public: void Describe(ODesc* d) const; void Dump(FILE* f); - void DumpStats(FILE* f); unsigned int MemoryAllocation() const { return padded_sizeof(*this) + first_state->TotalMemoryAllocation(); } diff --git a/src/NetVar.cc b/src/NetVar.cc index 8a901842fd..457fcae0ce 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -197,7 +197,6 @@ Val* pkt_profile_file; int load_sample_freq; double gap_report_freq; -RecordType* gap_info; int packet_filter_default; diff --git a/src/NetVar.h b/src/NetVar.h index 97018121f9..582abffe65 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -200,9 +200,6 @@ extern Val* pkt_profile_file; extern int load_sample_freq; -extern double gap_report_freq; -extern RecordType* gap_info; - extern int packet_filter_default; extern int sig_max_group_size; diff --git a/src/Sessions.cc b/src/Sessions.cc index 3194985515..aae6712ef2 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1163,15 +1163,11 @@ void NetSessions::GetStats(SessionStats& s) const s.cumulative_ICMP_conns = icmp_conns.NumCumulativeInserts(); s.num_fragments = fragments.Length(); s.num_packets = num_packets_processed; - s.num_timers = timer_mgr->Size(); - s.num_events_queued = num_events_queued; - s.num_events_dispatched = num_events_dispatched; s.max_TCP_conns = tcp_conns.MaxLength(); s.max_UDP_conns = udp_conns.MaxLength(); s.max_ICMP_conns = icmp_conns.MaxLength(); s.max_fragments = fragments.MaxLength(); - s.max_timers = timer_mgr->PeakSize(); } Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, diff --git a/src/Sessions.h b/src/Sessions.h index e8c53256ff..8da658633c 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -46,10 +46,6 @@ struct SessionStats { int num_fragments; int max_fragments; uint64 num_packets; - int num_timers; - int max_timers; - uint64 num_events_queued; - uint64 num_events_dispatched; }; // Drains and deletes a timer manager if it hasn't seen any advances diff --git a/src/Stats.cc b/src/Stats.cc index 00f603cba7..99e36625b8 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -14,7 +14,7 @@ #include "broker/Manager.h" #endif -int killed_by_inactivity = 0; +uint64 killed_by_inactivity = 0; uint64 tot_ack_events = 0; uint64 tot_ack_bytes = 0; @@ -82,7 +82,7 @@ void ProfileLogger::Log() struct timeval tv_utime = r.ru_utime; struct timeval tv_stime = r.ru_stime; - unsigned int total, malloced; + uint64 total, malloced; get_memory_usage(&total, &malloced); static unsigned int first_total = 0; @@ -110,7 +110,7 @@ void ProfileLogger::Log() file->Write(fmt("\n%.06f ------------------------\n", network_time)); } - file->Write(fmt("%.06f Memory: total=%dK total_adj=%dK malloced: %dK\n", + file->Write(fmt("%.06f Memory: total=%" PRId64 "K total_adj=%" PRId64 "K malloced: %" PRId64 "K\n", network_time, total / 1024, (total - first_total) / 1024, malloced / 1024)); @@ -120,7 +120,7 @@ void ProfileLogger::Log() int conn_mem_use = expensive ? sessions->ConnectionMemoryUsage() : 0; - file->Write(fmt("%.06f Conns: total=%d current=%d/%d ext=%d mem=%dK avg=%.1f table=%dK connvals=%dK\n", + file->Write(fmt("%.06f Conns: total=%" PRIu64 " current=%" PRIu64 "/%" PRIi32 " ext=%" PRIu64 " mem=%" PRIi32 "K avg=%.1f table=%" PRIu32 "K connvals=%" PRIu32 "K\n", network_time, Connection::TotalConnections(), Connection::CurrentConnections(), @@ -161,10 +161,10 @@ void ProfileLogger::Log() )); */ - file->Write(fmt("%.06f Connections expired due to inactivity: %d\n", + file->Write(fmt("%.06f Connections expired due to inactivity: %" PRIu64 "\n", network_time, killed_by_inactivity)); - file->Write(fmt("%.06f Total reassembler data: %" PRIu64"K\n", network_time, + file->Write(fmt("%.06f Total reassembler data: %" PRIu64 "K\n", network_time, Reassembler::TotalMemoryAllocation() / 1024)); // Signature engine. @@ -465,10 +465,10 @@ void PacketProfiler::ProfilePkt(double t, unsigned int bytes) double curr_Rtime = ptimestamp.tv_sec + ptimestamp.tv_usec / 1e6; - unsigned int curr_mem; + uint64 curr_mem; get_memory_usage(&curr_mem, 0); - file->Write(fmt("%.06f %.03f %d %d %.03f %.03f %.03f %d\n", + file->Write(fmt("%.06f %.03f %" PRIu64 " %" PRIu64 " %.03f %.03f %.03f %" PRIu64 "\n", t, time-last_timestamp, pkt_cnt, byte_cnt, curr_Rtime - last_Rtime, curr_Utime - last_Utime, diff --git a/src/Stats.h b/src/Stats.h index 1bcc2e18dc..7fbec8cab6 100644 --- a/src/Stats.h +++ b/src/Stats.h @@ -102,7 +102,7 @@ extern ProfileLogger* segment_logger; extern SampleLogger* sample_logger; // Connection statistics. -extern int killed_by_inactivity; +extern uint64 killed_by_inactivity; // Content gap statistics. extern uint64 tot_ack_events; @@ -127,9 +127,9 @@ protected: double update_freq; double last_Utime, last_Stime, last_Rtime; double last_timestamp, time; - unsigned int last_mem; - unsigned int pkt_cnt; - unsigned int byte_cnt; + uint64 last_mem; + uint64 pkt_cnt; + uint64 byte_cnt; }; #endif diff --git a/src/analyzer/protocol/tcp/functions.bif b/src/analyzer/protocol/tcp/functions.bif index 9fca05329a..75353180c6 100644 --- a/src/analyzer/protocol/tcp/functions.bif +++ b/src/analyzer/protocol/tcp/functions.bif @@ -63,26 +63,6 @@ function get_resp_seq%(cid: conn_id%): count } %} -## Returns statistics about TCP gaps. -## -## Returns: A record with TCP gap statistics. -## -## .. bro:see:: do_profiling -## net_stats -## resource_usage -## dump_rule_stats -## get_matcher_stats -function get_gap_summary%(%): gap_info - %{ - RecordVal* r = new RecordVal(gap_info); - r->Assign(0, new Val(tot_ack_events, TYPE_COUNT)); - r->Assign(1, new Val(tot_ack_bytes, TYPE_COUNT)); - r->Assign(2, new Val(tot_gap_events, TYPE_COUNT)); - r->Assign(3, new Val(tot_gap_bytes, TYPE_COUNT)); - - return r; - %} - ## Associates a file handle with a connection for writing TCP byte stream ## contents. ## diff --git a/src/bro.bif b/src/bro.bif index 948fc62684..ce16695afa 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -26,15 +26,8 @@ using namespace std; -RecordType* net_stats; -RecordType* bro_resources; -RecordType* matcher_stats; TableType* var_sizes; -// This one is extern, since it's used beyond just built-ins, -// and hence it's declared in NetVar.{h,cc}. -extern RecordType* gap_info; - static iosource::PktDumper* addl_pkt_dumper = 0; bro_int_t parse_int(const char*& fmt) @@ -1661,169 +1654,6 @@ function reading_traces%(%): bool return new Val(reading_traces, TYPE_BOOL); %} -## Returns packet capture statistics. Statistics include the number of -## packets *(i)* received by Bro, *(ii)* dropped, and *(iii)* seen on the -## link (not always available). -## -## Returns: A record of packet statistics. -## -## .. bro:see:: do_profiling -## resource_usage -## get_matcher_stats -## dump_rule_stats -## get_gap_summary -function net_stats%(%): NetStats - %{ - unsigned int recv = 0; - unsigned int drop = 0; - unsigned int link = 0; - unsigned int bytes_recv = 0; - - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) - { - iosource::PktSrc* ps = *i; - - struct iosource::PktSrc::Stats stat; - ps->Statistics(&stat); - recv += stat.received; - drop += stat.dropped; - link += stat.link; - bytes_recv += stat.bytes_received; - } - - RecordVal* ns = new RecordVal(net_stats); - ns->Assign(0, new Val(recv, TYPE_COUNT)); - ns->Assign(1, new Val(drop, TYPE_COUNT)); - ns->Assign(2, new Val(link, TYPE_COUNT)); - ns->Assign(3, new Val(bytes_recv, TYPE_COUNT)); - - return ns; - %} - -## Returns Bro process statistics. Statistics include real/user/sys CPU time, -## memory usage, page faults, number of TCP/UDP/ICMP connections, timers, -## and events queued/dispatched. -## -## Returns: A record with resource usage statistics. -## -## .. bro:see:: do_profiling -## net_stats -## get_matcher_stats -## dump_rule_stats -## get_gap_summary -function resource_usage%(%): bro_resources - %{ - struct rusage r; - - if ( getrusage(RUSAGE_SELF, &r) < 0 ) - reporter->InternalError("getrusage() failed in bro_resource_usage()"); - - double elapsed_time = current_time() - bro_start_time; - - double user_time = - double(r.ru_utime.tv_sec) + double(r.ru_utime.tv_usec) / 1e6; - double system_time = - double(r.ru_stime.tv_sec) + double(r.ru_stime.tv_usec) / 1e6; - - RecordVal* res = new RecordVal(bro_resources); - int n = 0; - - res->Assign(n++, new StringVal(bro_version())); - -#ifdef DEBUG - res->Assign(n++, new Val(1, TYPE_COUNT)); -#else - res->Assign(n++, new Val(0, TYPE_COUNT)); -#endif - - res->Assign(n++, new Val(bro_start_time, TYPE_TIME)); - - res->Assign(n++, new IntervalVal(elapsed_time, Seconds)); - res->Assign(n++, new IntervalVal(user_time, Seconds)); - res->Assign(n++, new IntervalVal(system_time, Seconds)); - - unsigned int total_mem; - get_memory_usage(&total_mem, 0); - res->Assign(n++, new Val(unsigned(total_mem), TYPE_COUNT)); - - res->Assign(n++, new Val(unsigned(r.ru_minflt), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_majflt), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_nswap), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_inblock), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_oublock), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_nivcsw), TYPE_COUNT)); - - SessionStats s; - if ( sessions ) - sessions->GetStats(s); - -#define ADD_STAT(x) \ - res->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); - - ADD_STAT(s.num_packets); - ADD_STAT(s.num_fragments); - ADD_STAT(s.max_fragments); - ADD_STAT(s.num_TCP_conns); - ADD_STAT(s.max_TCP_conns); - ADD_STAT(s.cumulative_TCP_conns); - ADD_STAT(s.num_UDP_conns); - ADD_STAT(s.max_UDP_conns); - ADD_STAT(s.cumulative_UDP_conns); - ADD_STAT(s.num_ICMP_conns); - ADD_STAT(s.max_ICMP_conns); - ADD_STAT(s.cumulative_ICMP_conns); - ADD_STAT(s.num_timers); - ADD_STAT(s.max_timers); - ADD_STAT(s.num_events_queued); - ADD_STAT(s.num_events_dispatched); - - res->Assign(n++, new Val(unsigned(Connection::TotalConnections()), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Connection::CurrentConnections()), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Connection::CurrentExternalConnections()), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(sessions->CurrentConnections()), TYPE_COUNT)); - - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FILE)), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FRAG)), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_TCP)), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)), TYPE_COUNT)); - - return res; - %} - -## Returns statistics about the regular expression engine. Statistics include -## the number of distinct matchers, DFA states, DFA state transitions, memory -## usage of DFA states, cache hits/misses, and average number of NFA states -## across all matchers. -## -## Returns: A record with matcher statistics. -## -## .. bro:see:: do_profiling -## net_stats -## resource_usage -## dump_rule_stats -## get_gap_summary -function get_matcher_stats%(%): matcher_stats - %{ - RuleMatcher::Stats s; - memset(&s, 0, sizeof(s)); - - if ( rule_matcher ) - rule_matcher->GetStats(&s); - - RecordVal* r = new RecordVal(matcher_stats); - r->Assign(0, new Val(s.matchers, TYPE_COUNT)); - r->Assign(1, new Val(s.dfa_states, TYPE_COUNT)); - r->Assign(2, new Val(s.computed, TYPE_COUNT)); - r->Assign(3, new Val(s.mem, TYPE_COUNT)); - r->Assign(4, new Val(s.hits, TYPE_COUNT)); - r->Assign(5, new Val(s.misses, TYPE_COUNT)); - r->Assign(6, new Val(s.avg_nfa_states, TYPE_COUNT)); - - return r; - %} ## Generates a table of the size of all global variables. The table index is ## the variable name and the value is the variable size in bytes. @@ -1964,8 +1794,7 @@ function record_fields%(rec: any%): record_field_table ## .. bro:see:: net_stats ## resource_usage ## get_matcher_stats -## dump_rule_stats -## get_gap_summary +## get_gap_stats function do_profiling%(%) : any %{ if ( profiling_logger ) @@ -2030,8 +1859,8 @@ function is_local_interface%(ip: addr%) : bool ## .. bro:see:: do_profiling ## resource_usage ## get_matcher_stats -## net_stats -## get_gap_summary +## get_net_stats +## get_gap_stats ## ## .. todo:: The return value should be changed to any or check appropriately. function dump_rule_stats%(f: file%): bool diff --git a/src/event.bif b/src/event.bif index ff6ec059fb..aca1086e66 100644 --- a/src/event.bif +++ b/src/event.bif @@ -366,26 +366,6 @@ event ack_above_hole%(c: connection%); ## the two. event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); -## Summarizes the amount of missing TCP payload at regular intervals. -## Internally, Bro tracks (1) the number of :bro:id:`ack_above_hole` events, -## including the number of bytes missing; and (2) the total number of TCP -## acks seen, with the total volume of bytes that have been acked. This event -## reports these statistics in :bro:id:`gap_report_freq` intervals for the -## purpose of determining packet loss. -## -## dt: The time that has passed since the last ``gap_report`` interval. -## -## info: The gap statistics. -## -## .. bro:see:: content_gap ack_above_hole -## -## .. note:: -## -## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that -## uses this event to estimate packet loss and report when a predefined -## threshold is exceeded. -event gap_report%(dt: interval, info: gap_info%); - ## Generated when a protocol analyzer confirms that a connection is indeed ## using that protocol. Bro's dynamic protocol detection heuristically activates ## analyzers as soon as it believes a connection *could* be using a particular diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h index 93c8e7f613..bcc8ac5dd2 100644 --- a/src/file_analysis/Manager.h +++ b/src/file_analysis/Manager.h @@ -302,6 +302,15 @@ public: */ std::string DetectMIME(const u_char* data, uint64 len) const; + uint64 CurrentFiles() + { return id_map.Length(); } + + uint64 MaxFiles() + { return id_map.MaxLength(); } + + uint64 CumulativeFiles() + { return id_map.NumCumulativeInserts(); } + protected: friend class FileTimer; diff --git a/src/main.cc b/src/main.cc index 73181c82f2..a0615d75da 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1172,8 +1172,8 @@ int main(int argc, char** argv) double time_net_start = current_time(true);; - unsigned int mem_net_start_total; - unsigned int mem_net_start_malloced; + uint64 mem_net_start_total; + uint64 mem_net_start_malloced; if ( time_bro ) { @@ -1181,7 +1181,7 @@ int main(int argc, char** argv) fprintf(stderr, "# initialization %.6f\n", time_net_start - time_start); - fprintf(stderr, "# initialization %uM/%uM\n", + fprintf(stderr, "# initialization %" PRIu64 "M/%" PRIu64 "M\n", mem_net_start_total / 1024 / 1024, mem_net_start_malloced / 1024 / 1024); } @@ -1190,8 +1190,8 @@ int main(int argc, char** argv) double time_net_done = current_time(true);; - unsigned int mem_net_done_total; - unsigned int mem_net_done_malloced; + uint64 mem_net_done_total; + uint64 mem_net_done_malloced; if ( time_bro ) { @@ -1200,7 +1200,7 @@ int main(int argc, char** argv) fprintf(stderr, "# total time %.6f, processing %.6f\n", time_net_done - time_start, time_net_done - time_net_start); - fprintf(stderr, "# total mem %uM/%uM, processing %uM/%uM\n", + fprintf(stderr, "# total mem %" PRId64 "M/%" PRId64 "M, processing %" PRId64 "M/%" PRId64 "M\n", mem_net_done_total / 1024 / 1024, mem_net_done_malloced / 1024 / 1024, (mem_net_done_total - mem_net_start_total) / 1024 / 1024, diff --git a/src/stats.bif b/src/stats.bif new file mode 100644 index 0000000000..d7e812df93 --- /dev/null +++ b/src/stats.bif @@ -0,0 +1,293 @@ + +%%{ // C segment +#include "util.h" +#include "threading/Manager.h" + +RecordType* ProcStats; +RecordType* NetStats; +RecordType* MatcherStats; +RecordType* ReassemblerStats; +RecordType* DNSStats; +RecordType* ConnStats; +RecordType* GapStats; +RecordType* EventStats; +RecordType* ThreadStats; +RecordType* PatternStats; +RecordType* TimerStats; +RecordType* FileAnalysisStats; +%%} + +## Returns packet capture statistics. Statistics include the number of +## packets *(i)* received by Bro, *(ii)* dropped, and *(iii)* seen on the +## link (not always available). +## +## Returns: A record of packet statistics. +## +## .. bro:see:: do_profiling +## get_proc_stats +## get_matcher_stats +## get_gap_stats +function get_net_stats%(%): NetStats + %{ + uint64 recv = 0; + uint64 drop = 0; + uint64 link = 0; + uint64 bytes_recv = 0; + + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + struct iosource::PktSrc::Stats stat; + ps->Statistics(&stat); + recv += stat.received; + drop += stat.dropped; + link += stat.link; + bytes_recv += stat.bytes_received; + } + + RecordVal* r = new RecordVal(NetStats); + int n = 0; + + r->Assign(n++, new Val(recv, TYPE_COUNT)); + r->Assign(n++, new Val(drop, TYPE_COUNT)); + r->Assign(n++, new Val(link, TYPE_COUNT)); + r->Assign(n++, new Val(bytes_recv, TYPE_COUNT)); + + return r; + %} + +function get_conn_stats%(%): ConnStats + %{ + RecordVal* r = new RecordVal(ConnStats); + int n = 0; + + r->Assign(n++, new Val(Connection::TotalConnections(), TYPE_COUNT)); + r->Assign(n++, new Val(Connection::CurrentConnections(), TYPE_COUNT)); + r->Assign(n++, new Val(Connection::CurrentExternalConnections(), TYPE_COUNT)); + r->Assign(n++, new Val(sessions->CurrentConnections(), TYPE_COUNT)); + + SessionStats s; + if ( sessions ) + sessions->GetStats(s); + +#define ADD_STAT(x) \ + r->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); + + ADD_STAT(s.num_packets); + ADD_STAT(s.num_fragments); + ADD_STAT(s.max_fragments); + ADD_STAT(s.num_TCP_conns); + ADD_STAT(s.max_TCP_conns); + ADD_STAT(s.cumulative_TCP_conns); + ADD_STAT(s.num_UDP_conns); + ADD_STAT(s.max_UDP_conns); + ADD_STAT(s.cumulative_UDP_conns); + ADD_STAT(s.num_ICMP_conns); + ADD_STAT(s.max_ICMP_conns); + ADD_STAT(s.cumulative_ICMP_conns); + + r->Assign(n++, new Val(killed_by_inactivity, TYPE_COUNT)); + + return r; + %} + +## Returns Bro process statistics. Statistics include real/user/sys CPU time, +## memory usage, page faults, number of TCP/UDP/ICMP connections, timers, +## and events queued/dispatched. +## +## Returns: A record with resource usage statistics. +## +## .. bro:see:: do_profiling +## get_net_stats +## get_matcher_stats +## get_gap_stats +function get_proc_stats%(%): ProcStats + %{ + struct rusage ru; + if ( getrusage(RUSAGE_SELF, &ru) < 0 ) + reporter->InternalError("getrusage() failed in get_proc_stats()"); + + RecordVal* r = new RecordVal(ProcStats); + int n = 0; + + double elapsed_time = current_time() - bro_start_time; + double user_time = + double(ru.ru_utime.tv_sec) + double(ru.ru_utime.tv_usec) / 1e6; + double system_time = + double(ru.ru_stime.tv_sec) + double(ru.ru_stime.tv_usec) / 1e6; + +#ifdef DEBUG + r->Assign(n++, new Val(1, TYPE_COUNT)); +#else + r->Assign(n++, new Val(0, TYPE_COUNT)); +#endif + + r->Assign(n++, new Val(bro_start_time, TYPE_TIME)); + + r->Assign(n++, new IntervalVal(elapsed_time, Seconds)); + r->Assign(n++, new IntervalVal(user_time, Seconds)); + r->Assign(n++, new IntervalVal(system_time, Seconds)); + + uint64 total_mem; + get_memory_usage(&total_mem, NULL); + r->Assign(n++, new Val(unsigned(total_mem), TYPE_COUNT)); + + r->Assign(n++, new Val(unsigned(ru.ru_minflt), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_majflt), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_nswap), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_inblock), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_oublock), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_nivcsw), TYPE_COUNT)); + + return r; + %} + +function get_event_stats%(%): EventStats + %{ + RecordVal* r = new RecordVal(EventStats); + int n = 0; + + r->Assign(n++, new Val(num_events_queued, TYPE_COUNT)); + r->Assign(n++, new Val(num_events_dispatched, TYPE_COUNT)); + + return r; + %} + +function get_reassembler_stats%(%): ReassemblerStats + %{ + RecordVal* r = new RecordVal(ReassemblerStats); + int n = 0; + + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_FILE), TYPE_COUNT)); + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_FRAG), TYPE_COUNT)); + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_TCP), TYPE_COUNT)); + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_UNKNOWN), TYPE_COUNT)); + + return r; + %} + +function get_dns_stats%(%): DNSStats + %{ + RecordVal* r = new RecordVal(DNSStats); + int n = 0; + + DNS_Mgr::Stats dstats; + dns_mgr->GetStats(&dstats); + + r->Assign(n++, new Val(unsigned(dstats.requests), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.successful), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.failed), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.pending), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.cached_hosts), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.cached_addresses), TYPE_COUNT)); + + return r; + %} + +function get_pattern_stats%(%): PatternStats + %{ + RecordVal* r = new RecordVal(PatternStats); + int n = 0; + + //DFA_State_Cache::Stats stats; + //dfa_state_cache->GetStats(&stats); + + //fprintf(f, "Computed dfa_states = %d; Classes = %d; Computed trans. = %d; Uncomputed trans. = %d\n", + // stats.dfa_states, EC()->NumClasses(), + // stats.computed, stats.uncomputed); +// + //fprintf(f, "DFA cache hits = %d; misses = %d\n", + // stats.hits, stats.misses); + + return r; + + %} + +function get_timer_stats%(%): TimerStats + %{ + RecordVal* r = new RecordVal(TimerStats); + int n = 0; + + r->Assign(n++, new Val(unsigned(timer_mgr->Size()), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(timer_mgr->PeakSize()), TYPE_COUNT)); + + return r; + %} + +function get_file_analysis_stats%(%): FileAnalysisStats + %{ + RecordVal* r = new RecordVal(FileAnalysisStats); + int n = 0; + + r->Assign(n++, new Val(file_mgr->CurrentFiles(), TYPE_COUNT)); + r->Assign(n++, new Val(file_mgr->MaxFiles(), TYPE_COUNT)); + r->Assign(n++, new Val(file_mgr->CumulativeFiles(), TYPE_COUNT)); + + return r; + %} + +function get_thread_stats%(%): ThreadStats + %{ + RecordVal* r = new RecordVal(ThreadStats); + int n = 0; + + r->Assign(n++, new Val(thread_mgr->NumThreads(), TYPE_COUNT)); + + return r; + %} + +## Returns statistics about TCP gaps. +## +## Returns: A record with TCP gap statistics. +## +## .. bro:see:: do_profiling +## get_net_stats +## get_proc_stats +## get_matcher_stats +function get_gap_stats%(%): GapStats + %{ + RecordVal* r = new RecordVal(GapStats); + int n = 0; + + r->Assign(n++, new Val(tot_ack_events, TYPE_COUNT)); + r->Assign(n++, new Val(tot_ack_bytes, TYPE_COUNT)); + r->Assign(n++, new Val(tot_gap_events, TYPE_COUNT)); + r->Assign(n++, new Val(tot_gap_bytes, TYPE_COUNT)); + + return r; + %} + +## Returns statistics about the regular expression engine. Statistics include +## the number of distinct matchers, DFA states, DFA state transitions, memory +## usage of DFA states, cache hits/misses, and average number of NFA states +## across all matchers. +## +## Returns: A record with matcher statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_gap_summary +function get_matcher_stats%(%): MatcherStats + %{ + RecordVal* r = new RecordVal(MatcherStats); + int n = 0; + + RuleMatcher::Stats s; + memset(&s, 0, sizeof(s)); + if ( rule_matcher ) + rule_matcher->GetStats(&s); + + r->Assign(n++, new Val(s.matchers, TYPE_COUNT)); + r->Assign(n++, new Val(s.dfa_states, TYPE_COUNT)); + r->Assign(n++, new Val(s.computed, TYPE_COUNT)); + r->Assign(n++, new Val(s.mem, TYPE_COUNT)); + r->Assign(n++, new Val(s.hits, TYPE_COUNT)); + r->Assign(n++, new Val(s.misses, TYPE_COUNT)); + r->Assign(n++, new Val(s.avg_nfa_states, TYPE_COUNT)); + + return r; + %} diff --git a/src/util.cc b/src/util.cc index 9a4b4de9f6..a6ce473b6c 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1655,9 +1655,9 @@ extern "C" void out_of_memory(const char* where) abort(); } -void get_memory_usage(unsigned int* total, unsigned int* malloced) +void get_memory_usage(uint64* total, uint64* malloced) { - unsigned int ret_total; + uint64 ret_total; #ifdef HAVE_MALLINFO struct mallinfo mi = mallinfo(); diff --git a/src/util.h b/src/util.h index 901bb44d1c..191e5449e1 100644 --- a/src/util.h +++ b/src/util.h @@ -502,8 +502,7 @@ inline int safe_vsnprintf(char* str, size_t size, const char* format, va_list al // Returns total memory allocations and (if available) amount actually // handed out by malloc. -extern void get_memory_usage(unsigned int* total, - unsigned int* malloced); +extern void get_memory_usage(uint64* total, uint64* malloced); // Class to be used as a third argument for STL maps to be able to use // char*'s as keys. Otherwise the pointer values will be compared instead of From 3c71d4ffa8cc063915dd54c461395961368e3866 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 8 Jan 2016 17:03:16 -0500 Subject: [PATCH 09/53] More stats collection extensions. --- scripts/base/init-bare.bro | 5 +++-- scripts/policy/misc/stats.bro | 32 +++++++++++++++++++++++++------- src/PriorityQueue.cc | 3 ++- src/PriorityQueue.h | 3 +++ src/Timer.h | 3 +++ src/cq.c | 9 +++++++++ src/cq.h | 1 + src/stats.bif | 1 + 8 files changed, 47 insertions(+), 10 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index fa9149c674..3d870da38f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -530,8 +530,9 @@ type MatcherStats: record { }; type TimerStats: record { - num_timers: count; ##< Current number of pending timers. - max_timers: count; ##< Maximum number of concurrent timers pending so far. + current: count; ##< Current number of pending timers. + max: count; ##< Maximum number of concurrent timers pending so far. + cumulative: count; }; type FileAnalysisStats: record { diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 877d32130b..a49d377bae 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -39,6 +39,16 @@ export { ## ICMP connections seen since last stats interval. icmp_conns: count &log; + ## Number of timers scheduled since last stats interval. + timers: count &log; + ## Current number of scheduled timers. + active_timers: count &log; + + ## Number of files seen since last stats interval. + files: count &log; + ## Current number of files actively being seen. + active_files: count &log; + ## Current size of TCP data in reassembly. reassem_tcp_size: count &log; ## Current size of File data in reassembly. @@ -74,14 +84,16 @@ event bro_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats"]); } -event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats) +event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats) { - local now = current_time(); + local now = network_time(); local ns = get_net_stats(); local cs = get_conn_stats(); local ps = get_proc_stats(); local es = get_event_stats(); local rs = get_reassembler_stats(); + local ts = get_timer_stats(); + local fs = get_file_analysis_stats(); if ( bro_is_terminating() ) # No more stats will be written or scheduled when Bro is @@ -90,7 +102,7 @@ event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: local info: Info = [$ts=now, $peer=peer_description, - $mem=ps$mem/1000000, + $mem=ps$mem/1048576, $pkts_proc=ns$pkts_recvd - last_ns$pkts_recvd, $active_tcp_conns=cs$num_tcp_conns, @@ -106,11 +118,17 @@ event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: $reassem_unknown_size=rs$unknown_size, $events_proc=es$num_events_dispatched - last_es$num_events_dispatched, - $events_queued=es$num_events_queued - last_es$num_events_queued + $events_queued=es$num_events_queued - last_es$num_events_queued, + + $timers=ts$cumulative - last_ts$cumulative, + $active_timers=ts$current, + + $files=fs$cumulative - last_fs$cumulative, + $active_files=fs$current ]; # Someone's going to have to explain what this is and add a field to the Info record. - # info$util = 100.0*((ps$user_time + ps$system_time) - (last_ps$user_time + last_ps$system_time))/(now-last_ts); + # info$util = 100.0*((ps$user_time + ps$system_time) - (last_ps$user_time + last_ps$system_time))/(now-then); if ( reading_live_traffic() ) { @@ -122,10 +140,10 @@ event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs) }; + schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs, ts, fs) }; } event bro_init() { - schedule stats_report_interval { check_stats(current_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats()) }; + schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats()) }; } diff --git a/src/PriorityQueue.cc b/src/PriorityQueue.cc index 75b731142e..4f969c4830 100644 --- a/src/PriorityQueue.cc +++ b/src/PriorityQueue.cc @@ -13,7 +13,7 @@ PriorityQueue::PriorityQueue(int initial_size) { max_heap_size = initial_size; heap = new PQ_Element*[max_heap_size]; - peak_heap_size = heap_size = 0; + peak_heap_size = heap_size = cumulative_num = 0; } PriorityQueue::~PriorityQueue() @@ -62,6 +62,7 @@ int PriorityQueue::Add(PQ_Element* e) BubbleUp(heap_size); + ++cumulative_num; if ( ++heap_size > peak_heap_size ) peak_heap_size = heap_size; diff --git a/src/PriorityQueue.h b/src/PriorityQueue.h index 87e10aa7ac..bb1caad592 100644 --- a/src/PriorityQueue.h +++ b/src/PriorityQueue.h @@ -4,6 +4,7 @@ #define __PriorityQueue__ #include +#include "util.h" class PriorityQueue; @@ -53,6 +54,7 @@ public: int Size() const { return heap_size; } int PeakSize() const { return peak_heap_size; } + uint64 CumulativeNum() const { return cumulative_num; } protected: int Resize(int new_size); @@ -92,6 +94,7 @@ protected: int heap_size; int peak_heap_size; int max_heap_size; + uint64 cumulative_num; }; #endif diff --git a/src/Timer.h b/src/Timer.h index 615c8bf69a..12d849cac2 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -109,6 +109,7 @@ public: virtual int Size() const = 0; virtual int PeakSize() const = 0; + virtual uint64 CumulativeNum() const = 0; double LastTimestamp() const { return last_timestamp; } // Returns time of last advance in global network time. @@ -148,6 +149,7 @@ public: int Size() const { return q->Size(); } int PeakSize() const { return q->PeakSize(); } + uint64 CumulativeNum() const { return q->CumulativeNum(); } unsigned int MemoryUsage() const; protected: @@ -170,6 +172,7 @@ public: int Size() const { return cq_size(cq); } int PeakSize() const { return cq_max_size(cq); } + uint64 CumulativeNum() const { return cq_cumulative_num(cq); } unsigned int MemoryUsage() const; protected: diff --git a/src/cq.c b/src/cq.c index 8005544400..16153f0a39 100644 --- a/src/cq.c +++ b/src/cq.c @@ -42,6 +42,7 @@ struct cq_handle { int lowmark; /* low bucket threshold */ int nextbucket; /* next bucket to check */ int noresize; /* don't resize while we're resizing */ + uint64_t cumulative_num; /* cumulative entries ever enqueued */ double lastpri; /* last priority */ double ysize; /* length of a year */ double bwidth; /* width of each bucket */ @@ -175,6 +176,7 @@ cq_enqueue(register struct cq_handle *hp, register double pri, } bp->pri = pri; bp->cookie = cookie; + ++hp->cumulative_num; if (++hp->qlen > hp->max_qlen) hp->max_qlen = hp->qlen; #ifdef DEBUG @@ -414,6 +416,13 @@ cq_max_size(struct cq_handle *hp) return hp->max_qlen; } +uint64_t +cq_cumulative_num(struct cq_handle *hp) +{ + return hp->cumulative_num; +} + + /* Return without doing anything if we fail to allocate a new bucket array */ static int cq_resize(register struct cq_handle *hp, register int grow) diff --git a/src/cq.h b/src/cq.h index 540cccde74..c79eefc790 100644 --- a/src/cq.h +++ b/src/cq.h @@ -5,6 +5,7 @@ void *cq_dequeue(struct cq_handle *, double); void *cq_remove(struct cq_handle *, double, void *); int cq_size(struct cq_handle *); int cq_max_size(struct cq_handle *); +uint64_t cq_cumulative_num(struct cq_handle *); unsigned int cq_memory_allocation(void); #ifdef DEBUG void cq_debug(struct cq_handle *, int); diff --git a/src/stats.bif b/src/stats.bif index d7e812df93..3a975145b6 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -214,6 +214,7 @@ function get_timer_stats%(%): TimerStats r->Assign(n++, new Val(unsigned(timer_mgr->Size()), TYPE_COUNT)); r->Assign(n++, new Val(unsigned(timer_mgr->PeakSize()), TYPE_COUNT)); + r->Assign(n++, new Val(timer_mgr->CumulativeNum(), TYPE_COUNT)); return r; %} From cfdabb901fea7b904e5aaeedc2fc2617efb9de88 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 9 Jan 2016 01:14:13 -0500 Subject: [PATCH 10/53] Continued stats cleanup and extension. --- scripts/base/init-bare.bro | 22 ++++++------- scripts/policy/misc/stats.bro | 4 +-- src/DFA.cc | 5 +-- src/DFA.h | 7 ++-- src/Func.cc | 1 - src/RuleMatcher.cc | 9 ++---- src/RuleMatcher.h | 6 ++-- src/Stats.cc | 6 ++-- src/stats.bif | 60 +++++++++++++++++++++-------------- 9 files changed, 59 insertions(+), 61 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 3d870da38f..7b4f2c857f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -502,8 +502,8 @@ type ProcStats: record { }; type EventStats: record { - num_events_queued: count; ##< Total number of events queued so far. - num_events_dispatched: count; ##< Total number of events dispatched so far. + queued: count; ##< Total number of events queued so far. + dispatched: count; ##< Total number of events dispatched so far. }; ## Summary statistics of all regular expression matchers. @@ -520,13 +520,13 @@ type ReassemblerStats: record { ## ## .. bro:see:: get_matcher_stats type MatcherStats: record { - matchers: count; ##< Number of distinct RE matchers. - dfa_states: count; ##< Number of DFA states across all matchers. - computed: count; ##< Number of computed DFA state transitions. - mem: count; ##< Number of bytes used by DFA states. - hits: count; ##< Number of cache hits. - misses: count; ##< Number of cache misses. - avg_nfa_states: count; ##< Average number of NFA states across all matchers. + matchers: count; ##< Number of distinct RE matchers. + nfa_states: count; ##< Number of NFA states across all matchers. + dfa_states: count; ##< Number of DFA states across all matchers. + computed: count; ##< Number of computed DFA state transitions. + mem: count; ##< Number of bytes used by DFA states. + hits: count; ##< Number of cache hits. + misses: count; ##< Number of cache misses. }; type TimerStats: record { @@ -560,10 +560,6 @@ type GapStats: record { gap_bytes: count; ##< How many bytes were missing in the gaps. }; -type PatternStats: record { - -}; - type ThreadStats: record { num_threads: count; }; diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index a49d377bae..a35ee4a90e 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -117,8 +117,8 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr $reassem_frag_size=rs$frag_size, $reassem_unknown_size=rs$unknown_size, - $events_proc=es$num_events_dispatched - last_es$num_events_dispatched, - $events_queued=es$num_events_queued - last_es$num_events_queued, + $events_proc=es$dispatched - last_es$dispatched, + $events_queued=es$queued - last_es$queued, $timers=ts$cumulative - last_ts$cumulative, $active_timers=ts$current, diff --git a/src/DFA.cc b/src/DFA.cc index 9b8b3e5d31..5885a9bf3b 100644 --- a/src/DFA.cc +++ b/src/DFA.cc @@ -9,8 +9,6 @@ unsigned int DFA_State::transition_counter = 0; -uint64 total_dfa_states = 0; - DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, NFA_state_list* arg_nfa_states, AcceptingSet* arg_accept) @@ -22,8 +20,6 @@ DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, mark = 0; centry = 0; - ++total_dfa_states; - SymPartition(ec); xtions = new DFA_State*[num_sym]; @@ -350,6 +346,7 @@ DFA_State* DFA_State_Cache::Lookup(const NFA_state_list& nfas, ++misses; return 0; } + ++hits; delete *hash; *hash = 0; diff --git a/src/DFA.h b/src/DFA.h index c329b929d4..a63beca9ac 100644 --- a/src/DFA.h +++ b/src/DFA.h @@ -19,8 +19,6 @@ class DFA_Machine; class DFA_State; struct CacheEntry; -extern uint64 total_dfa_states; - class DFA_State : public BroObj { public: DFA_State(int state_num, const EquivClass* ec, @@ -91,10 +89,9 @@ public: int NumEntries() const { return states.Length(); } struct Stats { - unsigned int dfa_states; - - // Sum over all NFA states per DFA state. + // Sum of all NFA states unsigned int nfa_states; + unsigned int dfa_states; unsigned int computed; unsigned int uncomputed; unsigned int mem; diff --git a/src/Func.cc b/src/Func.cc index ac3cda6dd6..ccb2570f70 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -653,7 +653,6 @@ void init_builtin_funcs() TimerStats = internal_type("TimerStats")->AsRecordType(); FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType(); ThreadStats = internal_type("ThreadStats")->AsRecordType(); - PatternStats = internal_type("PatternStats")->AsRecordType(); var_sizes = internal_type("var_sizes")->AsTableType(); diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index f40a5c4349..af4787086d 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -1174,7 +1174,7 @@ void RuleMatcher::GetStats(Stats* stats, RuleHdrTest* hdr_test) stats->mem = 0; stats->hits = 0; stats->misses = 0; - stats->avg_nfa_states = 0; + stats->nfa_states = 0; hdr_test = root; } @@ -1195,15 +1195,10 @@ void RuleMatcher::GetStats(Stats* stats, RuleHdrTest* hdr_test) stats->mem += cstats.mem; stats->hits += cstats.hits; stats->misses += cstats.misses; - stats->avg_nfa_states += cstats.nfa_states; + stats->nfa_states += cstats.nfa_states; } } - if ( stats->dfa_states ) - stats->avg_nfa_states /= stats->dfa_states; - else - stats->avg_nfa_states = 0; - for ( RuleHdrTest* h = hdr_test->child; h; h = h->sibling ) GetStats(stats, h); } diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h index 6ffc971db1..b16a1556f9 100644 --- a/src/RuleMatcher.h +++ b/src/RuleMatcher.h @@ -297,6 +297,9 @@ public: struct Stats { unsigned int matchers; // # distinct RE matchers + // NFA states across all matchers. + unsigned int nfa_states; + // # DFA states across all matchers unsigned int dfa_states; unsigned int computed; // # computed DFA state transitions @@ -305,9 +308,6 @@ public: // # cache hits (sampled, multiply by MOVE_TO_FRONT_SAMPLE_SIZE) unsigned int hits; unsigned int misses; // # cache misses - - // Average # NFA states per DFA state. - unsigned int avg_nfa_states; }; Val* BuildRuleStateValue(const Rule* rule, diff --git a/src/Stats.cc b/src/Stats.cc index 99e36625b8..cf364d5747 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -173,9 +173,9 @@ void ProfileLogger::Log() RuleMatcher::Stats stats; rule_matcher->GetStats(&stats); - file->Write(fmt("%06f RuleMatcher: matchers=%d dfa_states=%d ncomputed=%d " - "mem=%dK avg_nfa_states=%d\n", network_time, stats.matchers, - stats.dfa_states, stats.computed, stats.mem / 1024, stats.avg_nfa_states)); + file->Write(fmt("%06f RuleMatcher: matchers=%d nfa_states=%d dfa_states=%d " + "ncomputed=%d mem=%dK\n", network_time, stats.matchers, + stats.nfa_states, stats.dfa_states, stats.computed, stats.mem / 1024)); } file->Write(fmt("%.06f Timers: current=%d max=%d mem=%dK lag=%.2fs\n", diff --git a/src/stats.bif b/src/stats.bif index 3a975145b6..ac8541182f 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -12,7 +12,6 @@ RecordType* ConnStats; RecordType* GapStats; RecordType* EventStats; RecordType* ThreadStats; -RecordType* PatternStats; RecordType* TimerStats; RecordType* FileAnalysisStats; %%} @@ -157,6 +156,13 @@ function get_event_stats%(%): EventStats return r; %} +## Returns statistics about reassembler usage. +## +## Returns: A record with reassembler statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_reassembler_stats%(%): ReassemblerStats %{ RecordVal* r = new RecordVal(ReassemblerStats); @@ -170,6 +176,13 @@ function get_reassembler_stats%(%): ReassemblerStats return r; %} +## Returns statistics about DNS lookup activity. +## +## Returns: A record with DNS lookup statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_dns_stats%(%): DNSStats %{ RecordVal* r = new RecordVal(DNSStats); @@ -188,25 +201,13 @@ function get_dns_stats%(%): DNSStats return r; %} -function get_pattern_stats%(%): PatternStats - %{ - RecordVal* r = new RecordVal(PatternStats); - int n = 0; - - //DFA_State_Cache::Stats stats; - //dfa_state_cache->GetStats(&stats); - - //fprintf(f, "Computed dfa_states = %d; Classes = %d; Computed trans. = %d; Uncomputed trans. = %d\n", - // stats.dfa_states, EC()->NumClasses(), - // stats.computed, stats.uncomputed); -// - //fprintf(f, "DFA cache hits = %d; misses = %d\n", - // stats.hits, stats.misses); - - return r; - - %} - +## Returns statistics about timer usage. +## +## Returns: A record with timer usage statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_timer_stats%(%): TimerStats %{ RecordVal* r = new RecordVal(TimerStats); @@ -219,6 +220,13 @@ function get_timer_stats%(%): TimerStats return r; %} +## Returns statistics about file analysis. +## +## Returns: A record with file analysis statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_file_analysis_stats%(%): FileAnalysisStats %{ RecordVal* r = new RecordVal(FileAnalysisStats); @@ -231,6 +239,13 @@ function get_file_analysis_stats%(%): FileAnalysisStats return r; %} +## Returns statistics about thread usage. +## +## Returns: A record with thread usage statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_thread_stats%(%): ThreadStats %{ RecordVal* r = new RecordVal(ThreadStats); @@ -245,8 +260,7 @@ function get_thread_stats%(%): ThreadStats ## ## Returns: A record with TCP gap statistics. ## -## .. bro:see:: do_profiling -## get_net_stats +## .. bro:see:: get_net_stats ## get_proc_stats ## get_matcher_stats function get_gap_stats%(%): GapStats @@ -283,12 +297,12 @@ function get_matcher_stats%(%): MatcherStats rule_matcher->GetStats(&s); r->Assign(n++, new Val(s.matchers, TYPE_COUNT)); + r->Assign(n++, new Val(s.nfa_states, TYPE_COUNT)); r->Assign(n++, new Val(s.dfa_states, TYPE_COUNT)); r->Assign(n++, new Val(s.computed, TYPE_COUNT)); r->Assign(n++, new Val(s.mem, TYPE_COUNT)); r->Assign(n++, new Val(s.hits, TYPE_COUNT)); r->Assign(n++, new Val(s.misses, TYPE_COUNT)); - r->Assign(n++, new Val(s.avg_nfa_states, TYPE_COUNT)); return r; %} From 18a1e6f76b33732c84f54e3e4a07dc99bcce05ee Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 11 Jan 2016 09:25:36 -0500 Subject: [PATCH 11/53] Small stats script tweaks and beginning broker stats. --- scripts/policy/misc/stats.bro | 41 ++++++++++++++++------------------- src/stats.bif | 34 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index a35ee4a90e..b43326e89d 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -19,6 +19,20 @@ export { mem: count &log; ## Number of packets processed since the last stats interval. pkts_proc: count &log; + ## Number of bytes received since the last stats interval if + ## reading live traffic. + bytes_recv: count &log; + + ## Number of packets dropped since the last stats interval if + ## reading live traffic. + pkts_dropped: count &log &optional; + ## Number of packets seen on the link since the last stats + ## interval if reading live traffic. + pkts_link: count &log &optional; + ## Lag between the wall clock and packet timestamps if reading + ## live traffic. + pkt_lag: interval &log &optional; + ## Number of events processed since the last stats interval. events_proc: count &log; ## Number of events that have been queued since the last stats @@ -57,22 +71,6 @@ export { reassem_frag_size: count &log; ## Current size of unkown data in reassembly (this is only PIA buffer right now). reassem_unknown_size: count &log; - - ## Lag between the wall clock and packet timestamps if reading - ## live traffic. - lag: interval &log &optional; - ## Number of packets received since the last stats interval if - ## reading live traffic. - pkts_recv: count &log &optional; - ## Number of packets dropped since the last stats interval if - ## reading live traffic. - pkts_dropped: count &log &optional; - ## Number of packets seen on the link since the last stats - ## interval if reading live traffic. - pkts_link: count &log &optional; - ## Number of bytes received since the last stats interval if - ## reading live traffic. - bytes_recv: count &log &optional; }; ## Event to catch stats as they are written to the logging stream. @@ -86,7 +84,7 @@ event bro_init() &priority=5 event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats) { - local now = network_time(); + local nettime = network_time(); local ns = get_net_stats(); local cs = get_conn_stats(); local ps = get_proc_stats(); @@ -100,10 +98,11 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr # shutting down. return; - local info: Info = [$ts=now, + local info: Info = [$ts=nettime, $peer=peer_description, $mem=ps$mem/1048576, $pkts_proc=ns$pkts_recvd - last_ns$pkts_recvd, + $bytes_recv = ns$bytes_recvd - last_ns$bytes_recvd, $active_tcp_conns=cs$num_tcp_conns, $tcp_conns=cs$cumulative_tcp_conns - last_cs$cumulative_tcp_conns, @@ -132,15 +131,13 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr if ( reading_live_traffic() ) { - info$lag = now - network_time(); - info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd; + info$pkt_lag = current_time() - nettime; info$pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; info$pkts_link = ns$pkts_link - last_ns$pkts_link; - info$bytes_recv = ns$bytes_recvd - last_ns$bytes_recvd; } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs, ts, fs) }; + schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs) }; } event bro_init() diff --git a/src/stats.bif b/src/stats.bif index ac8541182f..2c5fd6151a 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -145,6 +145,13 @@ function get_proc_stats%(%): ProcStats return r; %} +## Returns statistics about the event engine. +## +## Returns: A record with event engine statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_event_stats%(%): EventStats %{ RecordVal* r = new RecordVal(EventStats); @@ -306,3 +313,30 @@ function get_matcher_stats%(%): MatcherStats return r; %} + +function get_broker_stats%(%): BrokerStats + %{ + RecordVal* r = new RecordVal(CommunicationStats); + int n = 0; + +#ifdef ENABLE_BROKER + auto cs = broker_mgr->ConsumeStatistics(); + + r->Assign(n++, new Val(cs.outgoing_peer_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.data_store_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.pending_query_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.response_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.outgoing_conn_status_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.incoming_conn_status_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.report_count, TYPE_COUNT)); + + //for ( const auto& s : cs.print_count ) + // file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second)); + //for ( const auto& s : cs.event_count ) + // file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second)); + //for ( const auto& s : cs.log_count ) + // file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second)); +#endif + + return r; + %} \ No newline at end of file From c1d7337a73ae600d75e632be000ec70defd9eef7 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 12 Jan 2016 15:35:29 -0600 Subject: [PATCH 12/53] Improve documentation of Bro script statements Added more documentation of the "delete" statement. Removed some other text that was probably more confusing than helpful. --- doc/script-reference/statements.rst | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/script-reference/statements.rst b/doc/script-reference/statements.rst index e2f93a5627..47e82eb074 100644 --- a/doc/script-reference/statements.rst +++ b/doc/script-reference/statements.rst @@ -277,16 +277,25 @@ Here are the statements that the Bro scripting language supports. .. bro:keyword:: delete The "delete" statement is used to remove an element from a - :bro:type:`set` or :bro:type:`table`. Nothing happens if the - specified element does not exist in the set or table. + :bro:type:`set` or :bro:type:`table`, or to remove a value from + a :bro:type:`record` field that has the :bro:attr:`&optional` attribute. + When attempting to remove an element from a set or table, + nothing happens if the specified index does not exist. + When attempting to remove a value from an "&optional" record field, + nothing happens if that field doesn't have a value. Example:: local myset = set("this", "test"); local mytable = table(["key1"] = 80/tcp, ["key2"] = 53/udp); + local myrec = MyRecordType($a = 1, $b = 2); + delete myset["test"]; delete mytable["key1"]; + # In this example, "b" must have the "&optional" attribute + delete myrec$b; + .. bro:keyword:: event The "event" statement immediately queues invocation of an event handler. @@ -532,8 +541,6 @@ Here are the statements that the Bro scripting language supports. end with either a :bro:keyword:`break`, :bro:keyword:`fallthrough`, or :bro:keyword:`return` statement (although "return" is allowed only if the "switch" statement is inside a function, hook, or event handler). - If a "case" (or "default") block contain more than one statement, then - there is no need to wrap them in braces. Note that the braces in a "switch" statement are always required (these do not indicate the presence of a `compound statement`_), and that no @@ -604,12 +611,9 @@ Here are the statements that the Bro scripting language supports. if ( skip_ahead() ) next; - [...] - if ( finish_up ) break; - [...] } .. _compound statement: From 3550a2b2d360c0176afe505e4375c903862fdd42 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 12 Jan 2016 15:45:06 -0600 Subject: [PATCH 13/53] Update documentation for DNS "Z" field According to RFC 2535, RFC 3655, and RFC 4035, the Z field has been partitioned into three 1-bit fields. Therefore, we cannot claim in the documentation that it always has the value zero. --- scripts/base/protocols/dns/main.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index 58a63293d0..05a44a0ba9 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -52,7 +52,7 @@ export { ## The Recursion Available bit in a response message indicates ## that the name server supports recursive queries. RA: bool &log &default=F; - ## A reserved field that is currently supposed to be zero in all + ## A reserved field that is usually zero in ## queries and responses. Z: count &log &default=0; ## The set of resource descriptions in the query answer. From 16adf2ff5aeeaff4140abf5c960c15c2ccc7e1b0 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 14:05:23 -0500 Subject: [PATCH 14/53] Add DNS stats to the stats.log --- scripts/policy/misc/stats.bro | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index b43326e89d..be84c5f35f 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -63,6 +63,11 @@ export { ## Current number of files actively being seen. active_files: count &log; + ## Number of DNS requests seen since last stats interval. + dns_requests: count &log; + ## Current number of DNS requests awaiting a reply. + active_dns_requests: count &log; + ## Current size of TCP data in reassembly. reassem_tcp_size: count &log; ## Current size of File data in reassembly. @@ -82,7 +87,7 @@ event bro_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats"]); } -event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats) +event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats, last_ds: DNSStats) { local nettime = network_time(); local ns = get_net_stats(); @@ -92,6 +97,7 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr local rs = get_reassembler_stats(); local ts = get_timer_stats(); local fs = get_file_analysis_stats(); + local ds = get_dns_stats(); if ( bro_is_terminating() ) # No more stats will be written or scheduled when Bro is @@ -123,7 +129,10 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr $active_timers=ts$current, $files=fs$cumulative - last_fs$cumulative, - $active_files=fs$current + $active_files=fs$current, + + $dns_requests=ds$requests - last_ds$requests, + $active_dns_requests=ds$pending ]; # Someone's going to have to explain what this is and add a field to the Info record. @@ -137,10 +146,10 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs) }; + schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs, ds) }; } event bro_init() { - schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats()) }; + schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats(), get_dns_stats()) }; } From ee763381b25b0456a01ca40f826fb9c9b9ca9ef8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 16:17:41 -0500 Subject: [PATCH 15/53] Fixing default stats collection interval to every 5 minutes. --- scripts/policy/misc/stats.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index be84c5f35f..d154da05e9 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -8,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 1sec &redef; + const stats_report_interval = 5min &redef; type Info: record { ## Timestamp for the measurement. From 6064134119bb119095dff60c6644114571850104 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 16:49:18 -0500 Subject: [PATCH 16/53] Removing Broker stats, it was broken and incomplete. --- src/stats.bif | 58 +++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/src/stats.bif b/src/stats.bif index 2c5fd6151a..f5c8ee4308 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -22,8 +22,7 @@ RecordType* FileAnalysisStats; ## ## Returns: A record of packet statistics. ## -## .. bro:see:: do_profiling -## get_proc_stats +## .. bro:see:: get_proc_stats ## get_matcher_stats ## get_gap_stats function get_net_stats%(%): NetStats @@ -100,8 +99,7 @@ function get_conn_stats%(%): ConnStats ## ## Returns: A record with resource usage statistics. ## -## .. bro:see:: do_profiling -## get_net_stats +## .. bro:see:: get_net_stats ## get_matcher_stats ## get_gap_stats function get_proc_stats%(%): ProcStats @@ -314,29 +312,29 @@ function get_matcher_stats%(%): MatcherStats return r; %} -function get_broker_stats%(%): BrokerStats - %{ - RecordVal* r = new RecordVal(CommunicationStats); - int n = 0; - -#ifdef ENABLE_BROKER - auto cs = broker_mgr->ConsumeStatistics(); - - r->Assign(n++, new Val(cs.outgoing_peer_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.data_store_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.pending_query_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.response_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.outgoing_conn_status_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.incoming_conn_status_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.report_count, TYPE_COUNT)); - - //for ( const auto& s : cs.print_count ) - // file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second)); - //for ( const auto& s : cs.event_count ) - // file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second)); - //for ( const auto& s : cs.log_count ) - // file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second)); -#endif - - return r; - %} \ No newline at end of file +# function get_broker_stats%(%): BrokerStats +# %{ +# RecordVal* r = new RecordVal(CommunicationStats); +# int n = 0; +# +# #ifdef ENABLE_BROKER +# auto cs = broker_mgr->ConsumeStatistics(); +# +# r->Assign(n++, new Val(cs.outgoing_peer_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.data_store_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.pending_query_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.response_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.outgoing_conn_status_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.incoming_conn_status_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.report_count, TYPE_COUNT)); +# +# //for ( const auto& s : cs.print_count ) +# // file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second)); +# //for ( const auto& s : cs.event_count ) +# // file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second)); +# //for ( const auto& s : cs.log_count ) +# // file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second)); +# #endif +# +# return r; +# %} \ No newline at end of file From 53db5d1711e2652596e8660d40789296013f9a0e Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 17:09:55 -0500 Subject: [PATCH 17/53] Removing some references to resource_usage() --- scripts/policy/frameworks/control/controllee.bro | 12 ++++++------ src/bro.bif | 2 -- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/policy/frameworks/control/controllee.bro b/scripts/policy/frameworks/control/controllee.bro index b4769764f4..6e3b5499b6 100644 --- a/scripts/policy/frameworks/control/controllee.bro +++ b/scripts/policy/frameworks/control/controllee.bro @@ -29,12 +29,12 @@ event Control::peer_status_request() if ( ! peer$connected ) next; - local res = resource_usage(); - status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", - network_time(), - peer$peer$descr, peer$host, - res$num_events_queued, res$num_events_dispatched, - res$blocking_input, res$blocking_output); + #local res = resource_usage(); + #status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", + # network_time(), + # peer$peer$descr, peer$host, + # res$num_events_queued, res$num_events_dispatched, + # res$blocking_input, res$blocking_output); } event Control::peer_status_response(status); diff --git a/src/bro.bif b/src/bro.bif index ce16695afa..5385a0e22f 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1792,7 +1792,6 @@ function record_fields%(rec: any%): record_field_table ## holds the name of the file. ## ## .. bro:see:: net_stats -## resource_usage ## get_matcher_stats ## get_gap_stats function do_profiling%(%) : any @@ -1857,7 +1856,6 @@ function is_local_interface%(ip: addr%) : bool ## Returns: True (unconditionally). ## ## .. bro:see:: do_profiling -## resource_usage ## get_matcher_stats ## get_net_stats ## get_gap_stats From 41a181d98d7afe06ae47255986fea26bde55cafe Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 21:22:09 -0500 Subject: [PATCH 18/53] Removing more broken functionality due to changed stats apis. --- .../policy/frameworks/control/controllee.bro | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/policy/frameworks/control/controllee.bro b/scripts/policy/frameworks/control/controllee.bro index 6e3b5499b6..1a62d294b7 100644 --- a/scripts/policy/frameworks/control/controllee.bro +++ b/scripts/policy/frameworks/control/controllee.bro @@ -22,30 +22,30 @@ event Control::id_value_request(id: string) event Control::peer_status_request() { - local status = ""; - for ( p in Communication::nodes ) - { - local peer = Communication::nodes[p]; - if ( ! peer$connected ) - next; - - #local res = resource_usage(); - #status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", - # network_time(), - # peer$peer$descr, peer$host, - # res$num_events_queued, res$num_events_dispatched, - # res$blocking_input, res$blocking_output); - } - - event Control::peer_status_response(status); + #local status = ""; + #for ( p in Communication::nodes ) + # { + # local peer = Communication::nodes[p]; + # if ( ! peer$connected ) + # next; + # + # #local res = resource_usage(); + # #status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", + # # network_time(), + # # peer$peer$descr, peer$host, + # # res$num_events_queued, res$num_events_dispatched, + # # res$blocking_input, res$blocking_output); + # } + # + #event Control::peer_status_response(status); } event Control::net_stats_request() { - local ns = net_stats(); - local reply = fmt("%.6f recvd=%d dropped=%d link=%d\n", network_time(), - ns$pkts_recvd, ns$pkts_dropped, ns$pkts_link); - event Control::net_stats_response(reply); + #local ns = net_stats(); + #local reply = fmt("%.6f recvd=%d dropped=%d link=%d\n", network_time(), + # ns$pkts_recvd, ns$pkts_dropped, ns$pkts_link); + #event Control::net_stats_response(reply); } event Control::configuration_update_request() From 6ef8a93dcaf5dc587207a1155f4eb2b4f22f1950 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 27 Jan 2016 14:56:42 -0600 Subject: [PATCH 19/53] Update traffic per core estimate in the cluster doc --- doc/cluster/index.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/cluster/index.rst b/doc/cluster/index.rst index 544ca5e0f8..6e426c005e 100644 --- a/doc/cluster/index.rst +++ b/doc/cluster/index.rst @@ -96,13 +96,13 @@ logging is done remotely to the manager, and normally very little is written to disk. The rule of thumb we have followed recently is to allocate approximately 1 -core for every 80Mbps of traffic that is being analyzed. However, this +core for every 250Mbps of traffic that is being analyzed. However, this estimate could be extremely traffic mix-specific. It has generally worked for mixed traffic with many users and servers. For example, if your traffic peaks around 2Gbps (combined) and you want to handle traffic at peak load, -you may want to have 26 cores available (2048 / 80 == 25.6). If the 80Mbps -estimate works for your traffic, this could be handled by 3 physical hosts -dedicated to being workers with each one containing dual 6-core processors. +you may want to have 8 cores available (2048 / 250 == 8.2). If the 250Mbps +estimate works for your traffic, this could be handled by 2 physical hosts +dedicated to being workers with each one containing a quad-core processor. Once a flow-based load balancer is put into place this model is extremely easy to scale. It is recommended that you estimate the amount of From 7ede9c65d2e40c24e6be1acfa815cbef991caa6e Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 1 Mar 2016 17:31:41 -0600 Subject: [PATCH 20/53] Add more documentation to sumstats framework scripts --- scripts/base/frameworks/sumstats/main.bro | 38 ++++++++++++------- .../frameworks/sumstats/plugins/average.bro | 4 +- .../sumstats/plugins/hll_unique.bro | 2 + .../base/frameworks/sumstats/plugins/last.bro | 2 + .../base/frameworks/sumstats/plugins/max.bro | 4 +- .../base/frameworks/sumstats/plugins/min.bro | 4 +- .../frameworks/sumstats/plugins/sample.bro | 4 +- .../frameworks/sumstats/plugins/std-dev.bro | 4 +- .../base/frameworks/sumstats/plugins/sum.bro | 6 ++- .../base/frameworks/sumstats/plugins/topk.bro | 5 +++ .../frameworks/sumstats/plugins/unique.bro | 6 ++- .../frameworks/sumstats/plugins/variance.bro | 6 ++- 12 files changed, 61 insertions(+), 24 deletions(-) diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro index 8dbdb61edd..edd80ede0f 100644 --- a/scripts/base/frameworks/sumstats/main.bro +++ b/scripts/base/frameworks/sumstats/main.bro @@ -5,7 +5,8 @@ module SumStats; export { - ## The various calculations are all defined as plugins. + ## Type to represent the calculations that are available. The calculations + ## are all defined as plugins. type Calculation: enum { PLACEHOLDER }; @@ -39,6 +40,7 @@ export { str: string &optional; }; + ## Represents a reducer. type Reducer: record { ## Observation stream identifier for the reducer ## to attach to. @@ -56,7 +58,7 @@ export { normalize_key: function(key: SumStats::Key): Key &optional; }; - ## Value calculated for an observation stream fed into a reducer. + ## Result calculated for an observation stream fed into a reducer. ## Most of the fields are added by plugins. type ResultVal: record { ## The time when the first observation was added to @@ -71,14 +73,15 @@ export { num: count &default=0; }; - ## Type to store results for multiple reducers. + ## Type to store a table of results for multiple reducers indexed by + ## observation stream identifier. type Result: table[string] of ResultVal; ## Type to store a table of sumstats results indexed by keys. type ResultTable: table[Key] of Result; - ## SumStats represent an aggregation of reducers along with - ## mechanisms to handle various situations like the epoch ending + ## Represents a SumStat, which consists of an aggregation of reducers along + ## with mechanisms to handle various situations like the epoch ending ## or thresholds being crossed. ## ## It's best to not access any global state outside @@ -101,21 +104,28 @@ export { ## The reducers for the SumStat. reducers: set[Reducer]; - ## Provide a function to calculate a value from the - ## :bro:see:`SumStats::Result` structure which will be used - ## for thresholding. - ## This is required if a *threshold* value is given. + ## A function that will be called once for each observation in order + ## to calculate a value from the :bro:see:`SumStats::Result` structure + ## which will be used for thresholding. + ## This function is required if a *threshold* value or + ## a *threshold_series* is given. threshold_val: function(key: SumStats::Key, result: SumStats::Result): double &optional; - ## The threshold value for calling the - ## *threshold_crossed* callback. + ## The threshold value for calling the *threshold_crossed* callback. + ## If you need more than one threshold value, then use + ## *threshold_series* instead. threshold: double &optional; - ## A series of thresholds for calling the - ## *threshold_crossed* callback. + ## A series of thresholds for calling the *threshold_crossed* + ## callback. These thresholds must be listed in ascending order, + ## because a threshold is not checked until the preceding one has + ## been crossed. threshold_series: vector of double &optional; ## A callback that is called when a threshold is crossed. + ## A threshold is crossed when the value returned from *threshold_val* + ## is greater than or equal to the threshold value, but only the first + ## time this happens within an epoch. threshold_crossed: function(key: SumStats::Key, result: SumStats::Result) &optional; ## A callback that receives each of the results at the @@ -130,6 +140,8 @@ export { }; ## Create a summary statistic. + ## + ## ss: The SumStat to create. global create: function(ss: SumStats::SumStat); ## Add data into an observation stream. This should be diff --git a/scripts/base/frameworks/sumstats/plugins/average.bro b/scripts/base/frameworks/sumstats/plugins/average.bro index 8f7f7b568f..160ca64d78 100644 --- a/scripts/base/frameworks/sumstats/plugins/average.bro +++ b/scripts/base/frameworks/sumstats/plugins/average.bro @@ -1,3 +1,5 @@ +##! Calculate the average. + @load ../main module SumStats; @@ -9,7 +11,7 @@ export { }; redef record ResultVal += { - ## For numeric data, this calculates the average of all values. + ## For numeric data, this is the average of all values. average: double &optional; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/hll_unique.bro b/scripts/base/frameworks/sumstats/plugins/hll_unique.bro index 494cbf4667..43cafcff7f 100644 --- a/scripts/base/frameworks/sumstats/plugins/hll_unique.bro +++ b/scripts/base/frameworks/sumstats/plugins/hll_unique.bro @@ -1,3 +1,5 @@ +##! Calculate the number of unique values (using the HyperLogLog algorithm). + @load base/frameworks/sumstats module SumStats; diff --git a/scripts/base/frameworks/sumstats/plugins/last.bro b/scripts/base/frameworks/sumstats/plugins/last.bro index 430c2e375b..ca04114f61 100644 --- a/scripts/base/frameworks/sumstats/plugins/last.bro +++ b/scripts/base/frameworks/sumstats/plugins/last.bro @@ -1,3 +1,5 @@ +##! Keep the last X observations. + @load base/frameworks/sumstats @load base/utils/queue diff --git a/scripts/base/frameworks/sumstats/plugins/max.bro b/scripts/base/frameworks/sumstats/plugins/max.bro index d43ad9dc38..adcc6ae113 100644 --- a/scripts/base/frameworks/sumstats/plugins/max.bro +++ b/scripts/base/frameworks/sumstats/plugins/max.bro @@ -1,3 +1,5 @@ +##! Find the maximum value. + @load ../main module SumStats; @@ -9,7 +11,7 @@ export { }; redef record ResultVal += { - ## For numeric data, this tracks the maximum value given. + ## For numeric data, this tracks the maximum value. max: double &optional; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/min.bro b/scripts/base/frameworks/sumstats/plugins/min.bro index 014755cf32..22cab1009c 100644 --- a/scripts/base/frameworks/sumstats/plugins/min.bro +++ b/scripts/base/frameworks/sumstats/plugins/min.bro @@ -1,3 +1,5 @@ +##! Find the minimum value. + @load ../main module SumStats; @@ -9,7 +11,7 @@ export { }; redef record ResultVal += { - ## For numeric data, this tracks the minimum value given. + ## For numeric data, this tracks the minimum value. min: double &optional; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro index 809d696896..0200e85949 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.bro +++ b/scripts/base/frameworks/sumstats/plugins/sample.bro @@ -1,3 +1,5 @@ +##! Keep a random sample of values. + @load base/frameworks/sumstats/main module SumStats; @@ -10,7 +12,7 @@ export { }; redef record Reducer += { - ## A number of sample Observations to collect. + ## The number of sample Observations to collect. num_samples: count &default=0; }; diff --git a/scripts/base/frameworks/sumstats/plugins/std-dev.bro b/scripts/base/frameworks/sumstats/plugins/std-dev.bro index 2e5b95b212..bfb02c82cc 100644 --- a/scripts/base/frameworks/sumstats/plugins/std-dev.bro +++ b/scripts/base/frameworks/sumstats/plugins/std-dev.bro @@ -1,3 +1,5 @@ +##! Calculate the standard deviation. + @load ./variance @load ../main @@ -5,7 +7,7 @@ module SumStats; export { redef enum Calculation += { - ## Find the standard deviation of the values. + ## Calculate the standard deviation of the values. STD_DEV }; diff --git a/scripts/base/frameworks/sumstats/plugins/sum.bro b/scripts/base/frameworks/sumstats/plugins/sum.bro index 074b4b72f3..fb1d96bcd4 100644 --- a/scripts/base/frameworks/sumstats/plugins/sum.bro +++ b/scripts/base/frameworks/sumstats/plugins/sum.bro @@ -1,11 +1,13 @@ +##! Calculate the sum. + @load ../main module SumStats; export { redef enum Calculation += { - ## Sums the values given. For string values, - ## this will be the number of strings given. + ## Calculate the sum of the values. For string values, + ## this will be the number of strings. SUM }; diff --git a/scripts/base/frameworks/sumstats/plugins/topk.bro b/scripts/base/frameworks/sumstats/plugins/topk.bro index 0ef0f01393..e7107cb4fb 100644 --- a/scripts/base/frameworks/sumstats/plugins/topk.bro +++ b/scripts/base/frameworks/sumstats/plugins/topk.bro @@ -1,3 +1,5 @@ +##! Keep the top-k (i.e., most frequently occurring) observations. + @load base/frameworks/sumstats module SumStats; @@ -9,10 +11,13 @@ export { }; redef enum Calculation += { + ## Keep a top-k list of values. TOPK }; redef record ResultVal += { + ## A handle which can be passed to some built-in functions to get + ## the top-k results. topk: opaque of topk &optional; }; diff --git a/scripts/base/frameworks/sumstats/plugins/unique.bro b/scripts/base/frameworks/sumstats/plugins/unique.bro index abfbe3669d..5fcaa1dc3c 100644 --- a/scripts/base/frameworks/sumstats/plugins/unique.bro +++ b/scripts/base/frameworks/sumstats/plugins/unique.bro @@ -1,10 +1,12 @@ +##! Calculate the number of unique values. + @load ../main module SumStats; export { redef record Reducer += { - ## Maximum number of unique elements to store. + ## Maximum number of unique values to store. unique_max: count &optional; }; @@ -15,7 +17,7 @@ export { redef record ResultVal += { ## If cardinality is being tracked, the number of unique - ## items is tracked here. + ## values is tracked here. unique: count &default=0; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/variance.bro b/scripts/base/frameworks/sumstats/plugins/variance.bro index 12d30cc4fe..989bf07eaf 100644 --- a/scripts/base/frameworks/sumstats/plugins/variance.bro +++ b/scripts/base/frameworks/sumstats/plugins/variance.bro @@ -1,3 +1,5 @@ +##! Calculate the variance. + @load ./average @load ../main @@ -5,12 +7,12 @@ module SumStats; export { redef enum Calculation += { - ## Find the variance of the values. + ## Calculate the variance of the values. VARIANCE }; redef record ResultVal += { - ## For numeric data, this calculates the variance. + ## For numeric data, this is the variance. variance: double &optional; }; } From f54a5b52e5d62c2394a1aa16e1176b829e54f152 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 12 Apr 2016 15:40:18 -0500 Subject: [PATCH 21/53] Improve documentation of the "for" statement --- doc/script-reference/statements.rst | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/doc/script-reference/statements.rst b/doc/script-reference/statements.rst index 47e82eb074..14e0cc3c32 100644 --- a/doc/script-reference/statements.rst +++ b/doc/script-reference/statements.rst @@ -315,30 +315,33 @@ Here are the statements that the Bro scripting language supports. .. bro:keyword:: for A "for" loop iterates over each element in a string, set, vector, or - table and executes a statement for each iteration. Currently, - modifying a container's membership while iterating over it may - result in undefined behavior, so avoid adding or removing elements - inside the loop. + table and executes a statement for each iteration (note that the order + in which the loop iterates over the elements in a set or a table is + nondeterministic). However, no loop iterations occur if the string, + set, vector, or table is empty. For each iteration of the loop, a loop variable will be assigned to an element if the expression evaluates to a string or set, or an index if the expression evaluates to a vector or table. Then the statement - is executed. However, the statement will not be executed if the expression - evaluates to an object with no elements. + is executed. If the expression is a table or a set with more than one index, then the loop variable must be specified as a comma-separated list of different loop variables (one for each index), enclosed in brackets. - A :bro:keyword:`break` statement can be used at any time to immediately - terminate the "for" loop, and a :bro:keyword:`next` statement can be - used to skip to the next loop iteration. - Note that the loop variable in a "for" statement is not allowed to be a global variable, and it does not need to be declared prior to the "for" statement. The type will be inferred from the elements of the expression. + Currently, modifying a container's membership while iterating over it may + result in undefined behavior, so do not add or remove elements + inside the loop. + + A :bro:keyword:`break` statement will immediately terminate the "for" + loop, and a :bro:keyword:`next` statement will skip to the next loop + iteration. + Example:: local myset = set(80/tcp, 81/tcp); From 16c0707b1d804ccfcc671fb9642a0c21ffd7219f Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 13 Apr 2016 14:16:31 -0500 Subject: [PATCH 22/53] Fix RFB analyzer to build on FreeBSD The auto-generated header rfb_pac.h had class member functions "major" and "minor" which were clashing with macros of the same name defined in /usr/include/sys/types.h on FreeBSD. Fixed by renaming the fields. --- src/analyzer/protocol/rfb/rfb-analyzer.pac | 2 +- src/analyzer/protocol/rfb/rfb-protocol.pac | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/analyzer/protocol/rfb/rfb-analyzer.pac b/src/analyzer/protocol/rfb/rfb-analyzer.pac index b63b9f4085..47b87cf5ef 100644 --- a/src/analyzer/protocol/rfb/rfb-analyzer.pac +++ b/src/analyzer/protocol/rfb/rfb-analyzer.pac @@ -81,7 +81,7 @@ refine connection RFB_Conn += { if ( client ) { // Set protocol version on client's version - int minor_version = bytestring_to_int(${msg.minor},10); + int minor_version = bytestring_to_int(${msg.versionminor},10); version = minor_version; // Apple specifies minor version "889" but talks v37 diff --git a/src/analyzer/protocol/rfb/rfb-protocol.pac b/src/analyzer/protocol/rfb/rfb-protocol.pac index 764046e747..8f795c1751 100644 --- a/src/analyzer/protocol/rfb/rfb-protocol.pac +++ b/src/analyzer/protocol/rfb/rfb-protocol.pac @@ -17,13 +17,13 @@ enum states { type RFBProtocolVersion (client: bool) = record { header: "RFB "; - major: bytestring &length=3; + versionmajor: bytestring &length=3; dot: "."; - minor: bytestring &length=3; + versionminor: bytestring &length=3; pad: uint8; } &let { proc: bool = $context.connection.handle_banners(client, this); - proc2: bool = $context.flow.proc_rfb_version(client, major, minor); + proc2: bool = $context.flow.proc_rfb_version(client, versionmajor, versionminor); } type RFBSecurityTypes = record { From 23d25628ad9473f2a0faecafb1d6eb157a141673 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 13 Apr 2016 16:55:28 -0400 Subject: [PATCH 23/53] Revert "Fix RFB analyzer to build on FreeBSD" This reverts commit 16c0707b1d804ccfcc671fb9642a0c21ffd7219f. --- src/analyzer/protocol/rfb/rfb-analyzer.pac | 2 +- src/analyzer/protocol/rfb/rfb-protocol.pac | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/analyzer/protocol/rfb/rfb-analyzer.pac b/src/analyzer/protocol/rfb/rfb-analyzer.pac index 47b87cf5ef..b63b9f4085 100644 --- a/src/analyzer/protocol/rfb/rfb-analyzer.pac +++ b/src/analyzer/protocol/rfb/rfb-analyzer.pac @@ -81,7 +81,7 @@ refine connection RFB_Conn += { if ( client ) { // Set protocol version on client's version - int minor_version = bytestring_to_int(${msg.versionminor},10); + int minor_version = bytestring_to_int(${msg.minor},10); version = minor_version; // Apple specifies minor version "889" but talks v37 diff --git a/src/analyzer/protocol/rfb/rfb-protocol.pac b/src/analyzer/protocol/rfb/rfb-protocol.pac index 8f795c1751..764046e747 100644 --- a/src/analyzer/protocol/rfb/rfb-protocol.pac +++ b/src/analyzer/protocol/rfb/rfb-protocol.pac @@ -17,13 +17,13 @@ enum states { type RFBProtocolVersion (client: bool) = record { header: "RFB "; - versionmajor: bytestring &length=3; + major: bytestring &length=3; dot: "."; - versionminor: bytestring &length=3; + minor: bytestring &length=3; pad: uint8; } &let { proc: bool = $context.connection.handle_banners(client, this); - proc2: bool = $context.flow.proc_rfb_version(client, versionmajor, versionminor); + proc2: bool = $context.flow.proc_rfb_version(client, major, minor); } type RFBSecurityTypes = record { From 362bf7aee12814781ef97242accb176423cd2a64 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 27 Apr 2016 15:34:47 -0500 Subject: [PATCH 24/53] Update docs and tests of the fmt() function Removed tests and documentation of the "%A" format specifier, which was removed in commit 7344052b. --- src/bro.bif | 2 -- testing/btest/Baseline/bifs.fmt/out | 5 ----- testing/btest/bifs/fmt.bro | 22 ++++++---------------- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 5d097734a4..f21f927f92 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1480,8 +1480,6 @@ function cat_sep%(sep: string, def: string, ...%): string ## ## - ``.``: Precision of floating point specifiers ``[efg]`` (< 128) ## -## - ``A``: Escape only NUL bytes (each one replaced with ``\0``) in a string -## ## - ``[DTdxsefg]``: Format specifier ## ## - ``[DT]``: ISO timestamp with microsecond precision diff --git a/testing/btest/Baseline/bifs.fmt/out b/testing/btest/Baseline/bifs.fmt/out index 5f380c1b22..2572f924fb 100644 --- a/testing/btest/Baseline/bifs.fmt/out +++ b/testing/btest/Baseline/bifs.fmt/out @@ -45,11 +45,6 @@ test 310 310 2 -1 2 2 -1 -2 -2 -1 2 diff --git a/testing/btest/bifs/fmt.bro b/testing/btest/bifs/fmt.bro index 93607c2740..7fc4dc38d7 100644 --- a/testing/btest/bifs/fmt.bro +++ b/testing/btest/bifs/fmt.bro @@ -65,26 +65,16 @@ event bro_init() print fmt("%.3g", 3.1e+2); print fmt("%.7g", 3.1e+2); - # Tests comparing "%As" and "%s" (the string length is printed instead - # of the string itself because the print command does its own escaping) - local s0 = "\x00\x07"; - local s1 = fmt("%As", s0); # expands \x00 to "\0" - local s2 = fmt("%s", s0); # expands \x00 to "\0", and \x07 to "^G" + # Tests of "%s" with non-printable characters (the string length is printed + # instead of the string itself because the print command does its own + # escaping) + local s0 = "\x00\x1f"; + local s1 = fmt("%s", s0); print |s0|; print |s1|; - print |s2|; - - s0 = "\x07\x1f"; - s1 = fmt("%As", s0); - s2 = fmt("%s", s0); # expands \x07 to "^G", and \x1f to "\x1f" - print |s0|; - print |s1|; - print |s2|; s0 = "\x7f\xff"; - s1 = fmt("%As", s0); - s2 = fmt("%s", s0); # expands \x7f to "^?", and \xff to "\xff" + s1 = fmt("%s", s0); print |s0|; print |s1|; - print |s2|; } From 373c872e939f97c498b029cd08d4b24c0ab71c70 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Fri, 29 Apr 2016 01:45:59 -0500 Subject: [PATCH 25/53] Fix a few incorrect type tags in Bro broker source code These are just used for error reporting. --- src/broker/Data.h | 2 +- src/broker/data.bif | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/broker/Data.h b/src/broker/Data.h index f212979853..0045ad58ad 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -243,7 +243,7 @@ public: RecordIterator(RecordVal* v, TypeTag tag, Frame* f) : OpaqueVal(bro_broker::opaque_of_record_iterator), - dat(require_data_type(v, TYPE_VECTOR, f)), + dat(require_data_type(v, TYPE_RECORD, f)), it(dat.fields.begin()) {} diff --git a/src/broker/data.bif b/src/broker/data.bif index 1788931d86..d526d0a779 100644 --- a/src/broker/data.bif +++ b/src/broker/data.bif @@ -87,7 +87,7 @@ function Broker::__refine_to_subnet%(d: Broker::Data%): subnet function Broker::__refine_to_port%(d: Broker::Data%): port %{ auto& a = bro_broker::require_data_type(d->AsRecordVal(), - TYPE_SUBNET, frame); + TYPE_PORT, frame); return new PortVal(a.number(), bro_broker::to_bro_port_proto(a.type())); %} @@ -101,7 +101,7 @@ function Broker::__refine_to_time%(d: Broker::Data%): time function Broker::__refine_to_interval%(d: Broker::Data%): interval %{ auto v = bro_broker::require_data_type(d->AsRecordVal(), - TYPE_TIME, frame).value; + TYPE_INTERVAL, frame).value; return new Val(v, TYPE_INTERVAL); %} From f2acaec9b7512418f7b71947da90810ab082486e Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 29 Apr 2016 13:50:52 -0700 Subject: [PATCH 26/53] XMPP: Add StartTLS event and update tests Also tiny cleanyp to the code. --- src/analyzer/protocol/xmpp/CMakeLists.txt | 1 + src/analyzer/protocol/xmpp/Plugin.cc | 5 +-- src/analyzer/protocol/xmpp/XMPP.cc | 3 +- src/analyzer/protocol/xmpp/XMPP.h | 10 +++--- src/analyzer/protocol/xmpp/events.bif | 5 +++ src/analyzer/protocol/xmpp/xmpp-analyzer.pac | 5 ++- src/analyzer/protocol/xmpp/xmpp.pac | 3 ++ .../Baseline/core.print-bpf-filters/output2 | 10 +++--- .../canonified_loaded_scripts.log | 5 +-- .../canonified_loaded_scripts.log | 7 +++-- testing/btest/Baseline/plugins.hooks/output | 31 +++++++++++++++---- 11 files changed, 59 insertions(+), 26 deletions(-) create mode 100644 src/analyzer/protocol/xmpp/events.bif diff --git a/src/analyzer/protocol/xmpp/CMakeLists.txt b/src/analyzer/protocol/xmpp/CMakeLists.txt index 408f01d47c..ec5bb84837 100644 --- a/src/analyzer/protocol/xmpp/CMakeLists.txt +++ b/src/analyzer/protocol/xmpp/CMakeLists.txt @@ -6,6 +6,7 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI bro_plugin_begin(Bro XMPP) bro_plugin_cc(Plugin.cc) bro_plugin_cc(XMPP.cc) +bro_plugin_bif(events.bif) bro_plugin_pac(xmpp.pac xmpp-analyzer.pac xmpp-protocol.pac) bro_plugin_end() diff --git a/src/analyzer/protocol/xmpp/Plugin.cc b/src/analyzer/protocol/xmpp/Plugin.cc index b4332b447b..d3bfcc5b10 100644 --- a/src/analyzer/protocol/xmpp/Plugin.cc +++ b/src/analyzer/protocol/xmpp/Plugin.cc @@ -1,6 +1,4 @@ // See the file in the main distribution directory for copyright. - - #include "plugin/Plugin.h" #include "XMPP.h" @@ -14,10 +12,9 @@ public: { AddComponent(new ::analyzer::Component("XMPP", ::analyzer::xmpp::XMPP_Analyzer::Instantiate)); - plugin::Configuration config; config.name = "Bro::XMPP"; - config.description = "XMPP analyzer StartTLS only"; + config.description = "XMPP analyzer (StartTLS only)"; return config; } } plugin; diff --git a/src/analyzer/protocol/xmpp/XMPP.cc b/src/analyzer/protocol/xmpp/XMPP.cc index ee2667a276..72229aeaba 100644 --- a/src/analyzer/protocol/xmpp/XMPP.cc +++ b/src/analyzer/protocol/xmpp/XMPP.cc @@ -9,14 +9,13 @@ using namespace analyzer::xmpp; XMPP_Analyzer::XMPP_Analyzer(Connection* conn) : tcp::TCP_ApplicationAnalyzer("XMPP", conn) { - interp = new binpac::XMPP::XMPP_Conn(this); + interp = unique_ptr(new binpac::XMPP::XMPP_Conn(this)); had_gap = false; tls_active = false; } XMPP_Analyzer::~XMPP_Analyzer() { - delete interp; } void XMPP_Analyzer::Done() diff --git a/src/analyzer/protocol/xmpp/XMPP.h b/src/analyzer/protocol/xmpp/XMPP.h index 628be7bb2d..202403748a 100644 --- a/src/analyzer/protocol/xmpp/XMPP.h +++ b/src/analyzer/protocol/xmpp/XMPP.h @@ -14,12 +14,12 @@ public: XMPP_Analyzer(Connection* conn); virtual ~XMPP_Analyzer(); - virtual void Done(); - virtual void DeliverStream(int len, const u_char* data, bool orig); - virtual void Undelivered(uint64 seq, int len, bool orig); + void Done() override; + void DeliverStream(int len, const u_char* data, bool orig) override; + void Undelivered(uint64 seq, int len, bool orig) override; // Overriden from tcp::TCP_ApplicationAnalyzer. - virtual void EndpointEOF(bool is_orig); + void EndpointEOF(bool is_orig) override; void StartTLS(); @@ -27,7 +27,7 @@ public: { return new XMPP_Analyzer(conn); } protected: - binpac::XMPP::XMPP_Conn* interp; + std::unique_ptr interp; bool had_gap; bool tls_active; diff --git a/src/analyzer/protocol/xmpp/events.bif b/src/analyzer/protocol/xmpp/events.bif new file mode 100644 index 0000000000..ee36bd5333 --- /dev/null +++ b/src/analyzer/protocol/xmpp/events.bif @@ -0,0 +1,5 @@ +## Generated when a XMPP connection goes encrypted after a successful +## StartTLS exchange between the client and the server. +## +## c: The connection. +event xmpp_starttls%(c: connection%); diff --git a/src/analyzer/protocol/xmpp/xmpp-analyzer.pac b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac index 90b51ec183..3240b57bb3 100644 --- a/src/analyzer/protocol/xmpp/xmpp-analyzer.pac +++ b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac @@ -25,7 +25,10 @@ refine connection XMPP_Conn += { client_starttls = true; if ( !is_orig && token == "proceed" && client_starttls ) + { bro_analyzer()->StartTLS(); + BifEvent::generate_xmpp_starttls(bro_analyzer(), bro_analyzer()->Conn()); + } else if ( !is_orig && token == "proceed" ) reporter->Weird(bro_analyzer()->Conn(), "XMPP: proceed without starttls"); @@ -37,6 +40,6 @@ refine connection XMPP_Conn += { }; refine typeattr XMPP_TOKEN += &let { - proc: bool = $context.connection.proc_xmpp_token(is_orig, name, rest); + proc: bool = $context.connection.proc_xmpp_token(is_orig, name, rest); }; diff --git a/src/analyzer/protocol/xmpp/xmpp.pac b/src/analyzer/protocol/xmpp/xmpp.pac index 42ec85f0cc..e6b5f4bba0 100644 --- a/src/analyzer/protocol/xmpp/xmpp.pac +++ b/src/analyzer/protocol/xmpp/xmpp.pac @@ -6,7 +6,10 @@ %include binpac.pac %include bro.pac + %extern{ +#include "events.bif.h" + namespace analyzer { namespace xmpp { class XMPP_Analyzer; } } namespace binpac { namespace XMPP { class XMPP_Conn; } } typedef analyzer::xmpp::XMPP_Analyzer* XMPPAnalyzer; diff --git a/testing/btest/Baseline/core.print-bpf-filters/output2 b/testing/btest/Baseline/core.print-bpf-filters/output2 index d0f448441b..3321684b43 100644 --- a/testing/btest/Baseline/core.print-bpf-filters/output2 +++ b/testing/btest/Baseline/core.print-bpf-filters/output2 @@ -21,7 +21,9 @@ 1 5060 1 5072 1 514 +1 5222 1 5223 +1 5269 2 53 1 5353 1 5355 @@ -48,8 +50,8 @@ 1 992 1 993 1 995 -55 and -54 or -55 port -37 tcp +57 and +56 or +57 port +39 tcp 18 udp diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 703db6ea63..65f93aa51d 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-18-11-39 +#open 2016-04-29-20-49-16 #fields name #types string scripts/base/init-bare.bro @@ -111,6 +111,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro + build/scripts/base/bif/plugins/Bro_XMPP.events.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro @@ -132,4 +133,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2016-04-26-18-11-39 +#close 2016-04-29-20-49-16 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index c7a3c03d09..6ea7dd5d17 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-18-11-49 +#open 2016-04-29-20-49-25 #fields name #types string scripts/base/init-bare.bro @@ -111,6 +111,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro + build/scripts/base/bif/plugins/Bro_XMPP.events.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro @@ -295,6 +296,8 @@ scripts/base/init-default.bro scripts/base/protocols/syslog/consts.bro scripts/base/protocols/syslog/main.bro scripts/base/protocols/tunnels/__load__.bro + scripts/base/protocols/xmpp/__load__.bro + scripts/base/protocols/xmpp/main.bro scripts/base/files/pe/__load__.bro scripts/base/files/pe/consts.bro scripts/base/files/pe/main.bro @@ -305,4 +308,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-04-26-18-11-49 +#close 2016-04-29-20-49-25 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 61099efaf9..186f3a4a2a 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -57,6 +57,8 @@ 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_SSL, 995/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_SYSLOG, 514/udp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_TEREDO, 3544/udp)) -> +0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) -> +0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) -> @@ -116,6 +118,8 @@ 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_SSL, 995/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_SYSLOG, 514/udp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_TEREDO, 3544/udp)) -> +0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) -> +0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_AYIYA, {5072/udp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_DHCP, {67<...>/udp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_DNP3_TCP, {20000<...>/tcp})) -> @@ -140,6 +144,7 @@ 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_SSL, {5223<...>/tcp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_SYSLOG, {514/udp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_TEREDO, {3544/udp})) -> +0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_XMPP, {5222<...>/tcp})) -> 0.000000 MetaHookPost CallFunction(Cluster::is_enabled, , ()) -> 0.000000 MetaHookPost CallFunction(Cluster::is_enabled, , ()) -> 0.000000 MetaHookPost CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) -> @@ -233,7 +238,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -354,7 +359,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -467,6 +472,7 @@ 0.000000 MetaHookPost LoadFile(./Bro_X509.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_X509.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_X509.types.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_XMPP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_ZIP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./acld) -> -1 0.000000 MetaHookPost LoadFile(./addrs) -> -1 @@ -644,6 +650,7 @@ 0.000000 MetaHookPost LoadFile(base<...>/urls) -> -1 0.000000 MetaHookPost LoadFile(base<...>/utils) -> -1 0.000000 MetaHookPost LoadFile(base<...>/x509) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/xmpp) -> -1 0.000000 MetaHookPost QueueEvent(NetControl::init()) -> false 0.000000 MetaHookPost QueueEvent(bro_init()) -> false 0.000000 MetaHookPost QueueEvent(filter_change_tracking()) -> false @@ -706,6 +713,8 @@ 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_SSL, 995/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_SYSLOG, 514/udp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_TEREDO, 3544/udp)) +0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) +0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) @@ -765,6 +774,8 @@ 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_SSL, 995/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_SYSLOG, 514/udp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_TEREDO, 3544/udp)) +0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) +0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_AYIYA, {5072/udp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_DHCP, {67<...>/udp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_DNP3_TCP, {20000<...>/tcp})) @@ -789,6 +800,7 @@ 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_SSL, {5223<...>/tcp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_SYSLOG, {514/udp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_TEREDO, {3544/udp})) +0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_XMPP, {5222<...>/tcp})) 0.000000 MetaHookPre CallFunction(Cluster::is_enabled, , ()) 0.000000 MetaHookPre CallFunction(Cluster::is_enabled, , ()) 0.000000 MetaHookPre CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) @@ -882,7 +894,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -1003,7 +1015,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1116,6 +1128,7 @@ 0.000000 MetaHookPre LoadFile(./Bro_X509.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_X509.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_X509.types.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_XMPP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_ZIP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./acld) 0.000000 MetaHookPre LoadFile(./addrs) @@ -1293,6 +1306,7 @@ 0.000000 MetaHookPre LoadFile(base<...>/urls) 0.000000 MetaHookPre LoadFile(base<...>/utils) 0.000000 MetaHookPre LoadFile(base<...>/x509) +0.000000 MetaHookPre LoadFile(base<...>/xmpp) 0.000000 MetaHookPre QueueEvent(NetControl::init()) 0.000000 MetaHookPre QueueEvent(bro_init()) 0.000000 MetaHookPre QueueEvent(filter_change_tracking()) @@ -1355,6 +1369,8 @@ 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_SSL, 995/tcp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_SYSLOG, 514/udp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_TEREDO, 3544/udp) +0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_XMPP, 5222/tcp) +0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_XMPP, 5269/tcp) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_BACKDOOR) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_INTERCONN) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_STEPPINGSTONE) @@ -1414,6 +1430,8 @@ 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_SSL, 995/tcp) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_SYSLOG, 514/udp) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_TEREDO, 3544/udp) +0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_XMPP, 5222/tcp) +0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_XMPP, 5269/tcp) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_AYIYA, {5072/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_DHCP, {67<...>/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_DNP3_TCP, {20000<...>/tcp}) @@ -1438,6 +1456,7 @@ 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_SSL, {5223<...>/tcp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_SYSLOG, {514/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, {3544/udp}) +0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_XMPP, {5222<...>/tcp}) 0.000000 | HookCallFunction Cluster::is_enabled() 0.000000 | HookCallFunction Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)}) 0.000000 | HookCallFunction Files::register_for_mime_type(Files::ANALYZER_PE, application/x-dosexec) @@ -1530,7 +1549,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1651,7 +1670,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() From da014e1eca6136ff729eb11aacdf11688bcbb64d Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 2 May 2016 16:20:53 -0400 Subject: [PATCH 27/53] Rename the reporting interval variable for stats. --- scripts/policy/misc/stats.bro | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index d154da05e9..50032f6ec4 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -8,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 5min &redef; + const report_interval = 5min &redef; type Info: record { ## Timestamp for the measurement. @@ -146,10 +146,10 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs, ds) }; + schedule report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs, ds) }; } event bro_init() { - schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats(), get_dns_stats()) }; + schedule report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats(), get_dns_stats()) }; } From f8f599832832e027b8019554eae2d430f2193251 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 2 May 2016 16:43:08 -0400 Subject: [PATCH 28/53] Fixing tests for stats improvements --- scripts/base/init-bare.bro | 44 ++++++++++++------- .../canonified_loaded_scripts.log | 5 ++- .../canonified_loaded_scripts.log | 5 ++- testing/btest/Baseline/plugins.hooks/output | 26 ++++++----- testing/btest/bifs/net_stats_trace.test | 2 +- testing/btest/bifs/resource_usage.bro | 9 ---- 6 files changed, 50 insertions(+), 41 deletions(-) delete mode 100644 testing/btest/bifs/resource_usage.bro diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index c433aae503..5430d52ba4 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -535,7 +535,7 @@ type ReassemblerStats: record { unknown_size: count; ##< Byte size of reassembly tracking for unknown purposes. }; -## Summary statistics of all regular expression matchers. +## Statistics of all regular expression matchers. ## ## .. bro:see:: get_matcher_stats type MatcherStats: record { @@ -548,37 +548,51 @@ type MatcherStats: record { misses: count; ##< Number of cache misses. }; +## Statistics of timers. +## +## .. bro:see:: get_timer_stats type TimerStats: record { current: count; ##< Current number of pending timers. max: count; ##< Maximum number of concurrent timers pending so far. - cumulative: count; + cumulative: count; ##< Cumulative number of timers scheduled. }; +## Statistics of file analysis. +## +## .. bro:see:: get_file_analysis_stats type FileAnalysisStats: record { - current: count; - max: count; - cumulative: count; + current: count; ##< Current number of files being analyzed. + max: count; ##< Maximum number of concurrent files so far. + cumulative: count; ##< Cumulative number of files analyzed. }; +## Statistics related to Bro's active use of DNS. These numbers are +## about Bro performing DNS queries on it's own, not traffic +## being seen. +## +## .. bro:see:: get_dns_stats type DNSStats: record { - requests: count; - successful: count; - failed: count; - pending: count; - cached_hosts: count; - cached_addresses: count; + requests: count; ##< Number of DNS requests made + successful: count; ##< Number of successful DNS replies. + failed: count; ##< Number of DNS reply failures. + pending: count; ##< Current pending queries. + cached_hosts: count; ##< Number of cached hosts. + cached_addresses: count; ##< Number of cached addresses. }; ## Statistics about number of gaps in TCP connections. ## ## .. bro:see:: get_gap_stats type GapStats: record { - ack_events: count; ##< How many ack events *could* have had gaps. - ack_bytes: count; ##< How many bytes those covered. - gap_events: count; ##< How many *did* have gaps. - gap_bytes: count; ##< How many bytes were missing in the gaps. + ack_events: count; ##< How many ack events *could* have had gaps. + ack_bytes: count; ##< How many bytes those covered. + gap_events: count; ##< How many *did* have gaps. + gap_bytes: count; ##< How many bytes were missing in the gaps. }; +## Statistics about threads. +## +## .. bro:see:: get_thread_stats type ThreadStats: record { num_threads: count; }; diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 703db6ea63..f3fbccdd52 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-18-11-39 +#open 2016-05-02-20-39-26 #fields name #types string scripts/base/init-bare.bro @@ -50,6 +50,7 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro + build/scripts/base/bif/stats.bif.bro build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/functions.bif.bro build/scripts/base/bif/bloom-filter.bif.bro @@ -132,4 +133,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2016-04-26-18-11-39 +#close 2016-05-02-20-39-26 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index c7a3c03d09..37cfa6ff28 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-18-11-49 +#open 2016-05-02-20-39-35 #fields name #types string scripts/base/init-bare.bro @@ -50,6 +50,7 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro + build/scripts/base/bif/stats.bif.bro build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/functions.bif.bro build/scripts/base/bif/bloom-filter.bif.bro @@ -305,4 +306,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-04-26-18-11-49 +#close 2016-05-02-20-39-35 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 61099efaf9..186fc55040 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -233,7 +233,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -354,7 +354,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -527,6 +527,7 @@ 0.000000 MetaHookPost LoadFile(./sftp) -> -1 0.000000 MetaHookPost LoadFile(./shunt) -> -1 0.000000 MetaHookPost LoadFile(./site) -> -1 +0.000000 MetaHookPost LoadFile(./stats.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./std-dev) -> -1 0.000000 MetaHookPost LoadFile(./store) -> -1 0.000000 MetaHookPost LoadFile(./store.bif.bro) -> -1 @@ -882,7 +883,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -1003,7 +1004,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1176,6 +1177,7 @@ 0.000000 MetaHookPre LoadFile(./sftp) 0.000000 MetaHookPre LoadFile(./shunt) 0.000000 MetaHookPre LoadFile(./site) +0.000000 MetaHookPre LoadFile(./stats.bif.bro) 0.000000 MetaHookPre LoadFile(./std-dev) 0.000000 MetaHookPre LoadFile(./store) 0.000000 MetaHookPre LoadFile(./store.bif.bro) @@ -1530,7 +1532,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1651,7 +1653,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -1699,7 +1701,7 @@ 1362692526.869344 MetaHookPost CallFunction(ChecksumOffloading::check, , ()) -> 1362692526.869344 MetaHookPost CallFunction(NetControl::check_conn, , (141.142.228.5)) -> 1362692526.869344 MetaHookPost CallFunction(filter_change_tracking, , ()) -> -1362692526.869344 MetaHookPost CallFunction(net_stats, , ()) -> +1362692526.869344 MetaHookPost CallFunction(get_net_stats, , ()) -> 1362692526.869344 MetaHookPost CallFunction(new_connection, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=, vlan=, inner_vlan=, dpd=, conn=, extract_orig=F, extract_resp=F, thresholds=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> 1362692526.869344 MetaHookPost DrainEvents() -> 1362692526.869344 MetaHookPost QueueEvent(ChecksumOffloading::check()) -> false @@ -1710,7 +1712,7 @@ 1362692526.869344 MetaHookPre CallFunction(ChecksumOffloading::check, , ()) 1362692526.869344 MetaHookPre CallFunction(NetControl::check_conn, , (141.142.228.5)) 1362692526.869344 MetaHookPre CallFunction(filter_change_tracking, , ()) -1362692526.869344 MetaHookPre CallFunction(net_stats, , ()) +1362692526.869344 MetaHookPre CallFunction(get_net_stats, , ()) 1362692526.869344 MetaHookPre CallFunction(new_connection, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=, vlan=, inner_vlan=, dpd=, conn=, extract_orig=F, extract_resp=F, thresholds=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) 1362692526.869344 MetaHookPre DrainEvents() 1362692526.869344 MetaHookPre QueueEvent(ChecksumOffloading::check()) @@ -1722,7 +1724,7 @@ 1362692526.869344 | HookCallFunction ChecksumOffloading::check() 1362692526.869344 | HookCallFunction NetControl::check_conn(141.142.228.5) 1362692526.869344 | HookCallFunction filter_change_tracking() -1362692526.869344 | HookCallFunction net_stats() +1362692526.869344 | HookCallFunction get_net_stats() 1362692526.869344 | HookCallFunction new_connection([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=, vlan=, inner_vlan=, dpd=, conn=, extract_orig=F, extract_resp=F, thresholds=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=]) 1362692526.869344 | HookDrainEvents 1362692526.869344 | HookQueueEvent ChecksumOffloading::check() @@ -2127,11 +2129,11 @@ 1362692527.080972 MetaHookPost CallFunction(filter_change_tracking, , ()) -> 1362692527.080972 MetaHookPost CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) -> 1362692527.080972 MetaHookPost CallFunction(get_file_handle, , (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> +1362692527.080972 MetaHookPost CallFunction(get_net_stats, , ()) -> 1362692527.080972 MetaHookPost CallFunction(get_port_transport_proto, , (80/tcp)) -> 1362692527.080972 MetaHookPost CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) -> 1362692527.080972 MetaHookPost CallFunction(is_tcp_port, , (59856/tcp)) -> 1362692527.080972 MetaHookPost CallFunction(net_done, , (1362692527.080972)) -> -1362692527.080972 MetaHookPost CallFunction(net_stats, , ()) -> 1362692527.080972 MetaHookPost CallFunction(reading_traces, , ()) -> 1362692527.080972 MetaHookPost CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80)) -> 1362692527.080972 MetaHookPost CallFunction(sub_bytes, , (HTTP, 0, 1)) -> @@ -2157,11 +2159,11 @@ 1362692527.080972 MetaHookPre CallFunction(filter_change_tracking, , ()) 1362692527.080972 MetaHookPre CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) 1362692527.080972 MetaHookPre CallFunction(get_file_handle, , (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) +1362692527.080972 MetaHookPre CallFunction(get_net_stats, , ()) 1362692527.080972 MetaHookPre CallFunction(get_port_transport_proto, , (80/tcp)) 1362692527.080972 MetaHookPre CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) 1362692527.080972 MetaHookPre CallFunction(is_tcp_port, , (59856/tcp)) 1362692527.080972 MetaHookPre CallFunction(net_done, , (1362692527.080972)) -1362692527.080972 MetaHookPre CallFunction(net_stats, , ()) 1362692527.080972 MetaHookPre CallFunction(reading_traces, , ()) 1362692527.080972 MetaHookPre CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80)) 1362692527.080972 MetaHookPre CallFunction(sub_bytes, , (HTTP, 0, 1)) @@ -2188,11 +2190,11 @@ 1362692527.080972 | HookCallFunction filter_change_tracking() 1362692527.080972 | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp) 1362692527.080972 | HookCallFunction get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) +1362692527.080972 | HookCallFunction get_net_stats() 1362692527.080972 | HookCallFunction get_port_transport_proto(80/tcp) 1362692527.080972 | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856<...>/tcp]) 1362692527.080972 | HookCallFunction is_tcp_port(59856/tcp) 1362692527.080972 | HookCallFunction net_done(1362692527.080972) -1362692527.080972 | HookCallFunction net_stats() 1362692527.080972 | HookCallFunction reading_traces() 1362692527.080972 | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80) 1362692527.080972 | HookCallFunction sub_bytes(HTTP, 0, 1) diff --git a/testing/btest/bifs/net_stats_trace.test b/testing/btest/bifs/net_stats_trace.test index fcf3e9ba0d..cd9ee52a27 100644 --- a/testing/btest/bifs/net_stats_trace.test +++ b/testing/btest/bifs/net_stats_trace.test @@ -4,5 +4,5 @@ event bro_done() { - print net_stats(); + print get_net_stats(); } diff --git a/testing/btest/bifs/resource_usage.bro b/testing/btest/bifs/resource_usage.bro deleted file mode 100644 index 5cf3f0f962..0000000000 --- a/testing/btest/bifs/resource_usage.bro +++ /dev/null @@ -1,9 +0,0 @@ -# -# @TEST-EXEC: bro -b %INPUT - -event bro_init() - { - local a = resource_usage(); - if ( a$version != bro_version() ) - exit(1); - } From 8a6ca053bf3ff6a3a27cd679d5487b5ae098e13e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 3 May 2016 11:16:50 -0700 Subject: [PATCH 29/53] Updating submodule(s). [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- aux/broctl | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 8844dc7522..f62f544a44 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.4-500 | 2016-05-03 11:16:50 -0700 + + * Updating submodule(s). + 2.4-498 | 2016-04-28 11:34:52 -0700 * Rename Broker::print to Broker::send_print and Broker::event to diff --git a/VERSION b/VERSION index ada78ab155..e7d45626ab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-498 +2.4-500 diff --git a/aux/broctl b/aux/broctl index 7df7878abf..6f12b4da74 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 7df7878abfd864f9ae5609918c0f04f58b5f5e2d +Subproject commit 6f12b4da74e9e0885e1bd8cb67c2eda2b33c93a5 From 75e69d8c098dd79b89b4ac383ddaae75088e5d32 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 5 May 2016 12:47:21 -0500 Subject: [PATCH 30/53] Fix some "make doc" warnings --- scripts/base/frameworks/netcontrol/main.bro | 25 +++++++++++-------- .../frameworks/netcontrol/plugins/debug.bro | 2 +- src/event.bif | 8 +++--- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/scripts/base/frameworks/netcontrol/main.bro b/scripts/base/frameworks/netcontrol/main.bro index 563188921d..f3ff97b79b 100644 --- a/scripts/base/frameworks/netcontrol/main.bro +++ b/scripts/base/frameworks/netcontrol/main.bro @@ -109,21 +109,24 @@ export { ## ## r: The rule to install. ## - ## Returns: If succesful, returns an ID string unique to the rule that can later - ## be used to refer to it. If unsuccessful, returns an empty string. The ID is also - ## assigned to ``r$id``. Note that "successful" means "a plugin knew how to handle - ## the rule", it doesn't necessarily mean that it was indeed successfully put in - ## place, because that might happen asynchronously and thus fail only later. + ## Returns: If succesful, returns an ID string unique to the rule that can + ## later be used to refer to it. If unsuccessful, returns an empty + ## string. The ID is also assigned to ``r$id``. Note that + ## "successful" means "a plugin knew how to handle the rule", it + ## doesn't necessarily mean that it was indeed successfully put in + ## place, because that might happen asynchronously and thus fail + ## only later. global add_rule: function(r: Rule) : string; ## Removes a rule. ## ## id: The rule to remove, specified as the ID returned by :bro:id:`add_rule` . ## - ## Returns: True if succesful, the relevant plugin indicated that it knew how - ## to handle the removal. Note that again "success" means the plugin accepted the - ## removal. They might still fail to put it into effect, as that might happen - ## asynchronously and thus go wrong at that point. + ## Returns: True if succesful, the relevant plugin indicated that it knew + ## how to handle the removal. Note that again "success" means the + ## plugin accepted the removal. They might still fail to put it + ## into effect, as that might happen asynchronously and thus go + ## wrong at that point. global remove_rule: function(id: string) : bool; ## Searches all rules affecting a certain IP address. @@ -156,7 +159,7 @@ export { ## r: The rule now removed. ## ## p: The state for the plugin that had the rule in place and now - ## removed it. + ## removed it. ## ## msg: An optional informational message by the plugin. global rule_removed: event(r: Rule, p: PluginState, msg: string &default=""); @@ -168,7 +171,7 @@ export { ## i: Additional flow information, if supported by the protocol. ## ## p: The state for the plugin that had the rule in place and now - ## removed it. + ## removed it. ## ## msg: An optional informational message by the plugin. global rule_timeout: event(r: Rule, i: FlowInfo, p: PluginState); diff --git a/scripts/base/frameworks/netcontrol/plugins/debug.bro b/scripts/base/frameworks/netcontrol/plugins/debug.bro index f421dc55e3..a26a151400 100644 --- a/scripts/base/frameworks/netcontrol/plugins/debug.bro +++ b/scripts/base/frameworks/netcontrol/plugins/debug.bro @@ -11,7 +11,7 @@ export { ## plugin simply logs the operations it receives. ## ## do_something: If true, the plugin will claim it supports all operations; if - ## false, it will indicate it doesn't support any. + ## false, it will indicate it doesn't support any. global create_debug: function(do_something: bool) : PluginState; } diff --git a/src/event.bif b/src/event.bif index ff6ec059fb..b6227af9ad 100644 --- a/src/event.bif +++ b/src/event.bif @@ -306,10 +306,10 @@ event packet_contents%(c: connection, contents: string%); ## t2: The new payload. ## ## tcp_flags: A string with the TCP flags of the packet triggering the -## inconsistency. In the string, each character corresponds to one set flag, -## as follows: ``S`` -> SYN; ``F`` -> FIN; ``R`` -> RST; ``A`` -> ACK; ``P`` -> -## PUSH. This string will not always be set, only if the information is available; -## it's "best effort". +## inconsistency. In the string, each character corresponds to one +## set flag, as follows: ``S`` -> SYN; ``F`` -> FIN; ``R`` -> RST; +## ``A`` -> ACK; ``P`` -> PUSH. This string will not always be set, +## only if the information is available; it's "best effort". ## ## .. bro:see:: tcp_rexmit tcp_contents event rexmit_inconsistency%(c: connection, t1: string, t2: string, tcp_flags: string%); From 28125e367ef99859f1a79dd9a828d622bda5dd2c Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 5 May 2016 15:18:50 -0500 Subject: [PATCH 31/53] Fix more "make doc" warnings --- doc/components/bro-plugins/kafka/README.rst | 1 + scripts/base/frameworks/netcontrol/main.bro | 2 +- scripts/base/frameworks/netcontrol/types.bro | 16 ++++++++-------- src/analyzer/protocol/ssh/events.bif | 2 +- src/bro.bif | 6 +++--- 5 files changed, 14 insertions(+), 13 deletions(-) create mode 120000 doc/components/bro-plugins/kafka/README.rst diff --git a/doc/components/bro-plugins/kafka/README.rst b/doc/components/bro-plugins/kafka/README.rst new file mode 120000 index 0000000000..6ca2195f17 --- /dev/null +++ b/doc/components/bro-plugins/kafka/README.rst @@ -0,0 +1 @@ +../../../../aux/plugins/kafka/README \ No newline at end of file diff --git a/scripts/base/frameworks/netcontrol/main.bro b/scripts/base/frameworks/netcontrol/main.bro index f3ff97b79b..0acd4d0661 100644 --- a/scripts/base/frameworks/netcontrol/main.bro +++ b/scripts/base/frameworks/netcontrol/main.bro @@ -120,7 +120,7 @@ export { ## Removes a rule. ## - ## id: The rule to remove, specified as the ID returned by :bro:id:`add_rule` . + ## id: The rule to remove, specified as the ID returned by :bro:id:`NetControl::add_rule`. ## ## Returns: True if succesful, the relevant plugin indicated that it knew ## how to handle the removal. Note that again "success" means the diff --git a/scripts/base/frameworks/netcontrol/types.bro b/scripts/base/frameworks/netcontrol/types.bro index 440d63d8bc..3147420c99 100644 --- a/scripts/base/frameworks/netcontrol/types.bro +++ b/scripts/base/frameworks/netcontrol/types.bro @@ -14,7 +14,7 @@ export { MAC, ##< Activity involving a MAC address. }; - ## Type of a :bro:id:`Flow` for defining a flow. + ## Type for defining a flow. type Flow: record { src_h: subnet &optional; ##< The source IP address/subnet. src_p: port &optional; ##< The source port number. @@ -27,10 +27,10 @@ export { ## Type defining the enity an :bro:id:`Rule` is operating on. type Entity: record { ty: EntityType; ##< Type of entity. - conn: conn_id &optional; ##< Used with :bro:id:`CONNECTION` . - flow: Flow &optional; ##< Used with :bro:id:`FLOW` . - ip: subnet &optional; ##< Used with bro:id:`ADDRESS`; can specifiy a CIDR subnet. - mac: string &optional; ##< Used with :bro:id:`MAC`. + conn: conn_id &optional; ##< Used with :bro:enum:`NetControl::CONNECTION`. + flow: Flow &optional; ##< Used with :bro:enum:`NetControl::FLOW`. + ip: subnet &optional; ##< Used with :bro:enum:`NetControl::ADDRESS` to specifiy a CIDR subnet. + mac: string &optional; ##< Used with :bro:enum:`NetControl::MAC`. }; ## Target of :bro:id:`Rule` action. @@ -68,7 +68,7 @@ export { WHITELIST, }; - ## Type of a :bro:id:`FlowMod` for defining a flow modification action. + ## Type for defining a flow modification action. type FlowMod: record { src_h: addr &optional; ##< The source IP address. src_p: count &optional; ##< The source port number. @@ -90,8 +90,8 @@ export { priority: int &default=default_priority; ##< Priority if multiple rules match an entity (larger value is higher priority). location: string &optional; ##< Optional string describing where/what installed the rule. - out_port: count &optional; ##< Argument for bro:id:`REDIRECT` rules. - mod: FlowMod &optional; ##< Argument for :bro:id:`MODIFY` rules. + out_port: count &optional; ##< Argument for :bro:enum:`NetControl::REDIRECT` rules. + mod: FlowMod &optional; ##< Argument for :bro:enum:`NetControl::MODIFY` rules. id: string &default=""; ##< Internally determined unique ID for this rule. Will be set when added. cid: count &default=0; ##< Internally determined unique numeric ID for this rule. Set when added. diff --git a/src/analyzer/protocol/ssh/events.bif b/src/analyzer/protocol/ssh/events.bif index 57b736ac85..2c8079d9b7 100644 --- a/src/analyzer/protocol/ssh/events.bif +++ b/src/analyzer/protocol/ssh/events.bif @@ -120,7 +120,7 @@ event ssh1_server_host_key%(c: connection, p: string, e: string%); ## This event is generated when an :abbr:`SSH (Secure Shell)` ## encrypted packet is seen. This event is not handled by default, but ## is provided for heuristic analysis scripts. Note that you have to set -## :bro:id:`SSH::skip_processing_after_detection` to false to use this +## :bro:id:`SSH::disable_analyzer_after_detection` to false to use this ## event. This carries a performance penalty. ## ## c: The connection over which the :abbr:`SSH (Secure Shell)` diff --git a/src/bro.bif b/src/bro.bif index f21f927f92..5c3228eecc 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2465,7 +2465,7 @@ function to_subnet%(sn: string%): subnet ## ## Returns: The *a* address as a :bro:type:`subnet`. ## -## .. bro:see:: to_subset +## .. bro:see:: to_subnet function addr_to_subnet%(a: addr%): subnet %{ int width = (a->AsAddr().GetFamily() == IPv4 ? 32 : 128); @@ -2479,7 +2479,7 @@ function addr_to_subnet%(a: addr%): subnet ## ## Returns: The *s* subnet as a :bro:type:`addr`. ## -## .. bro:see:: to_subset +## .. bro:see:: to_subnet function subnet_to_addr%(sn: subnet%): addr %{ return new AddrVal(sn->Prefix()); @@ -2491,7 +2491,7 @@ function subnet_to_addr%(sn: subnet%): addr ## ## Returns: The width of the subnet. ## -## .. bro:see:: to_subset +## .. bro:see:: to_subnet function subnet_width%(sn: subnet%): count %{ return new Val(sn->Width(), TYPE_COUNT); From 2d9127888ffe2a2cbe0b8eaea0ccaf601801e92d Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 5 May 2016 16:35:31 -0500 Subject: [PATCH 32/53] Add some missing Bro script documentation Also fixed a few reST formatting issues. --- doc/script-reference/log-files.rst | 2 ++ scripts/base/files/x509/main.bro | 1 + scripts/base/frameworks/cluster/main.bro | 2 +- scripts/base/frameworks/netcontrol/main.bro | 20 ++++++++++---------- scripts/base/frameworks/notice/main.bro | 2 ++ scripts/base/protocols/http/main.bro | 1 + scripts/base/protocols/rfb/main.bro | 1 + scripts/base/protocols/sip/main.bro | 1 + scripts/base/protocols/smtp/main.bro | 1 + scripts/base/protocols/socks/main.bro | 1 + scripts/base/protocols/ssh/main.bro | 1 + scripts/base/protocols/ssl/main.bro | 1 + scripts/base/protocols/syslog/main.bro | 3 ++- 13 files changed, 25 insertions(+), 12 deletions(-) diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index c3fbca95a0..3c1720afd1 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -39,6 +39,8 @@ Network Protocols +----------------------------+---------------------------------------+---------------------------------+ | rdp.log | RDP | :bro:type:`RDP::Info` | +----------------------------+---------------------------------------+---------------------------------+ +| rfb.log | Remote Framebuffer (RFB) | :bro:type:`RFB::Info` | ++----------------------------+---------------------------------------+---------------------------------+ | sip.log | SIP | :bro:type:`SIP::Info` | +----------------------------+---------------------------------------+---------------------------------+ | smtp.log | SMTP transactions | :bro:type:`SMTP::Info` | diff --git a/scripts/base/files/x509/main.bro b/scripts/base/files/x509/main.bro index c097b84560..bbf99f6a4d 100644 --- a/scripts/base/files/x509/main.bro +++ b/scripts/base/files/x509/main.bro @@ -6,6 +6,7 @@ module X509; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the X.509 log. type Info: record { ## Current timestamp. ts: time &log; diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 3451cb4169..55fc084641 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -68,7 +68,7 @@ export { ## Events raised by TimeMachine instances and handled by workers. const tm2worker_events = /EMPTY/ &redef; - ## Events sent by the control host (i.e. BroControl) when dynamically + ## Events sent by the control host (i.e., BroControl) when dynamically ## connecting to a running instance to update settings or request data. const control_events = Control::controller_events &redef; diff --git a/scripts/base/frameworks/netcontrol/main.bro b/scripts/base/frameworks/netcontrol/main.bro index 0acd4d0661..65537ed9cf 100644 --- a/scripts/base/frameworks/netcontrol/main.bro +++ b/scripts/base/frameworks/netcontrol/main.bro @@ -23,20 +23,20 @@ export { # ### Generic functions and events. # ### - # Activates a plugin. - # - # p: The plugin to acticate. - # - # priority: The higher the priority, the earlier this plugin will be checked - # whether it supports an operation, relative to other plugins. + ## Activates a plugin. + ## + ## p: The plugin to acticate. + ## + ## priority: The higher the priority, the earlier this plugin will be checked + ## whether it supports an operation, relative to other plugins. global activate: function(p: PluginState, priority: int); - # Event that is used to initialize plugins. Place all plugin initialization - # related functionality in this event. + ## Event that is used to initialize plugins. Place all plugin initialization + ## related functionality in this event. global NetControl::init: event(); - # Event that is raised once all plugins activated in ``NetControl::init`` have finished - # their initialization. + ## Event that is raised once all plugins activated in ``NetControl::init`` + ## have finished their initialization. global NetControl::init_done: event(); # ### diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 2418b499e5..a203f6a772 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -44,6 +44,7 @@ export { ACTION_ALARM, }; + ## Type that represents a set of actions. type ActionSet: set[Notice::Action]; ## The notice framework is able to do automatic notice suppression by @@ -52,6 +53,7 @@ export { ## suppression. const default_suppression_interval = 1hrs &redef; + ## The record type that is used for representing and logging notices. type Info: record { ## An absolute time indicating when the notice occurred, ## defaults to the current network time. diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index e70d166f11..2988a1a646 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -21,6 +21,7 @@ export { ## not. const default_capture_password = F &redef; + ## The record type which contains the fields of the HTTP log. type Info: record { ## Timestamp for when the request happened. ts: time &log; diff --git a/scripts/base/protocols/rfb/main.bro b/scripts/base/protocols/rfb/main.bro index 03e39a40f9..3bcb86890b 100644 --- a/scripts/base/protocols/rfb/main.bro +++ b/scripts/base/protocols/rfb/main.bro @@ -3,6 +3,7 @@ module RFB; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the RFB log. type Info: record { ## Timestamp for when the event happened. ts: time &log; diff --git a/scripts/base/protocols/sip/main.bro b/scripts/base/protocols/sip/main.bro index dc790ad560..f629049928 100644 --- a/scripts/base/protocols/sip/main.bro +++ b/scripts/base/protocols/sip/main.bro @@ -10,6 +10,7 @@ module SIP; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the SIP log. type Info: record { ## Timestamp for when the request happened. ts: time &log; diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro index 6df9bddb54..766c0850bc 100644 --- a/scripts/base/protocols/smtp/main.bro +++ b/scripts/base/protocols/smtp/main.bro @@ -7,6 +7,7 @@ module SMTP; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the SMTP log. type Info: record { ## Time when the message was first seen. ts: time &log; diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro index c63092f609..e22ed718c6 100644 --- a/scripts/base/protocols/socks/main.bro +++ b/scripts/base/protocols/socks/main.bro @@ -6,6 +6,7 @@ module SOCKS; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the SOCKS log. type Info: record { ## Time when the proxy connection was first detected. ts: time &log; diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro index fad2da0b8e..d547e92e8f 100644 --- a/scripts/base/protocols/ssh/main.bro +++ b/scripts/base/protocols/ssh/main.bro @@ -8,6 +8,7 @@ export { ## The SSH protocol logging stream identifier. redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the SSH log. type Info: record { ## Time when the SSH connection began. ts: time &log; diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 8483f473f4..4c61df916a 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -8,6 +8,7 @@ module SSL; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the SSL log. type Info: record { ## Time when the SSL connection was first detected. ts: time &log; diff --git a/scripts/base/protocols/syslog/main.bro b/scripts/base/protocols/syslog/main.bro index 593c8ab9a2..6e74760225 100644 --- a/scripts/base/protocols/syslog/main.bro +++ b/scripts/base/protocols/syslog/main.bro @@ -7,7 +7,8 @@ module Syslog; export { redef enum Log::ID += { LOG }; - + + ## The record type which contains the fields of the syslog log. type Info: record { ## Timestamp when the syslog message was seen. ts: time &log; From f596d30386e980b92fa33bdbee8222d54733e047 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 5 May 2016 17:23:15 -0500 Subject: [PATCH 33/53] Fix some scripting tutorial examples Some of the examples in the scripting tutorial were regularly getting out of sync with the base scripts (because the line numbering would need to be updated). Fixed this maintenance burden by using small example scripts instead of actual Bro scripts. These small example scripts do not need to be kept in sync with the bro base scripts. --- doc/scripting/data_type_record.bro | 25 +++++++++++++++++++ doc/scripting/http_main.bro | 7 ++++++ doc/scripting/index.rst | 6 ++--- .../output | 2 +- .../output | 2 +- ...-doc_scripting_data_type_record_bro.btest} | 2 +- ...include-doc_scripting_http_main_bro.btest} | 2 +- 7 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 doc/scripting/data_type_record.bro create mode 100644 doc/scripting/http_main.bro rename testing/btest/Baseline/{doc.sphinx.include-scripts_base_protocols_conn_main_bro => doc.sphinx.include-doc_scripting_data_type_record_bro}/output (97%) rename testing/btest/Baseline/{doc.sphinx.include-scripts_base_protocols_http_main_bro => doc.sphinx.include-doc_scripting_http_main_bro}/output (93%) rename testing/btest/doc/sphinx/{include-scripts_base_protocols_conn_main_bro.btest => include-doc_scripting_data_type_record_bro.btest} (97%) rename testing/btest/doc/sphinx/{include-scripts_base_protocols_http_main_bro.btest => include-doc_scripting_http_main_bro.btest} (93%) diff --git a/doc/scripting/data_type_record.bro b/doc/scripting/data_type_record.bro new file mode 100644 index 0000000000..2380137cac --- /dev/null +++ b/doc/scripting/data_type_record.bro @@ -0,0 +1,25 @@ +module Conn; + +export { + ## The record type which contains column fields of the connection log. + type Info: record { + ts: time &log; + uid: string &log; + id: conn_id &log; + proto: transport_proto &log; + service: string &log &optional; + duration: interval &log &optional; + orig_bytes: count &log &optional; + resp_bytes: count &log &optional; + conn_state: string &log &optional; + local_orig: bool &log &optional; + local_resp: bool &log &optional; + missed_bytes: count &log &default=0; + history: string &log &optional; + orig_pkts: count &log &optional; + orig_ip_bytes: count &log &optional; + resp_pkts: count &log &optional; + resp_ip_bytes: count &log &optional; + tunnel_parents: set[string] &log; + }; +} diff --git a/doc/scripting/http_main.bro b/doc/scripting/http_main.bro new file mode 100644 index 0000000000..5182accb35 --- /dev/null +++ b/doc/scripting/http_main.bro @@ -0,0 +1,7 @@ +module HTTP; + +export { + ## This setting changes if passwords used in Basic-Auth are captured or + ## not. + const default_capture_password = F &redef; +} diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index a776fc0ad3..597d8ec41a 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -362,8 +362,7 @@ decrypted from HTTP streams is stored in :bro:see:`HTTP::default_capture_password` as shown in the stripped down excerpt from :doc:`/scripts/base/protocols/http/main.bro` below. -.. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro - :lines: 9-11,20-22,125 +.. btest-include:: ${DOC_ROOT}/scripting/http_main.bro Because the constant was declared with the ``&redef`` attribute, if we needed to turn this option on globally, we could do so by adding the @@ -825,8 +824,7 @@ example of the ``record`` data type in the earlier sections, the :bro:type:`Conn::Info`, which corresponds to the fields logged into ``conn.log``, is shown by the excerpt below. -.. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/conn/main.bro - :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,75,98,101,105,108,112,116-117,122 +.. btest-include:: ${DOC_ROOT}/scripting/data_type_record.bro Looking at the structure of the definition, a new collection of data types is being defined as a type called ``Info``. Since this type diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_conn_main_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_type_record_bro/output similarity index 97% rename from testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_conn_main_bro/output rename to testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_type_record_bro/output index 83e9d5bea1..6d8760700a 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_conn_main_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_type_record_bro/output @@ -1,6 +1,6 @@ # @TEST-EXEC: cat %INPUT >output && btest-diff output -main.bro +data_type_record.bro module Conn; diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_http_main_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_http_main_bro/output similarity index 93% rename from testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_http_main_bro/output rename to testing/btest/Baseline/doc.sphinx.include-doc_scripting_http_main_bro/output index e3f7a39429..9f49450799 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_http_main_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_http_main_bro/output @@ -1,6 +1,6 @@ # @TEST-EXEC: cat %INPUT >output && btest-diff output -main.bro +http_main.bro module HTTP; diff --git a/testing/btest/doc/sphinx/include-scripts_base_protocols_conn_main_bro.btest b/testing/btest/doc/sphinx/include-doc_scripting_data_type_record_bro.btest similarity index 97% rename from testing/btest/doc/sphinx/include-scripts_base_protocols_conn_main_bro.btest rename to testing/btest/doc/sphinx/include-doc_scripting_data_type_record_bro.btest index 83e9d5bea1..6d8760700a 100644 --- a/testing/btest/doc/sphinx/include-scripts_base_protocols_conn_main_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_scripting_data_type_record_bro.btest @@ -1,6 +1,6 @@ # @TEST-EXEC: cat %INPUT >output && btest-diff output -main.bro +data_type_record.bro module Conn; diff --git a/testing/btest/doc/sphinx/include-scripts_base_protocols_http_main_bro.btest b/testing/btest/doc/sphinx/include-doc_scripting_http_main_bro.btest similarity index 93% rename from testing/btest/doc/sphinx/include-scripts_base_protocols_http_main_bro.btest rename to testing/btest/doc/sphinx/include-doc_scripting_http_main_bro.btest index e3f7a39429..9f49450799 100644 --- a/testing/btest/doc/sphinx/include-scripts_base_protocols_http_main_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_scripting_http_main_bro.btest @@ -1,6 +1,6 @@ # @TEST-EXEC: cat %INPUT >output && btest-diff output -main.bro +http_main.bro module HTTP; From 40e9724de723249aec9cccedf489c7c15b6d6879 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 7 May 2016 01:22:38 -0400 Subject: [PATCH 34/53] Switching all use of gmtime and localtime to use reentrant variants. This was causing occasional problems with the time on processes running lots of threads. The use of gmtime in the json formatter is the likely culprit due to the fact that the json formatter runs in threads. More evidence for this is that the problem only appears to exhibit when logs are being written as JSON. --- src/bro.bif | 18 +++++++++++++----- src/threading/formatters/JSON.cc | 21 ++++++++++++++------- src/util.cc | 9 ++++++++- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index f21f927f92..e2baf62550 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -145,12 +145,17 @@ static void do_fmt(const char*& fmt, Val* v, ODesc* d) } time_t time = time_t(v->InternalDouble()); + struct tm t; + int is_time_fmt = *fmt == 'T'; + if ( ! localtime_r(&time, &t) ) + s.AddSP(""); + if ( ! strftime(out_buf, sizeof(out_buf), is_time_fmt ? "%Y-%m-%d-%H:%M" : "%Y-%m-%d-%H:%M:%S", - localtime(&time)) ) + &t) ) s.AddSP(""); else @@ -3140,9 +3145,11 @@ function strftime%(fmt: string, d: time%) : string %{ static char buffer[128]; - time_t t = time_t(d); + time_t timeval = time_t(d); + struct tm t; - if ( strftime(buffer, 128, fmt->CheckString(), localtime(&t)) == 0 ) + if ( ! localtime_r(&timeval, &t) || + ! strftime(buffer, 128, fmt->CheckString(), &t) ) return new StringVal(""); return new StringVal(buffer); @@ -3160,9 +3167,10 @@ function strftime%(fmt: string, d: time%) : string function strptime%(fmt: string, d: string%) : time %{ const time_t timeval = time_t(); - struct tm t = *localtime(&timeval); + struct tm t; - if ( strptime(d->CheckString(), fmt->CheckString(), &t) == NULL ) + if ( ! localtime_r(&timeval, &t) || + ! strptime(d->CheckString(), fmt->CheckString(), &t) ) { reporter->Warning("strptime conversion failed: fmt:%s d:%s", fmt->CheckString(), d->CheckString()); return new Val(0.0, TYPE_TIME); diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index 3558baee5c..45c7be3e93 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -116,21 +116,28 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const { char buffer[40]; char buffer2[40]; - time_t t = time_t(val->val.double_val); + time_t the_time = time_t(val->val.double_val); + struct tm t; - if ( strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", gmtime(&t)) > 0 ) + desc->AddRaw("\"", 1); + + if ( ! gmtime_r(&the_time, &t) || + ! strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", &t) ) + { + GetThread()->Error(GetThread()->Fmt("json formatter: failure getting time: (%" PRIu64 ")", val->val.double_val)); + // This was a failure, doesn't really matter what gets put here + // but it should probably stand out... + desc->Add("2000-01-01T00:00:00.000000"); + } + else { double integ; double frac = modf(val->val.double_val, &integ); snprintf(buffer2, sizeof(buffer2), "%s.%06.0fZ", buffer, frac * 1000000); - desc->AddRaw("\"", 1); desc->Add(buffer2); - desc->AddRaw("\"", 1); } - else - GetThread()->Error(GetThread()->Fmt("strftime error for JSON: %" PRIu64)); - + desc->AddRaw("\"", 1); } else if ( timestamps == TS_EPOCH ) diff --git a/src/util.cc b/src/util.cc index 0ea89beb90..1f10d7446d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -571,7 +571,14 @@ const char* fmt_access_time(double t) { static char buf[256]; time_t time = (time_t) t; - strftime(buf, sizeof(buf), "%d/%m-%H:%M", localtime(&time)); + struct tm ts; + + if ( ! localtime_r(&time, &ts) ) + { + reporter->InternalError("unable to get time"); + } + + strftime(buf, sizeof(buf), "%d/%m-%H:%M", &ts); return buf; } From b23ed77819b228cdc4e118e2f72e823e78a27e06 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 7 May 2016 12:19:07 -0700 Subject: [PATCH 35/53] Updating submodule(s). [nomail] --- CHANGES | 5 +++++ VERSION | 2 +- aux/plugins | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index ed7e16ca1b..9217c29793 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.4-544 | 2016-05-07 12:19:07 -0700 + + * Switching all use of gmtime and localtime to use reentrant + variants. (Seth Hall) + 2.4-541 | 2016-05-06 17:58:45 -0700 * A set of new built-in function for gathering execution statistics: diff --git a/VERSION b/VERSION index a4706ae7f1..9851c2a833 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-541 +2.4-544 diff --git a/aux/plugins b/aux/plugins index ab61be0c4f..bacbf297e3 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit ab61be0c4f128c976f72dfa5a09a87cd842f387a +Subproject commit bacbf297e37f92e1a00f91e293a4e059a5b6aedd From 90223fe4285313858dadfad18b59ca579b236a60 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 9 May 2016 09:45:21 -0700 Subject: [PATCH 36/53] Updating submodule(s). [nomail] --- aux/plugins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/plugins b/aux/plugins index bacbf297e3..6bd2ac4846 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit bacbf297e37f92e1a00f91e293a4e059a5b6aedd +Subproject commit 6bd2ac48466b57cdda84a593faebc25a59d98a51 From d91dd8d9a870d2328abd1c052f62889255d8b5f5 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 10 May 2016 06:24:35 -0500 Subject: [PATCH 37/53] Fix Bro and unit tests when broker is not enabled When Bro was compiled with broker disabled, then some Bro scripts were referencing functions and types that were not defined. Fixed by adding @ifdefs to several scripts. Removed one @ifdef because it was causing several unit tests to fail. Also fixed the @TEST-REQUIRES check in tests that rely on broker so that such tests are skipped when broker is disabled. --- scripts/base/frameworks/broker/main.bro | 3 +++ scripts/base/frameworks/broker/store.bro | 8 ++++++++ scripts/base/frameworks/netcontrol/plugins/acld.bro | 3 +++ scripts/base/frameworks/netcontrol/plugins/broker.bro | 4 ++++ scripts/base/frameworks/openflow/plugins/broker.bro | 3 +++ scripts/base/init-default.bro | 2 -- testing/btest/broker/clone_store.bro | 2 +- testing/btest/broker/connection_updates.bro | 2 +- testing/btest/broker/data.bro | 2 +- testing/btest/broker/enable-and-exit.bro | 2 +- testing/btest/broker/master_store.bro | 2 +- testing/btest/broker/remote_event.test | 2 +- testing/btest/broker/remote_log.test | 2 +- testing/btest/broker/remote_print.test | 2 +- testing/btest/core/leaks/broker/clone_store.bro | 2 +- testing/btest/core/leaks/broker/data.bro | 2 +- testing/btest/core/leaks/broker/master_store.bro | 2 +- testing/btest/core/leaks/broker/remote_event.test | 2 +- testing/btest/core/leaks/broker/remote_log.test | 2 +- testing/btest/core/leaks/broker/remote_print.test | 2 +- .../scripts/base/frameworks/netcontrol/acld-hook.bro | 2 +- testing/btest/scripts/base/frameworks/netcontrol/acld.bro | 2 +- .../btest/scripts/base/frameworks/netcontrol/broker.bro | 2 +- .../scripts/base/frameworks/openflow/broker-basic.bro | 2 +- 24 files changed, 39 insertions(+), 20 deletions(-) diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index a0024055a7..0818855d8f 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -270,6 +270,8 @@ export { module Broker; +@ifdef ( Broker::__enable ) + function enable(flags: EndpointFlags &default = EndpointFlags()) : bool { return __enable(flags); @@ -370,3 +372,4 @@ function unsubscribe_to_logs(topic_prefix: string): bool return __unsubscribe_to_logs(topic_prefix); } +@endif diff --git a/scripts/base/frameworks/broker/store.bro b/scripts/base/frameworks/broker/store.bro index f93b701d1c..8640e80648 100644 --- a/scripts/base/frameworks/broker/store.bro +++ b/scripts/base/frameworks/broker/store.bro @@ -57,6 +57,8 @@ export { rocksdb: RocksDBOptions &default = RocksDBOptions(); }; +@ifdef ( Broker::__enable ) + ## Create a master data store which contains key-value pairs. ## ## id: a unique name for the data store. @@ -720,12 +722,16 @@ export { ## ## Returns: element in the collection that the iterator currently references. global record_iterator_value: function(it: opaque of Broker::RecordIterator): Broker::Data; + +@endif } @load base/bif/store.bif module Broker; +@ifdef ( Broker::__enable ) + function create_master(id: string, b: BackendType &default = MEMORY, options: BackendOptions &default = BackendOptions()): opaque of Broker::Handle { @@ -1095,3 +1101,5 @@ function record_iterator_value(it: opaque of Broker::RecordIterator): Broker::Da { return __record_iterator_value(it); } + +@endif diff --git a/scripts/base/frameworks/netcontrol/plugins/acld.bro b/scripts/base/frameworks/netcontrol/plugins/acld.bro index ba50558d9a..a2f0fa2cc0 100644 --- a/scripts/base/frameworks/netcontrol/plugins/acld.bro +++ b/scripts/base/frameworks/netcontrol/plugins/acld.bro @@ -6,6 +6,8 @@ module NetControl; @load ../plugin @load base/frameworks/broker +@ifdef ( Broker::__enable ) + export { type AclRule : record { command: string; @@ -292,3 +294,4 @@ function create_acld(config: AcldConfig) : PluginState return p; } +@endif diff --git a/scripts/base/frameworks/netcontrol/plugins/broker.bro b/scripts/base/frameworks/netcontrol/plugins/broker.bro index 82e1d20f07..0687d70f82 100644 --- a/scripts/base/frameworks/netcontrol/plugins/broker.bro +++ b/scripts/base/frameworks/netcontrol/plugins/broker.bro @@ -8,6 +8,8 @@ module NetControl; @load ../plugin @load base/frameworks/broker +@ifdef ( Broker::__enable ) + export { ## Instantiates the broker plugin. global create_broker: function(host: addr, host_port: port, topic: string, can_expire: bool &default=F) : PluginState; @@ -161,3 +163,5 @@ function create_broker(host: addr, host_port: port, topic: string, can_expire: b return p; } + +@endif diff --git a/scripts/base/frameworks/openflow/plugins/broker.bro b/scripts/base/frameworks/openflow/plugins/broker.bro index ba15cc6ad1..a67b941e08 100644 --- a/scripts/base/frameworks/openflow/plugins/broker.bro +++ b/scripts/base/frameworks/openflow/plugins/broker.bro @@ -5,6 +5,8 @@ module OpenFlow; +@ifdef ( Broker::__enable ) + export { redef enum Plugin += { BROKER, @@ -93,3 +95,4 @@ function broker_new(name: string, host: addr, host_port: port, topic: string, dp return c; } +@endif diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 7c9bb4605b..fb3048165a 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -37,10 +37,8 @@ @load base/frameworks/reporter @load base/frameworks/sumstats @load base/frameworks/tunnels -@ifdef ( Broker::enable ) @load base/frameworks/openflow @load base/frameworks/netcontrol -@endif @load base/protocols/conn @load base/protocols/dhcp diff --git a/testing/btest/broker/clone_store.bro b/testing/btest/broker/clone_store.bro index c810a0d209..1ed35826dc 100644 --- a/testing/btest/broker/clone_store.bro +++ b/testing/btest/broker/clone_store.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: btest-bg-run clone "bro -b ../clone.bro broker_port=$BROKER_PORT >clone.out" # @TEST-EXEC: btest-bg-run master "bro -b ../master.bro broker_port=$BROKER_PORT >master.out" diff --git a/testing/btest/broker/connection_updates.bro b/testing/btest/broker/connection_updates.bro index bd08fff924..d431a59dbe 100644 --- a/testing/btest/broker/connection_updates.bro +++ b/testing/btest/broker/connection_updates.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out" # @TEST-EXEC: btest-bg-run send "bro -b ../send.bro broker_port=$BROKER_PORT >send.out" diff --git a/testing/btest/broker/data.bro b/testing/btest/broker/data.bro index ab51caf68d..49474e3a5a 100644 --- a/testing/btest/broker/data.bro +++ b/testing/btest/broker/data.bro @@ -1,4 +1,4 @@ -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out diff --git a/testing/btest/broker/enable-and-exit.bro b/testing/btest/broker/enable-and-exit.bro index 5a73a71c30..78800b31b0 100644 --- a/testing/btest/broker/enable-and-exit.bro +++ b/testing/btest/broker/enable-and-exit.bro @@ -1,4 +1,4 @@ -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: bro -b %INPUT >output # @TEST-EXEC: btest-diff output diff --git a/testing/btest/broker/master_store.bro b/testing/btest/broker/master_store.bro index a8cc8d3ad2..09f0f82880 100644 --- a/testing/btest/broker/master_store.bro +++ b/testing/btest/broker/master_store.bro @@ -1,4 +1,4 @@ -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: btest-bg-run master "bro -b %INPUT >out" # @TEST-EXEC: btest-bg-wait 60 diff --git a/testing/btest/broker/remote_event.test b/testing/btest/broker/remote_event.test index bd3c087d9a..5118f1a5e8 100644 --- a/testing/btest/broker/remote_event.test +++ b/testing/btest/broker/remote_event.test @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out" # @TEST-EXEC: btest-bg-run send "bro -b ../send.bro broker_port=$BROKER_PORT >send.out" diff --git a/testing/btest/broker/remote_log.test b/testing/btest/broker/remote_log.test index 52a534c8f9..a55d2f7e94 100644 --- a/testing/btest/broker/remote_log.test +++ b/testing/btest/broker/remote_log.test @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: btest-bg-run recv "bro -b ../common.bro ../recv.bro broker_port=$BROKER_PORT >recv.out" # @TEST-EXEC: btest-bg-run send "bro -b ../common.bro ../send.bro broker_port=$BROKER_PORT >send.out" diff --git a/testing/btest/broker/remote_print.test b/testing/btest/broker/remote_print.test index e8e9e0f71d..c64e70fedc 100644 --- a/testing/btest/broker/remote_print.test +++ b/testing/btest/broker/remote_print.test @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out" # @TEST-EXEC: btest-bg-run send "bro -b ../send.bro broker_port=$BROKER_PORT >send.out" diff --git a/testing/btest/core/leaks/broker/clone_store.bro b/testing/btest/core/leaks/broker/clone_store.bro index a02e3b2880..c3b11a7a0d 100644 --- a/testing/btest/core/leaks/broker/clone_store.bro +++ b/testing/btest/core/leaks/broker/clone_store.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks # @TEST-GROUP: leak diff --git a/testing/btest/core/leaks/broker/data.bro b/testing/btest/core/leaks/broker/data.bro index 5ce53b93dd..d67c879fbf 100644 --- a/testing/btest/core/leaks/broker/data.bro +++ b/testing/btest/core/leaks/broker/data.bro @@ -1,4 +1,4 @@ -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks # @TEST-GROUP: leaks diff --git a/testing/btest/core/leaks/broker/master_store.bro b/testing/btest/core/leaks/broker/master_store.bro index 8f4286ef3e..11f32b49ae 100644 --- a/testing/btest/core/leaks/broker/master_store.bro +++ b/testing/btest/core/leaks/broker/master_store.bro @@ -1,4 +1,4 @@ -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks # @TEST-GROUP: leaks diff --git a/testing/btest/core/leaks/broker/remote_event.test b/testing/btest/core/leaks/broker/remote_event.test index c68a9e5beb..3f63fcba76 100644 --- a/testing/btest/core/leaks/broker/remote_event.test +++ b/testing/btest/core/leaks/broker/remote_event.test @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks # @TEST-GROUP: leak diff --git a/testing/btest/core/leaks/broker/remote_log.test b/testing/btest/core/leaks/broker/remote_log.test index bf608dd459..baeab906f1 100644 --- a/testing/btest/core/leaks/broker/remote_log.test +++ b/testing/btest/core/leaks/broker/remote_log.test @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks # @TEST-GROUP: leak diff --git a/testing/btest/core/leaks/broker/remote_print.test b/testing/btest/core/leaks/broker/remote_print.test index 34266ebf4c..26e6317034 100644 --- a/testing/btest/core/leaks/broker/remote_print.test +++ b/testing/btest/core/leaks/broker/remote_print.test @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks # @TEST-GROUP: leak diff --git a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro index 779799ab4f..e131ec1dc0 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out" # @TEST-EXEC: btest-bg-run send "bro -b -r $TRACES/tls/ecdhe.pcap --pseudo-realtime ../send.bro broker_port=$BROKER_PORT >send.out" diff --git a/testing/btest/scripts/base/frameworks/netcontrol/acld.bro b/testing/btest/scripts/base/frameworks/netcontrol/acld.bro index 83a9cfc1af..a509b23c00 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/acld.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/acld.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out" # @TEST-EXEC: btest-bg-run send "bro -b -r $TRACES/tls/ecdhe.pcap --pseudo-realtime ../send.bro broker_port=$BROKER_PORT >send.out" diff --git a/testing/btest/scripts/base/frameworks/netcontrol/broker.bro b/testing/btest/scripts/base/frameworks/netcontrol/broker.bro index 4dbf3a09d2..f9328a458d 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/broker.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/broker.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out" # @TEST-EXEC: btest-bg-run send "bro -b -r $TRACES/smtp.trace --pseudo-realtime ../send.bro broker_port=$BROKER_PORT >send.out" diff --git a/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro b/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro index 014f07390b..9250590013 100644 --- a/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro +++ b/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: grep -q ENABLE_BROKER:BOOL=true $BUILD/CMakeCache.txt # @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out" # @TEST-EXEC: btest-bg-run send "bro -b -r $TRACES/smtp.trace --pseudo-realtime ../send.bro broker_port=$BROKER_PORT >send.out" From 7e3991d8793f104f4ea30382c98da7858e6d1644 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 10 May 2016 15:37:50 -0500 Subject: [PATCH 38/53] Fix a bug in receiving remote logs via broker When receiving a remote log via broker, there was a bug that would prevent a log from being written if the log record contained a field without the &log attribute that was followed by a field with the &log attribute. Updated a test case to catch this error. --- src/broker/Data.cc | 9 ++++++--- .../btest/Baseline/broker.remote_log/recv.recv.out | 12 ++++++------ testing/btest/broker/remote_log.test | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/broker/Data.cc b/src/broker/Data.cc index fe3f271c49..bc4197a974 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -318,25 +318,27 @@ struct val_converter { auto rt = type->AsRecordType(); auto rval = new RecordVal(rt); + auto idx = 0u; for ( auto i = 0u; i < static_cast(rt->NumFields()); ++i ) { if ( require_log_attr && ! rt->FieldDecl(i)->FindAttr(ATTR_LOG) ) continue; - if ( i >= a.fields.size() ) + if ( idx >= a.fields.size() ) { Unref(rval); return nullptr; } - if ( ! a.fields[i] ) + if ( ! a.fields[idx] ) { rval->Assign(i, nullptr); + ++idx; continue; } - auto item_val = bro_broker::data_to_val(move(*a.fields[i]), + auto item_val = bro_broker::data_to_val(move(*a.fields[idx]), rt->FieldType(i)); if ( ! item_val ) @@ -346,6 +348,7 @@ struct val_converter { } rval->Assign(i, item_val); + ++idx; } return rval; diff --git a/testing/btest/Baseline/broker.remote_log/recv.recv.out b/testing/btest/Baseline/broker.remote_log/recv.recv.out index ef9cb8402d..2f4a31df51 100644 --- a/testing/btest/Baseline/broker.remote_log/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_log/recv.recv.out @@ -1,6 +1,6 @@ -wrote log, [msg=ping, num=0, nolog=no] -wrote log, [msg=ping, num=1, nolog=no] -wrote log, [msg=ping, num=2, nolog=no] -wrote log, [msg=ping, num=3, nolog=no] -wrote log, [msg=ping, num=4, nolog=no] -wrote log, [msg=ping, num=5, nolog=no] +wrote log, [msg=ping, nolog=no, num=0] +wrote log, [msg=ping, nolog=no, num=1] +wrote log, [msg=ping, nolog=no, num=2] +wrote log, [msg=ping, nolog=no, num=3] +wrote log, [msg=ping, nolog=no, num=4] +wrote log, [msg=ping, nolog=no, num=5] diff --git a/testing/btest/broker/remote_log.test b/testing/btest/broker/remote_log.test index 52a534c8f9..f43e7b398b 100644 --- a/testing/btest/broker/remote_log.test +++ b/testing/btest/broker/remote_log.test @@ -19,8 +19,8 @@ export { type Info: record { msg: string &log; - num: count &log; nolog: string &default="no"; + num: count &log; }; global log_test: event(rec: Test::Info); From 65607239c93b6e26459ccbfc3eba2981dd3edab3 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Wed, 11 May 2016 12:26:11 +0200 Subject: [PATCH 39/53] Added interpreter error for local event variables. Scheduling a local event variable resulted in a global lookup instead of evaluating the local variable. To prevent misunderstandings, this will trigger an error now. --- src/parse.y | 17 +++++++++++++---- .../btest/Baseline/language.event-local-var/out | 1 + testing/btest/Baseline/language.event/out | 1 + testing/btest/language/event-local-var.bro | 16 ++++++++++++++++ testing/btest/language/event.bro | 7 ++++--- 5 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 testing/btest/Baseline/language.event-local-var/out create mode 100644 testing/btest/language/event-local-var.bro diff --git a/src/parse.y b/src/parse.y index c67732835f..f9eb7cbe9b 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1474,11 +1474,20 @@ event: TOK_ID '(' opt_expr_list ')' { set_location(@1, @4); - $$ = new EventExpr($1, $3); - ID* id = lookup_ID($1, current_module.c_str()); - if ( id && id->IsDeprecated() ) - reporter->Warning("deprecated (%s)", id->Name()); + ID* id = lookup_ID($1, current_module.c_str()); + if ( id ) + { + if ( ! id->IsGlobal() ) + { + yyerror(fmt("local identifier \"%s\" cannot be used to reference an event", $1)); + YYERROR; + } + if ( id->IsDeprecated() ) + reporter->Warning("deprecated (%s)", id->Name()); + } + + $$ = new EventExpr($1, $3); } ; diff --git a/testing/btest/Baseline/language.event-local-var/out b/testing/btest/Baseline/language.event-local-var/out new file mode 100644 index 0000000000..2802c45d69 --- /dev/null +++ b/testing/btest/Baseline/language.event-local-var/out @@ -0,0 +1 @@ +error in /home/jgras/devel/bro/testing/btest/.tmp/language.event-local-var/event-local-var.bro, line 15: local identifier "v" cannot be used to reference an event, at or near ")" diff --git a/testing/btest/Baseline/language.event/out b/testing/btest/Baseline/language.event/out index 41c3e0d717..14fa9c1e8a 100644 --- a/testing/btest/Baseline/language.event/out +++ b/testing/btest/Baseline/language.event/out @@ -1,6 +1,7 @@ event statement event part1 event part2 +assign event variable (6) schedule statement in bro_init schedule statement in global schedule statement another in bro_init diff --git a/testing/btest/language/event-local-var.bro b/testing/btest/language/event-local-var.bro new file mode 100644 index 0000000000..01a6dff829 --- /dev/null +++ b/testing/btest/language/event-local-var.bro @@ -0,0 +1,16 @@ +# @TEST-EXEC-FAIL: bro -b %INPUT &> out +# @TEST-EXEC: btest-diff out + + +event e1(num: count) + { + print fmt("event 1: %s", num); + } + +event bro_init() +{ + # Test assigning a local event variable to an event + local v: event(num: count); + v = e1; + schedule 1sec { v(6) }; # This should fail +} diff --git a/testing/btest/language/event.bro b/testing/btest/language/event.bro index 39a3e0da48..d4eef24731 100644 --- a/testing/btest/language/event.bro +++ b/testing/btest/language/event.bro @@ -21,7 +21,7 @@ event e3(test: string) event e4(num: count) { - print "assign event variable"; + print fmt("assign event variable (%s)", num); } # Note: the name of this event is intentionally the same as one above @@ -30,6 +30,8 @@ event e3(test: string) print "event part2"; } +global e5: event(num: count); + event bro_init() { # Test calling an event with "event" statement @@ -43,9 +45,8 @@ event bro_init() event e3("foo"); # Test assigning an event variable to an event - local e5: event(num: count); e5 = e4; - event e5(6); # TODO: this does not do anything + event e5(6); } # scheduling in outside of an event handler shouldn't crash. From 5baeb4790dbef6eb747aa93a134a1c85d0a0e1cd Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 11 May 2016 11:25:28 -0700 Subject: [PATCH 40/53] Updating submodule [nomail] --- aux/plugins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/plugins b/aux/plugins index 6bd2ac4846..17e3254454 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit 6bd2ac48466b57cdda84a593faebc25a59d98a51 +Subproject commit 17e32544540156832a8e547e9d17538f2e8a67ec From a4e5591e18cc497c16a06eb0d3bf44ba388f8362 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 12 May 2016 15:58:09 -0500 Subject: [PATCH 41/53] Fix some failing plugin tests on OS X 10.11 By default, OS X 10.11 does not include openssl headers. Since building a Bro plugin #includes Bro headers, which #include openssl headers, we need to tell cmake to find these so that the compiler can use them. --- testing/btest/plugins/file-plugin/CMakeLists.txt | 3 +++ testing/btest/plugins/protocol-plugin/CMakeLists.txt | 3 +++ 2 files changed, 6 insertions(+) diff --git a/testing/btest/plugins/file-plugin/CMakeLists.txt b/testing/btest/plugins/file-plugin/CMakeLists.txt index 4823ddb08f..1d0941d9da 100644 --- a/testing/btest/plugins/file-plugin/CMakeLists.txt +++ b/testing/btest/plugins/file-plugin/CMakeLists.txt @@ -9,6 +9,9 @@ endif () set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake) +find_package(OpenSSL) +include_directories(${OPENSSL_INCLUDE_DIR}) + include(BroPlugin) bro_plugin_begin(Demo Foo) diff --git a/testing/btest/plugins/protocol-plugin/CMakeLists.txt b/testing/btest/plugins/protocol-plugin/CMakeLists.txt index 4bc8460c06..a10fff1d67 100644 --- a/testing/btest/plugins/protocol-plugin/CMakeLists.txt +++ b/testing/btest/plugins/protocol-plugin/CMakeLists.txt @@ -9,6 +9,9 @@ endif () set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake) +find_package(OpenSSL) +include_directories(${OPENSSL_INCLUDE_DIR}) + include(BroPlugin) bro_plugin_begin(Demo Foo) From 8f6cdbb48901409502ea2b2ff8376d4f64ac0490 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 13 May 2016 07:43:47 -0700 Subject: [PATCH 42/53] Fix test failing when we use &> instead of 2> On the shell of a few systems, that apparently masks the return code. (Namely - Debian and FreeBSD) --- testing/btest/language/event-local-var.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/btest/language/event-local-var.bro b/testing/btest/language/event-local-var.bro index c7a0f059e1..d4dd9d19a5 100644 --- a/testing/btest/language/event-local-var.bro +++ b/testing/btest/language/event-local-var.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC-FAIL: bro -b %INPUT &> out +# @TEST-EXEC-FAIL: bro -b %INPUT 2> out # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out From d091e9ca89d6467962937c62f1b2aa20dbbc0c34 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 17 May 2016 10:31:37 -0700 Subject: [PATCH 43/53] Fix duplication of new_connection_contents event Addresses BIT-1602 --- src/analyzer/protocol/tcp/TCP.cc | 11 +++-------- .../.stdout | 2 ++ .../base/protocols/conn/new_connection_contents.bro | 7 +++++++ 3 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.conn.new_connection_contents/.stdout create mode 100644 testing/btest/scripts/base/protocols/conn/new_connection_contents.bro diff --git a/src/analyzer/protocol/tcp/TCP.cc b/src/analyzer/protocol/tcp/TCP.cc index 8b3876c7ce..56c01fa358 100644 --- a/src/analyzer/protocol/tcp/TCP.cc +++ b/src/analyzer/protocol/tcp/TCP.cc @@ -408,11 +408,6 @@ void TCP_Analyzer::EnableReassembly() TCP_Reassembler::Forward, orig), new TCP_Reassembler(this, this, TCP_Reassembler::Forward, resp)); - - reassembling = 1; - - if ( new_connection_contents ) - Event(new_connection_contents); } void TCP_Analyzer::SetReassembler(TCP_Reassembler* rorig, @@ -423,10 +418,10 @@ void TCP_Analyzer::SetReassembler(TCP_Reassembler* rorig, resp->AddReassembler(rresp); rresp->SetDstAnalyzer(this); - reassembling = 1; - - if ( new_connection_contents ) + if ( new_connection_contents && reassembling == 0 ) Event(new_connection_contents); + + reassembling = 1; } const struct tcphdr* TCP_Analyzer::ExtractTCP_Header(const u_char*& data, diff --git a/testing/btest/Baseline/scripts.base.protocols.conn.new_connection_contents/.stdout b/testing/btest/Baseline/scripts.base.protocols.conn.new_connection_contents/.stdout new file mode 100644 index 0000000000..1581730b33 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.conn.new_connection_contents/.stdout @@ -0,0 +1,2 @@ +new_connection_contents for [orig_h=192.168.1.77, orig_p=57640/tcp, resp_h=66.198.80.67, resp_p=6667/tcp] +new_connection_contents for [orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp] diff --git a/testing/btest/scripts/base/protocols/conn/new_connection_contents.bro b/testing/btest/scripts/base/protocols/conn/new_connection_contents.bro new file mode 100644 index 0000000000..42919f6f13 --- /dev/null +++ b/testing/btest/scripts/base/protocols/conn/new_connection_contents.bro @@ -0,0 +1,7 @@ +# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT +# @TEST-EXEC: btest-diff .stdout + +event new_connection_contents(c: connection) + { + print fmt("new_connection_contents for %s", cat(c$id)); + } From 8539f8f96b63d62039df981d8bf926ac26941576 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 17 May 2016 11:07:30 -0700 Subject: [PATCH 44/53] SMTP: Support SSL upgrade via X-ANONYMOUSTLS This seems to be a non-standardized microsoft extension that, besides having a different name, works pretty much the same as StartTLS. We just treat it as such. I tested this against provided traffic and it works; I do not have traffic I can share for a testcase. --- src/analyzer/protocol/smtp/SMTP.cc | 5 +++++ src/analyzer/protocol/smtp/SMTP.h | 2 +- src/analyzer/protocol/smtp/SMTP_cmd.def | 7 ++++--- src/analyzer/protocol/smtp/events.bif | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/analyzer/protocol/smtp/SMTP.cc b/src/analyzer/protocol/smtp/SMTP.cc index efc55ecc74..8296f83cb3 100644 --- a/src/analyzer/protocol/smtp/SMTP.cc +++ b/src/analyzer/protocol/smtp/SMTP.cc @@ -756,6 +756,7 @@ void SMTP_Analyzer::UpdateState(const int cmd_code, const int reply_code, bool o break; case SMTP_CMD_STARTTLS: + case SMTP_CMD_X_ANONYMOUSTLS: if ( st != SMTP_READY ) UnexpectedCommand(cmd_code, reply_code); @@ -818,6 +819,10 @@ int SMTP_Analyzer::ParseCmd(int cmd_len, const char* cmd) if ( ! cmd ) return -1; + // special case because we cannot define our usual macros with "-" + if ( strncmp(cmd, "X-ANONYMOUSTLS", cmd_len) == 0 ) + return SMTP_CMD_X_ANONYMOUSTLS; + for ( int code = SMTP_CMD_EHLO; code < SMTP_CMD_LAST; ++code ) if ( ! strncasecmp(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) ) return code; diff --git a/src/analyzer/protocol/smtp/SMTP.h b/src/analyzer/protocol/smtp/SMTP.h index e8010d9aef..b4396f28f7 100644 --- a/src/analyzer/protocol/smtp/SMTP.h +++ b/src/analyzer/protocol/smtp/SMTP.h @@ -30,7 +30,7 @@ typedef enum { SMTP_IN_DATA, // 6: after DATA SMTP_AFTER_DATA, // 7: after . and before reply SMTP_IN_AUTH, // 8: after AUTH and 334 - SMTP_IN_TLS, // 9: after STARTTLS and 220 + SMTP_IN_TLS, // 9: after STARTTLS/X-ANONYMOUSTLS and 220 SMTP_QUIT, // 10: after QUIT SMTP_AFTER_GAP, // 11: after a gap is detected SMTP_GAP_RECOVERY, // 12: after the first reply after a gap diff --git a/src/analyzer/protocol/smtp/SMTP_cmd.def b/src/analyzer/protocol/smtp/SMTP_cmd.def index 545136048d..72ef292d17 100644 --- a/src/analyzer/protocol/smtp/SMTP_cmd.def +++ b/src/analyzer/protocol/smtp/SMTP_cmd.def @@ -11,6 +11,8 @@ SMTP_CMD_DEF(VRFY) SMTP_CMD_DEF(EXPN) SMTP_CMD_DEF(HELP) SMTP_CMD_DEF(NOOP) +SMTP_CMD_DEF(STARTTLS) // RFC 2487 +SMTP_CMD_DEF(X_ANONYMOUSTLS) // The following two commands never explicitly appear in user input. SMTP_CMD_DEF(CONN_ESTABLISHMENT) // not an explicit SMTP command @@ -20,15 +22,14 @@ SMTP_CMD_DEF(END_OF_DATA) // not an explicit SMTP command // become deprecated (RFC 2821). // Client SHOULD NOT use SEND/SOML/SAML -SMTP_CMD_DEF(SEND) +SMTP_CMD_DEF(SEND) SMTP_CMD_DEF(SOML) SMTP_CMD_DEF(SAML) // System SHOULD NOT support TURN in absence of authentication. -SMTP_CMD_DEF(TURN) +SMTP_CMD_DEF(TURN) // SMTP extensions not supported yet. -SMTP_CMD_DEF(STARTTLS) // RFC 2487 SMTP_CMD_DEF(BDAT) // RFC 3030 SMTP_CMD_DEF(ETRN) // RFC 1985 SMTP_CMD_DEF(AUTH) // RFC 2554 diff --git a/src/analyzer/protocol/smtp/events.bif b/src/analyzer/protocol/smtp/events.bif index cffe3ba202..898e98e0d1 100644 --- a/src/analyzer/protocol/smtp/events.bif +++ b/src/analyzer/protocol/smtp/events.bif @@ -99,8 +99,8 @@ event smtp_data%(c: connection, is_orig: bool, data: string%); ## .. bro:see:: smtp_data smtp_request smtp_reply event smtp_unexpected%(c: connection, is_orig: bool, msg: string, detail: string%); -## Generated if a connection switched to using TLS using STARTTLS. After this -## event no more SMTP events will be raised for the connection. See the SSL +## Generated if a connection switched to using TLS using STARTTLS or X-ANONYMOUSTLS. +## After this event no more SMTP events will be raised for the connection. See the SSL ## analyzer for related SSL events, which will now be generated. ## ## c: The connection. From 7c1aab0ce53aaf3694ca0bf74079d21152945b30 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 May 2016 14:31:16 -0700 Subject: [PATCH 45/53] Updating submodule(s). [nomail] --- aux/bro-aux | 2 +- aux/plugins | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/bro-aux b/aux/bro-aux index 50d33db5d1..cb771a3cf5 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 50d33db5d12b81187ea127a08903b444a3c4bd04 +Subproject commit cb771a3cf592d46643eea35d206b9f3e1a0758f7 diff --git a/aux/plugins b/aux/plugins index ebab672fa4..6bd2ac4846 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit ebab672fa404b26944a6df6fbfb1aaab95ec5d48 +Subproject commit 6bd2ac48466b57cdda84a593faebc25a59d98a51 From e2dee49fc1daf9bd49a7ed2c02f74166de85192a Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 May 2016 16:05:50 -0700 Subject: [PATCH 46/53] Fixing control frameworks net_stats and peer_status commands. Turns out the code implementing them went missing with the recent merge of the stats changes. For peer status, I removed most of the numbers that the code used to return because (1) we don't have access to that data anymore, and (2) even in 2.4 the numbers returned already didn't make sense (because they were global values, not per peer). We could consider just removing the peer_status command at all. --- aux/broctl | 2 +- scripts/policy/frameworks/control/controllee.bro | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index 6f12b4da74..2d6caeacea 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6f12b4da74e9e0885e1bd8cb67c2eda2b33c93a5 +Subproject commit 2d6caeacead070a6e12524bedebaf95f9f444bc6 diff --git a/scripts/policy/frameworks/control/controllee.bro b/scripts/policy/frameworks/control/controllee.bro index 7001721f69..9646d100ab 100644 --- a/scripts/policy/frameworks/control/controllee.bro +++ b/scripts/policy/frameworks/control/controllee.bro @@ -22,10 +22,26 @@ event Control::id_value_request(id: string) event Control::peer_status_request() { + local status = ""; + for ( p in Communication::nodes ) + { + local peer = Communication::nodes[p]; + if ( ! peer$connected ) + next; + + status += fmt("%.6f peer=%s host=%s\n", + network_time(), peer$peer$descr, peer$host); + } + + event Control::peer_status_response(status); } event Control::net_stats_request() { + local ns = get_net_stats(); + local reply = fmt("%.6f recvd=%d dropped=%d link=%d\n", network_time(), + ns$pkts_recvd, ns$pkts_dropped, ns$pkts_link); + event Control::net_stats_response(reply); } event Control::configuration_update_request() From bc868d72a19488a6fd43dc83f8ab05e3a9225b07 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 17 May 2016 16:13:33 -0700 Subject: [PATCH 47/53] Fix the way that child analyzers are added. Bro contains functionality to add child analyzers delayed, so that an just added analyzer does not influence the list of current analyzers (which, in some combinations of mostly UDP and traffic replay by PIA can lead to duplicate packets sent to the analyzer). Sadly, this feature was broken sometime in the past, leading to the aforementioned duplicate packets. Re-enabling this also necessitated some changes in the analyzer manager, which immediately timed out all connections when that feature was re-enabled. There currently is no testcase (this is a bit hard to trigger); however, I will add one with a later fix for DTLS. --- src/analyzer/Analyzer.cc | 16 +++++++++++++++- src/analyzer/Analyzer.h | 4 ++++ src/analyzer/Manager.cc | 8 -------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/analyzer/Analyzer.cc b/src/analyzer/Analyzer.cc index b4048af467..5cf3fcb58d 100644 --- a/src/analyzer/Analyzer.cc +++ b/src/analyzer/Analyzer.cc @@ -395,7 +395,7 @@ bool Analyzer::AddChildAnalyzer(Analyzer* analyzer, bool init) // the list. analyzer->parent = this; - children.push_back(analyzer); + new_children.push_back(analyzer); if ( init ) analyzer->Init(); @@ -474,6 +474,13 @@ Analyzer* Analyzer::FindChild(ID arg_id) return child; } + LOOP_OVER_GIVEN_CHILDREN(i, new_children) + { + Analyzer* child = (*i)->FindChild(arg_id); + if ( child ) + return child; + } + return 0; } @@ -489,6 +496,13 @@ Analyzer* Analyzer::FindChild(Tag arg_tag) return child; } + LOOP_OVER_GIVEN_CHILDREN(i, new_children) + { + Analyzer* child = (*i)->FindChild(arg_tag); + if ( child ) + return child; + } + return 0; } diff --git a/src/analyzer/Analyzer.h b/src/analyzer/Analyzer.h index 83157aadde..df77a990ce 100644 --- a/src/analyzer/Analyzer.h +++ b/src/analyzer/Analyzer.h @@ -427,6 +427,10 @@ public: /** * Returns a list of all direct child analyzers. + * + * Note that this does not include the list of analyzers that are + * currently queued up to be added. If you just added an analyzer, + * it will not immediately be in this list. */ const analyzer_list& GetChildren() { return children; } diff --git a/src/analyzer/Manager.cc b/src/analyzer/Manager.cc index 67aa6a0d33..6082f433da 100644 --- a/src/analyzer/Manager.cc +++ b/src/analyzer/Manager.cc @@ -361,7 +361,6 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn) icmp::ICMP_Analyzer* icmp = 0; TransportLayerAnalyzer* root = 0; pia::PIA* pia = 0; - bool analyzed = false; bool check_port = false; switch ( conn->ConnTransport() ) { @@ -383,7 +382,6 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn) case TRANSPORT_ICMP: { root = icmp = new icmp::ICMP_Analyzer(conn); DBG_ANALYZER(conn, "activated ICMP analyzer"); - analyzed = true; break; } @@ -495,16 +493,10 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn) if ( pia ) root->AddChildAnalyzer(pia->AsAnalyzer()); - if ( root->GetChildren().size() ) - analyzed = true; - conn->SetRootAnalyzer(root, pia); root->Init(); root->InitChildren(); - if ( ! analyzed ) - conn->SetLifetime(non_analyzed_lifetime); - PLUGIN_HOOK_VOID(HOOK_SETUP_ANALYZER_TREE, HookSetupAnalyzerTree(conn)); return true; From 573b5426466b269eae03550885ca048d901db42e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 May 2016 16:23:47 -0700 Subject: [PATCH 48/53] Updating submodule(s). [nomail] --- aux/broctl | 2 +- aux/plugins | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broctl b/aux/broctl index 2d6caeacea..9cce8be1a9 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 2d6caeacead070a6e12524bedebaf95f9f444bc6 +Subproject commit 9cce8be1a9c02b275f8a51d175e4729bdb0afee4 diff --git a/aux/plugins b/aux/plugins index 6bd2ac4846..ebab672fa4 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit 6bd2ac48466b57cdda84a593faebc25a59d98a51 +Subproject commit ebab672fa404b26944a6df6fbfb1aaab95ec5d48 From b1c0306e4aad7f45012e9b235bdce3eae39e45b1 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 May 2016 16:25:21 -0700 Subject: [PATCH 49/53] Updating submodule(s). [nomail] --- CHANGES | 14 ++++++++++++++ VERSION | 2 +- aux/bro-aux | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 5addb633f5..bc04e3fe18 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,18 @@ +2.4-563 | 2016-05-17 16:25:21 -0700 + + * Fix duplication of new_connection_contents event. Addresses + BIT-1602 (Johanna Amann) + + * SMTP: Support SSL upgrade via X-ANONYMOUSTLS This seems to be a + non-standardized microsoft extension that, besides having a + different name, works pretty much the same as StartTLS. We just + treat it as such. (Johanna Amann) + + * Fixing control framework's net_stats and peer_status commands. For + the latter, this removes most of the values returned, as we don't + have access to them anymore. (Robin Sommer) + 2.4-555 | 2016-05-16 20:10:15 -0700 * Fix failing plugin tests on OS X 10.11. (Daniel Thayer) diff --git a/VERSION b/VERSION index 0372b35bee..e55267007d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-555 +2.4-563 diff --git a/aux/bro-aux b/aux/bro-aux index cb771a3cf5..50d33db5d1 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit cb771a3cf592d46643eea35d206b9f3e1a0758f7 +Subproject commit 50d33db5d12b81187ea127a08903b444a3c4bd04 From 39bdc397a0d0250be8b2f8a6a5858384653aa384 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 17 May 2016 16:36:46 -0700 Subject: [PATCH 50/53] DTLS: Fix interaction with STUN Now the DTLS analyzer cleanly skips all STUN messages; no warnings should be logged to dpd.log and parsing should work flawlessly with intermixed STUN messages. --- src/analyzer/protocol/ssl/DTLS.cc | 11 +++++++++++ .../dpd.log | 0 .../ssl.log | 10 ++++++++++ testing/btest/Traces/tls/webrtc-stun.pcap | Bin 0 -> 3662 bytes .../base/protocols/ssl/dtls-stun-dpd.test | 10 ++++++++++ 5 files changed, 31 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.protocols.ssl.dtls-stun-dpd/dpd.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.ssl.dtls-stun-dpd/ssl.log create mode 100644 testing/btest/Traces/tls/webrtc-stun.pcap create mode 100644 testing/btest/scripts/base/protocols/ssl/dtls-stun-dpd.test diff --git a/src/analyzer/protocol/ssl/DTLS.cc b/src/analyzer/protocol/ssl/DTLS.cc index c90e414031..d6c108671f 100644 --- a/src/analyzer/protocol/ssl/DTLS.cc +++ b/src/analyzer/protocol/ssl/DTLS.cc @@ -35,6 +35,17 @@ void DTLS_Analyzer::Done() void DTLS_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, uint64 seq, const IP_Hdr* ip, int caplen) { Analyzer::DeliverPacket(len, data, orig, seq, ip, caplen); + + // In this case the packet cannot be a DTLS packet and there + // is a nearly 100% chance that it is a STUN packet. Skip it without complaining. + // For details on the first two packets and the given masks, see the STUN RFC. + printf("Packet %d %d %d\n", data[0], data[1], orig); + if ( len > 20 && ( data[0] & 0xFE ) == 0 && data[1] > 0 && ( data[1] & 0xEF ) <= 12 ) + { + printf("Ignored! %d %d\n", data[0], data[1]); + return; + } + interp->NewData(orig, data, data + len); } diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.dtls-stun-dpd/dpd.log b/testing/btest/Baseline/scripts.base.protocols.ssl.dtls-stun-dpd/dpd.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.dtls-stun-dpd/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.dtls-stun-dpd/ssl.log new file mode 100644 index 0000000000..0328a5e982 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.dtls-stun-dpd/ssl.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2016-05-17-23-36-28 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1463527314.688817 CXWv6p3arKYeMETxOg 192.168.6.82 51462 74.201.205.9 43044 DTLSv10 TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA secp256r1 - F - - T Fk1e6E3pbe7faF41T5 FjQcYL1EtJ5VueihC7 CN=mantis.tokbox.com,O=Tokbox,L=San Francisco,ST=California,C=US CN=mantis.tokbox.com,O=Tokbox,L=San Francisco,ST=California,C=US CN=a CN=a +#close 2016-05-17-23-36-28 diff --git a/testing/btest/Traces/tls/webrtc-stun.pcap b/testing/btest/Traces/tls/webrtc-stun.pcap new file mode 100644 index 0000000000000000000000000000000000000000..6eb5f90372bb877d304e0fa6eda9a69adbe01a25 GIT binary patch literal 3662 zcmcgu2~ZT*7QNjw3^N16HiLo^7!(Bg27B0p`N|@K;=;ZNhyw@=3M3$4)Ub=91|o_; zL?IdhQE&;kvDgYGAu0-z-~tJX5oHOAnkX3OzlRVO>!-^4RlloVb-&mB?m2VMx%W*= z-KnohV8ZAnl>!iGl2ts<^J6P6>_yLbz^rXuUnFl3UHoprB{tXrY>O}euu}dof-YgX zENQsLX<(J;zOufSGt!>=gO-*CW78zL@ zi`JN01O%Aj=(=d=05c*1q=^Q=J`k$4WRu0otH1_9wD9F7gTDl&kGFQ$mH4c^B5FMB zl&B9({mtpKFw0Mvmw=l)=bleq6{xsCc}CoH~M>KLPm@ z=2Da&H;lY_@Nga7>QWhB;mgPQ8BK@f?f#OPerqBSL;(@3d!)dMD4>zC7=c z?Z*YDb)LMhclOy=N^ovgo?(AauDAXDUi1rcv&Pn$Sqge9FvCCkt|VfHHxtKl^R6m4 zg0U)B{3fdMOYUfHCNP|&O4kQBw7eqEf#SSbEk}Ew0?;8UzqZBhN~2q?H`!&og2Ng%?t9__RAeYP_rc zYi7|}&$^Aa6@?o)k@4uuELuQw;UlmBb1;K#uoco^3#|^8kWMSoGGGd+G@s^xF&Kde zDEJEUAQy5_CtL`&U_+~c3G9PKv;xh9xu|1WgDkB=vtbJGK?!*DRG3clXa;aW6;wbG zcZHm7?HPEU&Ob0a- zfs5iyfvI>u$f9_vsF%wi1U`yD0s|o_Aq+)8x+tI*I51A}w}Vcb2w(^+SYB=Of#i(w3U4n$I4%V-o~6`Ch;b zd$)6;n1tam*?wf6juo$mT= z?k!&IpsI5u>|liz_u8}-#Ga!(fILPHKa4}F1-QFP}4!KIQ37L`<9$RG)lWC~aAHKe#Kwn3Uh zTzGEbEXignK~s(v2bJD}LVow4F+MxNHf`;PfycDF>7-R-b|d9|Ugvh@0t#@lKyFEd`)ohV^Bi@lpo zIUia+y|wXh<_C%LeVIE-9>uN%d&-W~hJ!aR?WCe&ADP&FG5E}5l|uijJ-e4}%Ujp^ zN6xZ7!Apu1t{T!7>^Rlk8u@t8NyyT?p!KP!t+i+nR(S8~k;ZH^8l%Yo6&@gmGlprh z7=#rfJocIi@(TFP$loVy+m$>2scY7-v2A~;8gE;0F!+76YfdPvNnMk8Q9P6)E69Cv zyK!b;i2l6I!gDNKFp952USQ;1$GFgmh~hE1YchU9xTtFeIgam|jEj7)u^&T_KFHfE z-g+0^$)!LprjkiJ?F@-&SxBc^Kq!e>D0U%$Mh@KW>!D; zRd3u^nK)F_;&C9ovwfh)v9hC=8}O=cs`_c}St_+g!=_#5XR+pk0A)$To)bpvUBspj za=fkTf7dnbI&JyVDD#_l>Ts5j1(M&OS7~OP|q6dAV4# zs4X5hFFJLpxXh5h96eifJVBa9>>u+P3Cq#ZY%@~&HrCAIl#}P4RxJpY+uUBb!pV8J zjOASAX0MVy=WiVAog@pdoqo2l+{M{rPockNjar<}!ooR2_X2hfsJedV-WhuehglbB`& zsJpyD(_+q56lZB>jp|CHP>*-UpdQ#Ehrcslg~(q^YFpV#%}q9W)paC_82Ld$)r*W(t{XH z^~SBAwVX(^LyaPzV58rv**NA=HL Lc+taIudnhSXzjo6 literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/protocols/ssl/dtls-stun-dpd.test b/testing/btest/scripts/base/protocols/ssl/dtls-stun-dpd.test new file mode 100644 index 0000000000..e005e82e03 --- /dev/null +++ b/testing/btest/scripts/base/protocols/ssl/dtls-stun-dpd.test @@ -0,0 +1,10 @@ +# @TEST-EXEC: bro -r $TRACES/tls/webrtc-stun.pcap %INPUT +# @TEST-EXEC: btest-diff ssl.log +# @TEST-EXEC: touch dpd.log +# @TEST-EXEC: btest-diff dpd.log + +event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) + { + print version, client_random, session_id, ciphers; + } + From 74d0493289016ab9b00b91e6c30f2f83bdf926b8 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 17 May 2016 16:39:44 -0700 Subject: [PATCH 51/53] Forgot to remove debug output. --- src/analyzer/protocol/ssl/DTLS.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/analyzer/protocol/ssl/DTLS.cc b/src/analyzer/protocol/ssl/DTLS.cc index d6c108671f..cadfcb2ff7 100644 --- a/src/analyzer/protocol/ssl/DTLS.cc +++ b/src/analyzer/protocol/ssl/DTLS.cc @@ -39,7 +39,6 @@ void DTLS_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, uint64 // In this case the packet cannot be a DTLS packet and there // is a nearly 100% chance that it is a STUN packet. Skip it without complaining. // For details on the first two packets and the given masks, see the STUN RFC. - printf("Packet %d %d %d\n", data[0], data[1], orig); if ( len > 20 && ( data[0] & 0xFE ) == 0 && data[1] > 0 && ( data[1] & 0xEF ) <= 12 ) { printf("Ignored! %d %d\n", data[0], data[1]); From 9b2c81e00a94f3cbb3be7aba7d95e215059db797 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 17 May 2016 16:58:30 -0700 Subject: [PATCH 52/53] DTLS: Fix binpac bug with DTLSv1.2 client hellos --- src/analyzer/protocol/ssl/tls-handshake-protocol.pac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/analyzer/protocol/ssl/tls-handshake-protocol.pac b/src/analyzer/protocol/ssl/tls-handshake-protocol.pac index b24352d099..3b65e63ee7 100644 --- a/src/analyzer/protocol/ssl/tls-handshake-protocol.pac +++ b/src/analyzer/protocol/ssl/tls-handshake-protocol.pac @@ -75,7 +75,7 @@ type ClientHello(rec: HandshakeRecord) = record { session_len : uint8; session_id : uint8[session_len]; dtls_cookie: case client_version of { - DTLSv10 -> cookie: ClientHelloCookie(rec); + DTLSv10, DTLSv12 -> cookie: ClientHelloCookie(rec); default -> nothing: bytestring &length=0; }; csuit_len : uint16 &check(csuit_len > 1 && csuit_len % 2 == 0); From e6b680ab09487e410258d3eb28e588160ca92624 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 17 May 2016 17:08:20 -0700 Subject: [PATCH 53/53] DTLS: Use magix constant from rfc5389 for stun detection. --- src/analyzer/protocol/ssl/DTLS.cc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/analyzer/protocol/ssl/DTLS.cc b/src/analyzer/protocol/ssl/DTLS.cc index cadfcb2ff7..5301e962d4 100644 --- a/src/analyzer/protocol/ssl/DTLS.cc +++ b/src/analyzer/protocol/ssl/DTLS.cc @@ -36,14 +36,9 @@ void DTLS_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, uint64 { Analyzer::DeliverPacket(len, data, orig, seq, ip, caplen); - // In this case the packet cannot be a DTLS packet and there - // is a nearly 100% chance that it is a STUN packet. Skip it without complaining. - // For details on the first two packets and the given masks, see the STUN RFC. - if ( len > 20 && ( data[0] & 0xFE ) == 0 && data[1] > 0 && ( data[1] & 0xEF ) <= 12 ) - { - printf("Ignored! %d %d\n", data[0], data[1]); + // In this case the packet is a STUN packet. Skip it without complaining. + if ( len > 20 && data[4] == 0x21 && data[5] == 0x12 && data[6] == 0xa4 && data[7] == 0x42 ) return; - } interp->NewData(orig, data, data + len); }