From 51203d71934436c7f30922dfc6e0f8d4800a9f62 Mon Sep 17 00:00:00 2001 From: Mike Smiley Date: Tue, 3 Feb 2015 14:29:34 -0500 Subject: [PATCH 01/37] "id" not defined for debug code "id" not defined for debug code when using -DPROFILE_BRO_FUNCTIONS --- src/Func.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Func.cc b/src/Func.cc index d66e9c71fa..693a4535d4 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -323,7 +323,7 @@ int BroFunc::IsPure() const Val* BroFunc::Call(val_list* args, Frame* parent) const { #ifdef PROFILE_BRO_FUNCTIONS - DEBUG_MSG("Function: %s\n", id->Name()); + DEBUG_MSG("Function: %s\n", Name()); #endif SegmentProfiler(segment_logger, location); From 565ad360c6f0afa0583b472eadde841b7521e9d1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 3 Feb 2015 17:02:45 -0600 Subject: [PATCH 02/37] Add x509 canonifier to a unit test. --- CHANGES | 4 ++++ VERSION | 2 +- .../btest/scripts/policy/protocols/ssl/validate-ocsp.bro | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index 2d4ce98c31..76e09d27c1 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-406 | 2015-02-03 17:02:45 -0600 + + * Add x509 canonifier to a unit test. (Jon Siwek) + 2.3-405 | 2015-02-02 11:14:24 -0600 * Fix memory leak in new split_string* functions. (Jon Siwek) diff --git a/VERSION b/VERSION index 57d75d10d0..15530c9394 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-405 +2.3-406 diff --git a/testing/btest/scripts/policy/protocols/ssl/validate-ocsp.bro b/testing/btest/scripts/policy/protocols/ssl/validate-ocsp.bro index e7e3c3ff8e..3f88638ee3 100644 --- a/testing/btest/scripts/policy/protocols/ssl/validate-ocsp.bro +++ b/testing/btest/scripts/policy/protocols/ssl/validate-ocsp.bro @@ -1,10 +1,10 @@ # @TEST-EXEC: bro -C -r $TRACES/tls/ocsp-stapling.trace %INPUT -# @TEST-EXEC: btest-diff ssl.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-x509-names btest-diff ssl.log # @TEST-EXEC: bro -C -r $TRACES/tls/ocsp-stapling-twimg.trace %INPUT # @TEST-EXEC: mv ssl.log ssl-twimg.log -# @TEST-EXEC: btest-diff ssl-twimg.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-x509-names btest-diff ssl-twimg.log # @TEST-EXEC: bro -C -r $TRACES/tls/ocsp-stapling-digicert.trace %INPUT # @TEST-EXEC: mv ssl.log ssl-digicert.log -# @TEST-EXEC: btest-diff ssl-digicert.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-x509-names btest-diff ssl-digicert.log @load protocols/ssl/validate-ocsp From a97cd1f3a24d93e81190cc28ba283d26f31035c8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 5 Feb 2015 09:09:08 -0500 Subject: [PATCH 03/37] Fix a bug in the core files framework with handling the BOF buffer. - Any files where the total size was below the size of the default bof_buffer size couldn't have stream analyzers successfully attached because the bof_buffer never reached the full size and was never flushed. This branch explicitly marks the buf_buffer as full and flushes it when the file is being removed. --- src/file_analysis/File.cc | 18 +++++++++++------- .../files.log | 10 ++++++++++ .../file-analysis/big-bof-buffer.bro | 6 ++++++ 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.file-analysis.big-bof-buffer/files.log create mode 100644 testing/btest/scripts/base/frameworks/file-analysis/big-bof-buffer.bro diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index 50617f27b6..cc1f86412c 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -492,18 +492,22 @@ void File::EndOfFile() if ( done ) return; - if ( ! did_mime_type && - LookupFieldDefaultCount(missing_bytes_idx) == 0 ) - DetectMIME(); - - analyzers.DrainModifications(); - if ( file_reassembler ) { file_reassembler->Flush(); - analyzers.DrainModifications(); } + // Mark the bof_buffer as full in case it isn't yet + // so that the whole thing can be flushed out to + // any stream analyzers. + if ( ! bof_buffer.full ) + { + bof_buffer.full = true; + DeliverStream((const u_char*) "", 0); + } + + analyzers.DrainModifications(); + done = true; file_analysis::Analyzer* a = 0; diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.big-bof-buffer/files.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.big-bof-buffer/files.log new file mode 100644 index 0000000000..cebe140bda --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.big-bof-buffer/files.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path files +#open 2015-02-05-13-55-41 +#fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted +#types time string set[addr] set[addr] set[string] string count set[string] string string interval bool bool count count count count bool string string string string string +1362692527.009512 FakNcS1Jfe01uljb3 192.150.187.43 141.142.228.5 CXWv6p3arKYeMETxOg HTTP 0 MD5,SHA1 text/plain - 0.000263 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 - - +#close 2015-02-05-13-55-41 diff --git a/testing/btest/scripts/base/frameworks/file-analysis/big-bof-buffer.bro b/testing/btest/scripts/base/frameworks/file-analysis/big-bof-buffer.bro new file mode 100644 index 0000000000..0f7e23ddcf --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/big-bof-buffer.bro @@ -0,0 +1,6 @@ +# @TEST-EXEC: bro -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: btest-diff files.log + +@load frameworks/files/hash-all-files + +redef default_file_bof_buffer_size=5000; From 8859c73bde5d392be6081bade798e26b79e0e56e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 5 Feb 2015 10:04:04 -0600 Subject: [PATCH 04/37] Add/fix log fields in x509 diff canonifier. --- testing/scripts/diff-remove-x509-names | 30 +++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/testing/scripts/diff-remove-x509-names b/testing/scripts/diff-remove-x509-names index 4863efc990..4534cb7d87 100755 --- a/testing/scripts/diff-remove-x509-names +++ b/testing/scripts/diff-remove-x509-names @@ -3,19 +3,25 @@ # A diff canonifier that removes all X.509 Distinguished Name subject fields # because that output can differ depending on installed OpenSSL version. -BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1; cs_col = -1; ci_col = -1 } +BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1; is_col = -1; cs_col = -1; ci_col = -1; cert_subj_col = -1; cert_issuer_col = -1 } /^#fields/ { for ( i = 2; i < NF; ++i ) { if ( $i == "subject" ) s_col = i-1; - if ( $i == "issuer_subject" ) + if ( $i == "issuer" ) i_col = i-1; + if ( $i == "issuer_subject" ) + is_col = i-1; if ( $i == "client_subject" ) cs_col = i-1; - if ( $i == "client_issuer_subject" ) + if ( $i == "client_issuer" ) ci_col = i-1; + if ( $i == "certificate.subject" ) + cert_subj_col = i-1; + if ( $i == "certificate.issuer" ) + cert_issuer_col = i-1; } } @@ -31,6 +37,12 @@ i_col >= 0 { $i_col = "+"; } +is_col >= 0 { + if ( $is_col != "-" ) + # Mark that it's set, but ignore content. + $is_col = "+"; +} + cs_col >= 0 { if ( $cs_col != "-" ) # Mark that it's set, but ignore content. @@ -43,6 +55,18 @@ ci_col >= 0 { $ci_col = "+"; } +cert_subj_col >= 0 { + if ( $cert_subj_col != "-" ) + # Mark that it's set, but ignore content. + $cert_subj_col = "+"; +} + +cert_issuer_col >= 0 { + if ( $cert_issuer_col != "-" ) + # Mark that it's set, but ignore content. + $cert_issuer_col = "+"; +} + { print; } From 9592f6422530aff4873d31453954acacd6034e43 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 5 Feb 2015 12:44:10 -0500 Subject: [PATCH 05/37] Update the SOCKS analyzer to support user/pass login. - This addresses BIT-1011 - Add a new field to socks.log; "password". - Two new events; socks_login_userpass and socks_login_reply. - One new weird for unsupported authentication method. - A new test for authenticated socks traffic. - Credit to Nicolas Retrain for the initial patch. Thanks! --- scripts/base/protocols/socks/main.bro | 22 ++++++- src/analyzer/protocol/socks/SOCKS.cc | 3 +- src/analyzer/protocol/socks/events.bif | 16 +++++ .../protocol/socks/socks-analyzer.pac | 40 ++++++++++++ .../protocol/socks/socks-protocol.pac | 57 +++++++++++++++--- .../socks.log | 10 +++ .../tunnel.log | 10 +++ .../socks.log | 10 +-- .../socks.log | 10 +-- testing/btest/Traces/socks-auth.pcap | Bin 0 -> 1326 bytes .../base/protocols/socks/socks-auth.bro | 5 ++ 11 files changed, 162 insertions(+), 21 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/socks.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/tunnel.log create mode 100644 testing/btest/Traces/socks-auth.pcap create mode 100644 testing/btest/scripts/base/protocols/socks/socks-auth.bro diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro index 713161d442..f60c3ce41c 100644 --- a/scripts/base/protocols/socks/main.bro +++ b/scripts/base/protocols/socks/main.bro @@ -16,8 +16,10 @@ export { id: conn_id &log; ## Protocol version of SOCKS. version: count &log; - ## Username for the proxy if extracted from the network. + ## Username used to request a login to the proxy. user: string &log &optional; + ## Password used to request a login to the proxy. + password: string &log &optional; ## Server status for the attempt at using the proxy. status: string &log &optional; ## Client requested SOCKS address. Could be an address, a name @@ -91,3 +93,21 @@ event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Addres if ( "SOCKS" in c$service ) Log::write(SOCKS::LOG, c$socks); } + +event socks_login_userpass(c: connection, user: string, password: string) &priority=5 + { + # Authentication only possible with the version 5. + set_session(c, 5); + + c$socks$user = user; + c$socks$password = password; + } + +event socks_login_reply(c: connection, code: count) &priority=5 + { + # Authentication only possible with the version 5. + set_session(c, 5); + + c$socks$status = v5_status[code]; + } + diff --git a/src/analyzer/protocol/socks/SOCKS.cc b/src/analyzer/protocol/socks/SOCKS.cc index e678528f35..ec1e85653b 100644 --- a/src/analyzer/protocol/socks/SOCKS.cc +++ b/src/analyzer/protocol/socks/SOCKS.cc @@ -57,8 +57,7 @@ void SOCKS_Analyzer::DeliverStream(int len, const u_char* data, bool orig) // with the rest of the conneciton. // // Note that we assume that no payload data arrives before both endpoints - // are done with there part of the SOCKS protocol. - + // are done with their part of the SOCKS protocol. if ( ! pia ) { pia = new pia::PIA_TCP(Conn()); diff --git a/src/analyzer/protocol/socks/events.bif b/src/analyzer/protocol/socks/events.bif index 4f1f8ad1cd..ece69140a1 100644 --- a/src/analyzer/protocol/socks/events.bif +++ b/src/analyzer/protocol/socks/events.bif @@ -27,3 +27,19 @@ event socks_request%(c: connection, version: count, request_type: count, sa: SOC ## p: The destination port for the proxied traffic. event socks_reply%(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port%); +## Generated when a SOCKS client performs username and password based login. +## +## c: The parent connection of the proxy. +## +## user: The given username. +## +## password: The given password. +event socks_login_userpass%(c: connection, user: string, password: string%); + +## Generated when a SOCKS server replies to a login attempt. +## +## c: The parent connection of the proxy. +## +## code: The response code for the attempted login. +event socks_login_reply%(c: connection, code: count%); + diff --git a/src/analyzer/protocol/socks/socks-analyzer.pac b/src/analyzer/protocol/socks/socks-analyzer.pac index db98b3f4b3..7d634e2f46 100644 --- a/src/analyzer/protocol/socks/socks-analyzer.pac +++ b/src/analyzer/protocol/socks/socks-analyzer.pac @@ -148,6 +148,31 @@ refine connection SOCKS_Conn += { return true; %} + function socks5_auth_request_userpass(request: SOCKS5_Auth_Request_UserPass): bool + %{ + StringVal* user = new StringVal(${request.username}.length(), (const char*) ${request.username}.begin()); + StringVal* pass = new StringVal(${request.password}.length(), (const char*) ${request.password}.begin()); + + BifEvent::generate_socks_login_userpass(bro_analyzer(), + bro_analyzer()->Conn(), + user, pass); + return true; + %} + + function socks5_unsupported_authentication(auth_method: uint8): bool + %{ + reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_%d", auth_method)); + return true; + %} + + function socks5_auth_reply(reply: SOCKS5_Auth_Reply): bool + %{ + BifEvent::generate_socks_login_reply(bro_analyzer(), + bro_analyzer()->Conn(), + ${reply.code}); + return true; + %} + function version_error(version: uint8): bool %{ bro_analyzer()->ProtocolViolation(fmt("unsupported/unknown SOCKS version %d", version)); @@ -176,3 +201,18 @@ refine typeattr SOCKS5_Request += &let { refine typeattr SOCKS5_Reply += &let { proc: bool = $context.connection.socks5_reply(this); }; + +refine typeattr SOCKS5_Auth_Negotiation_Reply += &let { +}; + +refine typeattr SOCKS5_Auth_Request_UserPass += &let { + proc: bool = $context.connection.socks5_auth_request_userpass(this); +}; + +refine typeattr SOCKS5_Auth_Reply += &let { + proc: bool = $context.connection.socks5_auth_reply(this); +}; + +refine typeattr SOCKS5_Unsupported_Authentication += &let { + proc: bool = $context.connection.socks5_unsupported_authentication($context.connection.v5_auth_method()); +}; diff --git a/src/analyzer/protocol/socks/socks-protocol.pac b/src/analyzer/protocol/socks/socks-protocol.pac index 05ca4bc861..4e48ea0672 100644 --- a/src/analyzer/protocol/socks/socks-protocol.pac +++ b/src/analyzer/protocol/socks/socks-protocol.pac @@ -2,9 +2,10 @@ type SOCKS_Version(is_orig: bool) = record { version: uint8; msg: case version of { - 4 -> socks4_msg: SOCKS4_Message(is_orig); - 5 -> socks5_msg: SOCKS5_Message(is_orig); - default -> socks_msg_fail: SOCKS_Version_Error(version); + 1 -> socks5_auth_msg: SOCKS5_Auth_Message(is_orig); + 4 -> socks4_msg: SOCKS4_Message(is_orig); + 5 -> socks5_msg: SOCKS5_Message(is_orig); + default -> socks_msg_fail: SOCKS_Version_Error(version); }; }; @@ -14,10 +15,11 @@ type SOCKS_Version_Error(version: uint8) = record { # SOCKS5 Implementation type SOCKS5_Message(is_orig: bool) = case $context.connection.v5_past_authentication() of { - true -> msg: SOCKS5_Real_Message(is_orig); false -> auth: SOCKS5_Auth_Negotiation(is_orig); + true -> msg: SOCKS5_Real_Message(is_orig); }; + type SOCKS5_Auth_Negotiation(is_orig: bool) = case is_orig of { true -> req: SOCKS5_Auth_Negotiation_Request; false -> rep: SOCKS5_Auth_Negotiation_Reply; @@ -32,6 +34,32 @@ type SOCKS5_Auth_Negotiation_Reply = record { selected_auth_method: uint8; } &let { past_auth = $context.connection.set_v5_past_authentication(); + set_auth = $context.connection.set_v5_auth_method(selected_auth_method); +}; + +type SOCKS5_Auth_Message(is_orig: bool) = case is_orig of { + true -> req: SOCKS5_Auth_Request; + false -> rep: SOCKS5_Auth_Reply; +}; + +type SOCKS5_Auth_Request = case $context.connection.v5_auth_method() of { + 0x02 -> userpass : SOCKS5_Auth_Request_UserPass; + default -> unsupported : SOCKS5_Unsupported_Authentication; +}; + +type SOCKS5_Unsupported_Authentication = record { + crap: bytestring &restofdata; +}; + +type SOCKS5_Auth_Request_UserPass = record { + ulen : uint8; + username : bytestring &length=ulen; + plen : uint8; + password : bytestring &length=plen; +}; + +type SOCKS5_Auth_Reply = record { + code : uint8; }; type SOCKS5_Real_Message(is_orig: bool) = case is_orig of { @@ -55,10 +83,10 @@ type SOCKS5_Address = record { } &byteorder = bigendian; type SOCKS5_Request = record { - command: uint8; - reserved: uint8; - remote_name: SOCKS5_Address; - port: uint16; + command : uint8; + reserved : uint8; + remote_name : SOCKS5_Address; + port : uint16; } &byteorder = bigendian; type SOCKS5_Reply = record { @@ -99,10 +127,12 @@ type SOCKS4_Reply = record { refine connection SOCKS_Conn += { %member{ bool v5_authenticated_; + uint8 selected_auth_method_; %} %init{ v5_authenticated_ = false; + selected_auth_method_ = 255; %} function v5_past_authentication(): bool @@ -115,5 +145,16 @@ refine connection SOCKS_Conn += { v5_authenticated_ = true; return true; %} + + function set_v5_auth_method(method: uint8): bool + %{ + selected_auth_method_ = method; + return true; + %} + + function v5_auth_method(): uint8 + %{ + return selected_auth_method_; + %} }; diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/socks.log new file mode 100644 index 0000000000..cc5fa80191 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/socks.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path socks +#open 2015-02-05-16-13-12 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user password status request.host request.name request_p bound.host bound.name bound_p +#types time string addr port addr port count string string string addr string port addr string port +1368517392.724989 CXWv6p3arKYeMETxOg 192.168.0.2 55951 192.168.0.1 1080 5 bob alice succeeded 192.168.0.2 - 22 192.168.0.1 - 55951 +#close 2015-02-05-16-13-12 diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/tunnel.log b/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/tunnel.log new file mode 100644 index 0000000000..d53238df93 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/tunnel.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path tunnel +#open 2015-02-05-16-13-12 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action +#types time string addr port addr port enum enum +1368517392.728523 - 192.168.0.2 0 192.168.0.1 1080 Tunnel::SOCKS Tunnel::DISCOVER +#close 2015-02-05-16-13-12 diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log index 148e4adf02..f69df31b66 100644 --- a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log +++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path socks -#open 2013-08-26-19-04-20 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user status request.host request.name request_p bound.host bound.name bound_p -#types time string addr port addr port count string string addr string port addr string port -1340213015.276495 CjhGID4nQcgTWjvg4c 10.0.0.55 53994 60.190.189.214 8124 5 - succeeded - www.osnews.com 80 192.168.0.31 - 2688 -#close 2013-08-26-19-04-20 +#open 2015-02-05-17-39-14 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user password status request.host request.name request_p bound.host bound.name bound_p +#types time string addr port addr port count string string string addr string port addr string port +1340213015.276495 CjhGID4nQcgTWjvg4c 10.0.0.55 53994 60.190.189.214 8124 5 - - succeeded - www.osnews.com 80 192.168.0.31 - 2688 +#close 2015-02-05-17-39-14 diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log index d706a11da3..de7b26f875 100644 --- a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log +++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path socks -#open 2013-08-26-19-04-20 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user status request.host request.name request_p bound.host bound.name bound_p -#types time string addr port addr port count string string addr string port addr string port -1340113261.914619 CXWv6p3arKYeMETxOg 10.0.0.50 59580 85.194.84.197 1080 5 - succeeded - www.google.com 443 0.0.0.0 - 443 -#close 2013-08-26-19-04-20 +#open 2015-02-05-17-39-29 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user password status request.host request.name request_p bound.host bound.name bound_p +#types time string addr port addr port count string string string addr string port addr string port +1340113261.914619 CXWv6p3arKYeMETxOg 10.0.0.50 59580 85.194.84.197 1080 5 - - succeeded - www.google.com 443 0.0.0.0 - 443 +#close 2015-02-05-17-39-29 diff --git a/testing/btest/Traces/socks-auth.pcap b/testing/btest/Traces/socks-auth.pcap new file mode 100644 index 0000000000000000000000000000000000000000..1570e229473da081b50bd3906a3263503896de6a GIT binary patch literal 1326 zcmaKsPiWIn9LIkzFX_UfGfpY9D5wV$MAW)E5-sZxrA#(N@HTb2xuGEQ<~hefgf^Ff z?y`ypWlTXt@T54QNb%3Wfs`J-dGe;Xrv84frsSp7!mlJDeDZyt_xrss>(46+s1W^H zEdYi$=J(*@yAOSE9;2`oPSwl-kr!CAid?8JV@=wjO=e-!>&?Rx1mE3X;fWA*S{i& z13igDJP3bxhz&rZ#(^t@okwgesH`BDbqKrYBowqz!IEb><FdsoPSj;elW54rQ$+{kxuPd#Iyhx zzeOpVI}Dh_gj8KR3;whqrg{=lr+G;xGCp%>M!SCd*6bX?cbQV$q|{SF!HBFN!%%8P zNPQH+ZB$^1etNR^zfw;N;&4wQO8s=dLu?X}31)4d0TZ}q12|v+EW5)GH~I95uaJvKz{QEvA^*sZ@mQR#e>ZU7(~phT976yr03CO%@0dt zxfS9wD^3GQ*GmUBeoi2LnPztVZk%i>m^_S5gV#v@_b6=6iOl_p%*#lh^h{U(QH^}q qAvzXq>*gfGvW3xXUhLZ8h}b?~G93G0)x%voJ}wcNhPU)?z4Ql$s7wg} literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/protocols/socks/socks-auth.bro b/testing/btest/scripts/base/protocols/socks/socks-auth.bro new file mode 100644 index 0000000000..2123dc1d45 --- /dev/null +++ b/testing/btest/scripts/base/protocols/socks/socks-auth.bro @@ -0,0 +1,5 @@ +# @TEST-EXEC: bro -r $TRACES/socks-auth.pcap %INPUT +# @TEST-EXEC: btest-diff socks.log +# @TEST-EXEC: btest-diff tunnel.log + +@load base/protocols/socks From 530c3c0c6b19fe0e708a076a79567f9fa8334216 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 8 Feb 2015 18:20:38 -0800 Subject: [PATCH 06/37] Changing load order for plugin scripts. This can be need if they depends on each other. --- src/plugin/Manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 2ca34d94f3..ab0b85676b 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -172,7 +172,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ // Load {bif,scripts}/__load__.bro automatically. - string init = dir + "scripts/__load__.bro"; + string init = dir + "lib/bif/__load__.bro"; if ( is_file(init) ) { @@ -180,7 +180,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ scripts_to_load.push_back(init); } - init = dir + "lib/bif/__load__.bro"; + init = dir + "scripts/__load__.bro"; if ( is_file(init) ) { From 23b9705a7bbd6333b767b8908be2ddcf7017b5a5 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 8 Feb 2015 18:21:23 -0800 Subject: [PATCH 07/37] Fixing analyzer tag types for some Files::* functions. --- CHANGES | 6 ++++++ VERSION | 2 +- scripts/base/frameworks/files/main.bro | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/CHANGES b/CHANGES index d1031765cc..3367c878cd 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ +2.3-413 | 2015-02-08 18:23:05 -0800 + + * Fixing analyzer tag types for some Files::* functions. (Robin Sommer) + + * Changing load order for plugin scripts. (Robin Sommer) + 2.3-411 | 2015-02-05 10:05:48 -0600 * Fix file analysis of files with total size below the bof_buffer size diff --git a/VERSION b/VERSION index defa33cc31..fca56f2eeb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-411 +2.3-413 diff --git a/scripts/base/frameworks/files/main.bro b/scripts/base/frameworks/files/main.bro index e335d4be9d..94a46578c0 100644 --- a/scripts/base/frameworks/files/main.bro +++ b/scripts/base/frameworks/files/main.bro @@ -267,7 +267,7 @@ export { ## mts: The set of MIME types, each in the form "foo/bar" (case-insensitive). ## ## Returns: True if the MIME types were successfully registered. - global register_for_mime_types: function(tag: Analyzer::Tag, mts: set[string]) : bool; + global register_for_mime_types: function(tag: Files::Tag, mts: set[string]) : bool; ## Registers a MIME type for an analyzer. If a future file with this type is seen, ## the analyzer will be automatically assigned to parsing it. The function *adds* @@ -278,20 +278,20 @@ export { ## mt: The MIME type in the form "foo/bar" (case-insensitive). ## ## Returns: True if the MIME type was successfully registered. - global register_for_mime_type: function(tag: Analyzer::Tag, mt: string) : bool; + global register_for_mime_type: function(tag: Files::Tag, mt: string) : bool; ## Returns a set of all MIME types currently registered for a specific analyzer. ## ## tag: The tag of the analyzer. ## ## Returns: The set of MIME types. - global registered_mime_types: function(tag: Analyzer::Tag) : set[string]; + global registered_mime_types: function(tag: Files::Tag) : set[string]; ## Returns a table of all MIME-type-to-analyzer mappings currently registered. ## ## Returns: A table mapping each analyzer to the set of MIME types ## registered for it. - global all_registered_mime_types: function() : table[Analyzer::Tag] of set[string]; + global all_registered_mime_types: function() : table[Files::Tag] of set[string]; ## Event that can be handled to access the Info record as it is sent on ## to the logging framework. @@ -306,8 +306,8 @@ redef record fa_file += { global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table(); # Store the MIME type to analyzer mappings. -global mime_types: table[Analyzer::Tag] of set[string]; -global mime_type_to_analyzers: table[string] of set[Analyzer::Tag]; +global mime_types: table[Files::Tag] of set[string]; +global mime_type_to_analyzers: table[string] of set[Files::Tag]; global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table(); @@ -401,7 +401,7 @@ function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool return result; } -function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : bool +function register_for_mime_types(tag: Files::Tag, mime_types: set[string]) : bool { local rc = T; @@ -414,7 +414,7 @@ function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : return rc; } -function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool +function register_for_mime_type(tag: Files::Tag, mt: string) : bool { if ( tag !in mime_types ) { @@ -431,12 +431,12 @@ function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool return T; } -function registered_mime_types(tag: Analyzer::Tag) : set[string] +function registered_mime_types(tag: Files::Tag) : set[string] { return tag in mime_types ? mime_types[tag] : set(); } -function all_registered_mime_types(): table[Analyzer::Tag] of set[string] +function all_registered_mime_types(): table[Files::Tag] of set[string] { return mime_types; } @@ -451,7 +451,7 @@ function describe(f: fa_file): string return handler$describe(f); } -event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5 +event get_file_handle(tag: Files::Tag, c: connection, is_orig: bool) &priority=5 { if ( tag !in registered_protocols ) return; From 5f0a27ca31443ee3c308e49ff5b6e6b1c2fec963 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 9 Feb 2015 12:10:49 -0800 Subject: [PATCH 08/37] Submodule update - newest sqlite version --- src/3rdparty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/3rdparty b/src/3rdparty index 7e15efe9d2..f2e34d731e 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 7e15efe9d28d46bfa662fcdd1cbb15ce1db285c9 +Subproject commit f2e34d731ed29bb993fbb065846faa342a8c824f From 88af106b6b5de8497499e1f6be9d317b6fbb4707 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 11 Feb 2015 13:56:34 -0600 Subject: [PATCH 09/37] Fix use of deprecated gperftools headers. As of gperftools 2.0 (Feb. 2012), they've been renamed in to gperftools/ instead of google/, and as of gperftools 2.2, including the later emits deprecation warnings. --- src/util.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util.h b/src/util.h index db77888c16..50c33d5608 100644 --- a/src/util.h +++ b/src/util.h @@ -48,8 +48,8 @@ #endif #ifdef USE_PERFTOOLS_DEBUG -#include -#include +#include +#include extern HeapLeakChecker* heap_checker; #endif From 961fd06cad004f1f167ebbf65f241349a2ea9b63 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 12 Feb 2015 17:06:38 -0600 Subject: [PATCH 10/37] Refactor SOCKS5 user/pass authentication support. - Rename event "socks_login_userpass" to "socks_login_userpass_request" - Rename event "socks_login_reply" to "socks_login_userpass_reply" - Split unsupported authN weird into 2 types: method vs. version Addresses BIT-1011 --- scripts/base/protocols/socks/main.bro | 4 +- src/analyzer/protocol/socks/events.bif | 6 +-- .../protocol/socks/socks-analyzer.pac | 40 ++++++++------ .../protocol/socks/socks-protocol.pac | 53 +++++++++++++++++-- src/analyzer/protocol/socks/socks.pac | 2 +- testing/btest/Baseline/plugins.writer/output | 4 +- 6 files changed, 82 insertions(+), 27 deletions(-) diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro index f60c3ce41c..e052962888 100644 --- a/scripts/base/protocols/socks/main.bro +++ b/scripts/base/protocols/socks/main.bro @@ -94,7 +94,7 @@ event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Addres Log::write(SOCKS::LOG, c$socks); } -event socks_login_userpass(c: connection, user: string, password: string) &priority=5 +event socks_login_userpass_request(c: connection, user: string, password: string) &priority=5 { # Authentication only possible with the version 5. set_session(c, 5); @@ -103,7 +103,7 @@ event socks_login_userpass(c: connection, user: string, password: string) &prior c$socks$password = password; } -event socks_login_reply(c: connection, code: count) &priority=5 +event socks_login_userpass_reply(c: connection, code: count) &priority=5 { # Authentication only possible with the version 5. set_session(c, 5); diff --git a/src/analyzer/protocol/socks/events.bif b/src/analyzer/protocol/socks/events.bif index ece69140a1..224f570817 100644 --- a/src/analyzer/protocol/socks/events.bif +++ b/src/analyzer/protocol/socks/events.bif @@ -34,12 +34,12 @@ event socks_reply%(c: connection, version: count, reply: count, sa: SOCKS::Addre ## user: The given username. ## ## password: The given password. -event socks_login_userpass%(c: connection, user: string, password: string%); +event socks_login_userpass_request%(c: connection, user: string, password: string%); -## Generated when a SOCKS server replies to a login attempt. +## Generated when a SOCKS server replies to a username/password login attempt. ## ## c: The parent connection of the proxy. ## ## code: The response code for the attempted login. -event socks_login_reply%(c: connection, code: count%); +event socks_login_userpass_reply%(c: connection, code: count%); diff --git a/src/analyzer/protocol/socks/socks-analyzer.pac b/src/analyzer/protocol/socks/socks-analyzer.pac index 7d634e2f46..b8c4165a54 100644 --- a/src/analyzer/protocol/socks/socks-analyzer.pac +++ b/src/analyzer/protocol/socks/socks-analyzer.pac @@ -148,28 +148,34 @@ refine connection SOCKS_Conn += { return true; %} - function socks5_auth_request_userpass(request: SOCKS5_Auth_Request_UserPass): bool + function socks5_auth_request_userpass(request: SOCKS5_Auth_Request_UserPass_v1): bool %{ StringVal* user = new StringVal(${request.username}.length(), (const char*) ${request.username}.begin()); StringVal* pass = new StringVal(${request.password}.length(), (const char*) ${request.password}.begin()); - BifEvent::generate_socks_login_userpass(bro_analyzer(), - bro_analyzer()->Conn(), - user, pass); + BifEvent::generate_socks_login_userpass_request(bro_analyzer(), + bro_analyzer()->Conn(), + user, pass); return true; %} - function socks5_unsupported_authentication(auth_method: uint8): bool + function socks5_unsupported_authentication_method(auth_method: uint8): bool %{ - reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_%d", auth_method)); + reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_method_%d", auth_method)); + return true; + %} + + function socks5_unsupported_authentication_version(auth_method: uint8, version: uint8): bool + %{ + reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_%d_%d", auth_method, version)); return true; %} - function socks5_auth_reply(reply: SOCKS5_Auth_Reply): bool + function socks5_auth_reply_userpass(reply: SOCKS5_Auth_Reply_UserPass_v1): bool %{ - BifEvent::generate_socks_login_reply(bro_analyzer(), - bro_analyzer()->Conn(), - ${reply.code}); + BifEvent::generate_socks_login_userpass_reply(bro_analyzer(), + bro_analyzer()->Conn(), + ${reply.code}); return true; %} @@ -205,14 +211,18 @@ refine typeattr SOCKS5_Reply += &let { refine typeattr SOCKS5_Auth_Negotiation_Reply += &let { }; -refine typeattr SOCKS5_Auth_Request_UserPass += &let { +refine typeattr SOCKS5_Auth_Request_UserPass_v1 += &let { proc: bool = $context.connection.socks5_auth_request_userpass(this); }; -refine typeattr SOCKS5_Auth_Reply += &let { - proc: bool = $context.connection.socks5_auth_reply(this); +refine typeattr SOCKS5_Auth_Reply_UserPass_v1 += &let { + proc: bool = $context.connection.socks5_auth_reply_userpass(this); }; -refine typeattr SOCKS5_Unsupported_Authentication += &let { - proc: bool = $context.connection.socks5_unsupported_authentication($context.connection.v5_auth_method()); +refine typeattr SOCKS5_Unsupported_Authentication_Method += &let { + proc: bool = $context.connection.socks5_unsupported_authentication_method($context.connection.v5_auth_method()); +}; + +refine typeattr SOCKS5_Unsupported_Authentication_Version += &let { + proc: bool = $context.connection.socks5_unsupported_authentication_version($context.connection.v5_auth_method(), version); }; diff --git a/src/analyzer/protocol/socks/socks-protocol.pac b/src/analyzer/protocol/socks/socks-protocol.pac index 4e48ea0672..d9c31d2377 100644 --- a/src/analyzer/protocol/socks/socks-protocol.pac +++ b/src/analyzer/protocol/socks/socks-protocol.pac @@ -1,8 +1,12 @@ +type SOCKS_Message(is_orig: bool) = case $context.connection.v5_in_auth_sub_negotiation() of { + true -> auth: SOCKS5_Auth_Message(is_orig); + false -> msg: SOCKS_Version(is_orig); +}; + type SOCKS_Version(is_orig: bool) = record { version: uint8; msg: case version of { - 1 -> socks5_auth_msg: SOCKS5_Auth_Message(is_orig); 4 -> socks4_msg: SOCKS4_Message(is_orig); 5 -> socks5_msg: SOCKS5_Message(is_orig); default -> socks_msg_fail: SOCKS_Version_Error(version); @@ -33,6 +37,7 @@ type SOCKS5_Auth_Negotiation_Request = record { type SOCKS5_Auth_Negotiation_Reply = record { selected_auth_method: uint8; } &let { + in_auth_sub_neg = $context.connection.set_v5_in_auth_sub_negotiation(selected_auth_method == 0 || selected_auth_method == 0xff ? false : true); past_auth = $context.connection.set_v5_past_authentication(); set_auth = $context.connection.set_v5_auth_method(selected_auth_method); }; @@ -44,21 +49,48 @@ type SOCKS5_Auth_Message(is_orig: bool) = case is_orig of { type SOCKS5_Auth_Request = case $context.connection.v5_auth_method() of { 0x02 -> userpass : SOCKS5_Auth_Request_UserPass; - default -> unsupported : SOCKS5_Unsupported_Authentication; + default -> unsupported : SOCKS5_Unsupported_Authentication_Method; }; -type SOCKS5_Unsupported_Authentication = record { +type SOCKS5_Unsupported_Authentication_Method = record { + crap: bytestring &restofdata; +}; + +type SOCKS5_Unsupported_Authentication_Version(version: uint8) = record { crap: bytestring &restofdata; }; type SOCKS5_Auth_Request_UserPass = record { + version: uint8; + msg: case version of { + 1 -> v1: SOCKS5_Auth_Request_UserPass_v1; + default -> unsupported: SOCKS5_Unsupported_Authentication_Version(version); + }; +}; + +type SOCKS5_Auth_Request_UserPass_v1 = record { ulen : uint8; username : bytestring &length=ulen; plen : uint8; password : bytestring &length=plen; }; -type SOCKS5_Auth_Reply = record { +type SOCKS5_Auth_Reply = case $context.connection.v5_auth_method() of { + 0x02 -> userpass : SOCKS5_Auth_Reply_UserPass; + default -> unsupported : SOCKS5_Unsupported_Authentication_Method; +} &let { + in_auth_sub_neg = $context.connection.set_v5_in_auth_sub_negotiation(false); +}; + +type SOCKS5_Auth_Reply_UserPass = record { + version: uint8; + msg: case version of { + 1 -> v1: SOCKS5_Auth_Reply_UserPass_v1; + default -> unsupported: SOCKS5_Unsupported_Authentication_Version(version); + }; +}; + +type SOCKS5_Auth_Reply_UserPass_v1 = record { code : uint8; }; @@ -126,15 +158,28 @@ type SOCKS4_Reply = record { refine connection SOCKS_Conn += { %member{ + bool v5_in_auth_sub_negotiation_; bool v5_authenticated_; uint8 selected_auth_method_; %} %init{ + v5_in_auth_sub_negotiation_ = false; v5_authenticated_ = false; selected_auth_method_ = 255; %} + function v5_in_auth_sub_negotiation(): bool + %{ + return v5_in_auth_sub_negotiation_; + %} + + function set_v5_in_auth_sub_negotiation(b: bool): bool + %{ + v5_in_auth_sub_negotiation_ = b; + return true; + %} + function v5_past_authentication(): bool %{ return v5_authenticated_; diff --git a/src/analyzer/protocol/socks/socks.pac b/src/analyzer/protocol/socks/socks.pac index a9c4099508..9aed2820af 100644 --- a/src/analyzer/protocol/socks/socks.pac +++ b/src/analyzer/protocol/socks/socks.pac @@ -20,7 +20,7 @@ connection SOCKS_Conn(bro_analyzer: BroAnalyzer) { %include socks-protocol.pac flow SOCKS_Flow(is_orig: bool) { - datagram = SOCKS_Version(is_orig) withcontext(connection, this); + datagram = SOCKS_Message(is_orig) withcontext(connection, this); }; %include socks-analyzer.pac diff --git a/testing/btest/Baseline/plugins.writer/output b/testing/btest/Baseline/plugins.writer/output index 0882718f03..f7b33992ea 100644 --- a/testing/btest/Baseline/plugins.writer/output +++ b/testing/btest/Baseline/plugins.writer/output @@ -17,6 +17,6 @@ Demo::Foo - A Foo test logging writer (dynamic, version 1.0) [http] 1340213020.732963|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|GET|www.osnews.com|/images/icons/17.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- [http] 1340213021.300269|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|6|GET|www.osnews.com|/images/left.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- [http] 1340213021.861584|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|7|GET|www.osnews.com|/images/icons/32.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- -[packet_filter] 1412721099.419280|bro|ip or not ip|T|T -[socks] 1340213015.276495|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|-|succeeded|-|www.osnews.com|80|192.168.0.31|-|2688 +[packet_filter] 1423781675.402129|bro|ip or not ip|T|T +[socks] 1340213015.276495|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|-|-|succeeded|-|www.osnews.com|80|192.168.0.31|-|2688 [tunnel] 1340213015.276495|-|10.0.0.55|0|60.190.189.214|8124|Tunnel::SOCKS|Tunnel::DISCOVER From 062baefde09483277bbc94574cfc45793779f98e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Feb 2015 11:24:32 -0600 Subject: [PATCH 11/37] Add 'while' statement to Bro language. --- src/SerialTypes.h | 1 + src/Stmt.cc | 122 +++++++++++++++++++++- src/Stmt.h | 29 +++++ src/StmtEnums.h | 1 + src/parse.y | 6 ++ src/scan.l | 1 + testing/btest/Baseline/language.while/out | 12 +++ testing/btest/core/leaks/while.bro | 80 ++++++++++++++ testing/btest/language/while.bro | 77 ++++++++++++++ 9 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/language.while/out create mode 100644 testing/btest/core/leaks/while.bro create mode 100644 testing/btest/language/while.bro diff --git a/src/SerialTypes.h b/src/SerialTypes.h index d2f227838c..e50ec3889f 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -181,6 +181,7 @@ SERIAL_STMT(INIT_STMT, 17) SERIAL_STMT(NULL_STMT, 18) SERIAL_STMT(WHEN_STMT, 19) SERIAL_STMT(FALLTHROUGH_STMT, 20) +SERIAL_STMT(WHILE_STMT, 21) #define SERIAL_TYPE(name, val) SERIAL_CONST(name, val, BRO_TYPE) SERIAL_TYPE(BRO_TYPE, 1) diff --git a/src/Stmt.cc b/src/Stmt.cc index cb716b3f15..d2f8c48cee 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -23,7 +23,7 @@ const char* stmt_name(BroStmtTag t) "print", "event", "expr", "if", "when", "switch", "for", "next", "break", "return", "add", "delete", "list", "bodylist", - "", "fallthrough", + "", "fallthrough", "while", "null", }; @@ -1127,6 +1127,126 @@ bool EventStmt::DoUnserialize(UnserialInfo* info) return event_expr != 0; } +WhileStmt::WhileStmt(Expr* arg_loop_condition, Stmt* arg_body) + : loop_condition(arg_loop_condition), body(arg_body) + { + if ( ! loop_condition->IsError() && + ! IsBool(loop_condition->Type()->Tag()) ) + loop_condition->Error("while conditional must be boolean"); + } + +WhileStmt::~WhileStmt() + { + Unref(loop_condition); + Unref(body); + } + +int WhileStmt::IsPure() const + { + return loop_condition->IsPure() && body->IsPure(); + } + +void WhileStmt::Describe(ODesc* d) const + { + Stmt::Describe(d); + + if ( d->IsReadable() ) + d->Add("("); + + loop_condition->Describe(d); + + if ( d->IsReadable() ) + d->Add(")"); + + d->SP(); + d->PushIndent(); + body->AccessStats(d); + body->Describe(d); + d->PopIndent(); + } + +TraversalCode WhileStmt::Traverse(TraversalCallback* cb) const + { + TraversalCode tc = cb->PreStmt(this); + HANDLE_TC_STMT_PRE(tc); + + tc = loop_condition->Traverse(cb); + HANDLE_TC_STMT_PRE(tc); + + tc = body->Traverse(cb); + HANDLE_TC_STMT_PRE(tc); + + tc = cb->PostStmt(this); + HANDLE_TC_STMT_POST(tc); + } + +Val* WhileStmt::Exec(Frame* f, stmt_flow_type& flow) const + { + RegisterAccess(); + flow = FLOW_NEXT; + Val* rval = 0; + + for ( ; ; ) + { + Val* cond = loop_condition->Eval(f); + + if ( ! cond ) + break; + + bool cont = cond->AsBool(); + Unref(cond); + + if ( ! cont ) + break; + + flow = FLOW_NEXT; + rval = body->Exec(f, flow); + + if ( flow == FLOW_BREAK || flow == FLOW_RETURN ) + break; + } + + if ( flow == FLOW_LOOP || flow == FLOW_BREAK ) + flow = FLOW_NEXT; + + return rval; + } + +Stmt* WhileStmt::Simplify() + { + loop_condition = simplify_expr(loop_condition, SIMPLIFY_GENERAL); + + if ( loop_condition->IsConst() && loop_condition->IsZero() ) + return new NullStmt(); + + body = simplify_stmt(body); + return this; + } + +IMPLEMENT_SERIAL(WhileStmt, SER_WHILE_STMT); + +bool WhileStmt::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_WHILE_STMT, Stmt); + + if ( ! loop_condition->Serialize(info) ) + return false; + + return body->Serialize(info); + } + +bool WhileStmt::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(Stmt); + loop_condition = Expr::Unserialize(info); + + if ( ! loop_condition ) + return false; + + body = Stmt::Unserialize(info); + return body != 0; + } + ForStmt::ForStmt(id_list* arg_loop_vars, Expr* loop_expr) : ExprStmt(STMT_FOR, loop_expr) { diff --git a/src/Stmt.h b/src/Stmt.h index 32b90b4190..79406fd51b 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -310,6 +310,35 @@ protected: EventExpr* event_expr; }; +class WhileStmt : public Stmt { +public: + + WhileStmt(Expr* loop_condition, Stmt* body); + + ~WhileStmt(); + + int IsPure() const; + + void Describe(ODesc* d) const; + + TraversalCode Traverse(TraversalCallback* cb) const; + +protected: + friend class Stmt; + + DECLARE_SERIAL(WhileStmt); + + WhileStmt() + { loop_condition = 0; body = 0; } + + Val* Exec(Frame* f, stmt_flow_type& flow) const; + + Stmt* Simplify(); + + Expr* loop_condition; + Stmt* body; +}; + class ForStmt : public ExprStmt { public: ForStmt(id_list* loop_vars, Expr* loop_expr); diff --git a/src/StmtEnums.h b/src/StmtEnums.h index d34f642594..ad99c2365a 100644 --- a/src/StmtEnums.h +++ b/src/StmtEnums.h @@ -17,6 +17,7 @@ typedef enum { STMT_LIST, STMT_EVENT_BODY_LIST, STMT_INIT, STMT_FALLTHROUGH, + STMT_WHILE, STMT_NULL #define NUM_STMTS (int(STMT_NULL) + 1) } BroStmtTag; diff --git a/src/parse.y b/src/parse.y index f74880dc13..8054718d45 100644 --- a/src/parse.y +++ b/src/parse.y @@ -16,6 +16,7 @@ %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET %token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE %token TOK_TIME TOK_TIMEOUT TOK_TIMER TOK_TYPE TOK_UNION TOK_VECTOR TOK_WHEN +%token TOK_WHILE %token TOK_ATTR_ADD_FUNC TOK_ATTR_ENCRYPT TOK_ATTR_DEFAULT %token TOK_ATTR_OPTIONAL TOK_ATTR_REDEF TOK_ATTR_ROTATE_INTERVAL @@ -1340,6 +1341,11 @@ stmt: $1->AsForStmt()->AddBody($2); } + | TOK_WHILE '(' expr ')' stmt + { + $$ = new WhileStmt($3, $5); + } + | TOK_NEXT ';' opt_no_test { set_location(@1, @2); diff --git a/src/scan.l b/src/scan.l index ae11382fb3..b13215e4b8 100644 --- a/src/scan.l +++ b/src/scan.l @@ -221,6 +221,7 @@ export return TOK_EXPORT; fallthrough return TOK_FALLTHROUGH; file return TOK_FILE; for return TOK_FOR; +while return TOK_WHILE; function return TOK_FUNCTION; global return TOK_GLOBAL; "?$" return TOK_HAS_FIELD; diff --git a/testing/btest/Baseline/language.while/out b/testing/btest/Baseline/language.while/out new file mode 100644 index 0000000000..d37792c0b4 --- /dev/null +++ b/testing/btest/Baseline/language.while/out @@ -0,0 +1,12 @@ +10 +s +ss +sss +{ +7, +1, +9, +5, +3 +} +[number 0, number 1, number 2, number 3, number 4, number 5, number 6, number 7, number 8, number 9, number 10, number 11, number 12] diff --git a/testing/btest/core/leaks/while.bro b/testing/btest/core/leaks/while.bro new file mode 100644 index 0000000000..eac6f2622e --- /dev/null +++ b/testing/btest/core/leaks/while.bro @@ -0,0 +1,80 @@ +# @TEST-GROUP: leaks +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: btest-bg-wait 30 + +function test_noop() + { + while ( F ) + print "noooooooooo"; + } + +function test_it() + { + local i = 0; + + while ( i < 10 ) + ++i; + + print i; + } + +function test_break() + { + local s = ""; + + while ( T ) + { + s += "s"; + print s; + + if ( s == "sss" ) + break; + } + } + +function test_next() + { + local s: set[count]; + local i = 0; + + while ( 9 !in s ) + { + ++i; + + if ( i % 2 == 0 ) + next; + + add s[i]; + } + + print s; + } + +function test_return(): vector of string + { + local i = 0; + local rval: vector of string; + + while ( T ) + { + rval[i] = fmt("number %d", i); + ++i; + + if ( i == 13 ) + return rval; + } + + rval[0] = "noooo"; + return rval; + } + +event new_connection(c: connection) + { + test_noop(); + test_it(); + test_break(); + test_next(); + print test_return(); + } diff --git a/testing/btest/language/while.bro b/testing/btest/language/while.bro new file mode 100644 index 0000000000..6828b00b41 --- /dev/null +++ b/testing/btest/language/while.bro @@ -0,0 +1,77 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +function test_noop() + { + while ( F ) + print "noooooooooo"; + } + +function test_it() + { + local i = 0; + + while ( i < 10 ) + ++i; + + print i; + } + +function test_break() + { + local s = ""; + + while ( T ) + { + s += "s"; + print s; + + if ( s == "sss" ) + break; + } + } + +function test_next() + { + local s: set[count]; + local i = 0; + + while ( 9 !in s ) + { + ++i; + + if ( i % 2 == 0 ) + next; + + add s[i]; + } + + print s; + } + +function test_return(): vector of string + { + local i = 0; + local rval: vector of string; + + while ( T ) + { + rval[i] = fmt("number %d", i); + ++i; + + if ( i == 13 ) + return rval; + } + + rval[0] = "noooo"; + return rval; + } + +event bro_init() + { + test_noop(); + test_it(); + test_break(); + test_next(); + print test_return(); + } From 8e4f4b46f7591a3779adc8e21d688f81ee436721 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Feb 2015 16:23:43 -0600 Subject: [PATCH 12/37] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index d43cc790e5..9b6dd56242 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit d43cc790e5b8709b5e032e52ad0e00936494739b +Subproject commit 9b6dd5624254de9d18618562887979da1158da43 From 4bcb9d2d920862660feba80cfb13356952843201 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Feb 2015 18:04:17 -0600 Subject: [PATCH 13/37] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index 9b6dd56242..420c5b42c0 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 9b6dd5624254de9d18618562887979da1158da43 +Subproject commit 420c5b42c0c90f22fc7a862fc491c8e554d05381 From b00bd7702f8962bcf8507adb0abe967c4c02426c Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 13 Feb 2015 22:02:54 -0500 Subject: [PATCH 14/37] Add the ability to remove surrounding braces from the JSON formatter. --- src/threading/formatters/JSON.cc | 13 ++++++++++--- src/threading/formatters/JSON.h | 3 +++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index 472023e0f8..e1a5713461 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -15,7 +15,7 @@ using namespace threading::formatter; -JSON::JSON(MsgThread* t, TimeFormat tf) : Formatter(t) +JSON::JSON(MsgThread* t, TimeFormat tf) : Formatter(t), surrounding_braces(true) { timestamps = tf; } @@ -27,7 +27,8 @@ JSON::~JSON() bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields, Value** vals) const { - desc->AddRaw("{"); + if ( surrounding_braces ) + desc->AddRaw("{"); for ( int i = 0; i < num_fields; i++ ) { @@ -41,7 +42,8 @@ bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields, return false; } - desc->AddRaw("}"); + if ( surrounding_braces ) + desc->AddRaw("}"); return true; } @@ -217,3 +219,8 @@ threading::Value* JSON::ParseValue(const string& s, const string& name, TypeTag GetThread()->Error("JSON formatter does not support parsing yet."); return NULL; } + +void JSON::SurroundingBraces(bool use_braces) + { + surrounding_braces = use_braces; + } diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h index d7859f83fb..04209fbde9 100644 --- a/src/threading/formatters/JSON.h +++ b/src/threading/formatters/JSON.h @@ -27,8 +27,11 @@ public: threading::Value** vals) const; virtual threading::Value* ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; + void SurroundingBraces(bool use_braces); + private: TimeFormat timestamps; + bool surrounding_braces; }; }} From 2e043c0ff6e31f4dfaef81ddcddb1fe596fd0850 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 16 Feb 2015 12:11:59 -0800 Subject: [PATCH 15/37] Creating the installation directory for plugins at install time. --- src/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 13c6e45006..f13a4a6ebf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -408,6 +408,9 @@ add_dependencies(bro bif_loader_plugins) # Install *.bif.bro. install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/base/bif DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base) +# Create plugin directory at install time. +install(DIRECTORY DESTINATION ${BRO_PLUGIN_INSTALL_PATH}) + # Make clean removes the bif directory. set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_BINARY_DIR}/scripts/base/bif) From ff16f6215ad6f3cf8208b4e35756b75410a38403 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 16 Feb 2015 12:49:54 -0800 Subject: [PATCH 16/37] Removing setting installation plugin path from bro-path-dev.sh . Also, adding to existing BRO_PLUGIN_PATH rather than replacing. Addresses #1312 --- CMakeLists.txt | 4 ++-- aux/bro-aux | 2 +- cmake | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c0ff6c09d4..04ac197f74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,12 +31,12 @@ configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n" - "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n" + "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":${BRO_PLUGIN_PATH}\n" "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n") file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n" - "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n" + "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":${BRO_PLUGIN_PATH}\n" "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n") file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) diff --git a/aux/bro-aux b/aux/bro-aux index 3714d3594c..c011f3a724 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 3714d3594ce0d2b8a757c04e6e7d901d6b559915 +Subproject commit c011f3a7243a8a1dc8be7eff2f45799be7ee85f4 diff --git a/cmake b/cmake index 1316c07f70..0147a2e05b 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 1316c07f7059647b6c4a496ea36e4b83bb5d8f0f +Subproject commit 0147a2e05b613a044ac30374874acdb8bc216feb From d36422fde156caf7d4f2cddf3883d1812f3fb7e7 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 16 Feb 2015 13:37:59 -0800 Subject: [PATCH 17/37] Explicitly removing some old scripts on install. Some scripts have moved into plugins, but may cause confusion if they stick around from old installations. Explicitl removing them on install. We had this problem before in other cases, and it should be ok to help people upgrading a bit here, even though hardcoding these isn't great. --- src/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f13a4a6ebf..6d24172b97 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -414,3 +414,12 @@ install(DIRECTORY DESTINATION ${BRO_PLUGIN_INSTALL_PATH}) # Make clean removes the bif directory. set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_BINARY_DIR}/scripts/base/bif) +# Remove some stale files and scripts that previous Bro versions put in +# place, yet make confuse us now. This makes upgrading easier. +install(CODE " + file(REMOVE_RECURSE + ${BRO_SCRIPT_INSTALL_PATH}/base/frameworks/logging/writers/dataseries.bro + ${BRO_SCRIPT_INSTALL_PATH}/base/frameworks/logging/writers/elasticsearch.bro + ${BRO_SCRIPT_INSTALL_PATH}/policy/tuning/logs-to-elasticsearch.bro + ) +") From ab3cdf494a216cd20d528f982af3834ff4623695 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 16 Feb 2015 13:40:21 -0800 Subject: [PATCH 18/37] Updating submodules. --- aux/bro-aux | 2 +- cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/bro-aux b/aux/bro-aux index c011f3a724..63675de3cc 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit c011f3a7243a8a1dc8be7eff2f45799be7ee85f4 +Subproject commit 63675de3cc7bc3eb2a3645860224c372d3f7f36a diff --git a/cmake b/cmake index 0147a2e05b..9623367210 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 0147a2e05b613a044ac30374874acdb8bc216feb +Subproject commit 962336721040fdf55a6b264f8bbc84153b54d9a5 From 0f96d0625273f748793988f67756b4317c1e074e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 16 Feb 2015 14:24:32 -0800 Subject: [PATCH 19/37] Making plugin names case-insensitive for some internal comparisions. Makes the plugin system a bit more tolerant against spelling inconsistencies that would be hard to catch otherwise. --- src/plugin/Manager.cc | 22 ++++++++++++---------- src/util.cc | 7 +++++++ src/util.h | 3 +++ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index ab0b85676b..b891a0faab 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -79,18 +79,19 @@ void Manager::SearchDynamicPlugins(const std::string& dir) std::string name; std::getline(in, name); strstrip(name); + string lower_name = strtolower(name); if ( name.empty() ) reporter->FatalError("empty plugin magic file %s", magic.c_str()); - if ( dynamic_plugins.find(name) != dynamic_plugins.end() ) + if ( dynamic_plugins.find(lower_name) != dynamic_plugins.end() ) { DBG_LOG(DBG_PLUGINS, "Found already known plugin %s in %s, ignoring", name.c_str(), dir.c_str()); return; } // Record it, so that we can later activate it. - dynamic_plugins.insert(std::make_pair(name, dir)); + dynamic_plugins.insert(std::make_pair(lower_name, dir)); DBG_LOG(DBG_PLUGINS, "Found plugin %s in %s", name.c_str(), dir.c_str()); return; @@ -135,7 +136,7 @@ void Manager::SearchDynamicPlugins(const std::string& dir) bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_not_found) { - dynamic_plugin_map::iterator m = dynamic_plugins.find(name); + dynamic_plugin_map::iterator m = dynamic_plugins.find(strtolower(name)); if ( m == dynamic_plugins.end() ) { @@ -230,7 +231,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ // Make sure the name the plugin reports is consistent with // what we expect from its magic file. - if ( string(current_plugin->Name()) != name ) + if ( strtolower(current_plugin->Name()) != strtolower(name) ) reporter->FatalError("inconsistent plugin name: %s vs %s", current_plugin->Name().c_str(), name.c_str()); @@ -297,7 +298,7 @@ void Manager::UpdateInputFiles() static bool plugin_cmp(const Plugin* a, const Plugin* b) { - return a->Name() < b->Name(); + return strtolower(a->Name()) < strtolower(b->Name()); } void Manager::RegisterPlugin(Plugin *plugin) @@ -318,10 +319,11 @@ void Manager::RegisterBifFile(const char* plugin, bif_init_func c) { bif_init_func_map* bifs = BifFilesInternal(); - bif_init_func_map::iterator i = bifs->find(plugin); + std::string lower_plugin = strtolower(plugin); + bif_init_func_map::iterator i = bifs->find(lower_plugin); if ( i == bifs->end() ) - i = bifs->insert(std::make_pair(std::string(plugin), new bif_init_func_list())).first; + i = bifs->insert(std::make_pair(lower_plugin, new bif_init_func_list())).first; i->second->push_back(c); } @@ -348,7 +350,7 @@ void Manager::InitBifs() for ( plugin_list::iterator i = Manager::ActivePluginsInternal()->begin(); i != Manager::ActivePluginsInternal()->end(); i++ ) { - bif_init_func_map::const_iterator b = bifs->find((*i)->Name()); + bif_init_func_map::const_iterator b = bifs->find(strtolower((*i)->Name())); if ( b != bifs->end() ) { @@ -397,7 +399,7 @@ Manager::inactive_plugin_list Manager::InactivePlugins() const for ( plugin_list::const_iterator j = all->begin(); j != all->end(); j++ ) { - if ( (*i).first == (*j)->Name() ) + if ( (*i).first == strtolower((*j)->Name()) ) { found = true; break; @@ -434,7 +436,7 @@ Manager::bif_init_func_map* Manager::BifFilesInternal() static bool hook_cmp(std::pair a, std::pair b) { if ( a.first == b.first ) - return a.second->Name() < a.second->Name(); + return strtolower(a.second->Name()) < strtolower(a.second->Name()); // Reverse sort. return a.first > b.first; diff --git a/src/util.cc b/src/util.cc index 60a92af45f..ac2a942ed3 100644 --- a/src/util.cc +++ b/src/util.cc @@ -541,6 +541,13 @@ bool is_printable(const char* s, int len) return true; } +std::string strtolower(const std::string& s) + { + std::string t = s; + std::transform(t.begin(), t.end(), t.begin(), ::tolower); + return t; + } + const char* fmt_bytes(const char* data, int len) { static char buf[1024]; diff --git a/src/util.h b/src/util.h index 50c33d5608..f65e0fb7d0 100644 --- a/src/util.h +++ b/src/util.h @@ -159,6 +159,9 @@ int strstr_n(const int big_len, const unsigned char* big, extern int fputs(int len, const char* s, FILE* fp); extern bool is_printable(const char* s, int len); +// Return a lower-cased version of the string. +extern std::string strtolower(const std::string& s); + extern const char* fmt_bytes(const char* data, int len); // Note: returns a pointer into a shared buffer. From b6bbf90643e272814174b6eb065fdbab60e1d938 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 16 Feb 2015 14:32:08 -0800 Subject: [PATCH 20/37] Updating plugin tests. The init-plugin scripts now expects a destination directory. Normally that would be a new subdirectory, but for the tests to keep working we can also put it right into the current directory. --- testing/btest/Baseline/plugins.api-version-mismatch/output | 2 +- testing/btest/plugins/api-version-mismatch.sh | 2 +- testing/btest/plugins/bifs-and-scripts-install.sh | 4 ++-- testing/btest/plugins/bifs-and-scripts.sh | 2 +- testing/btest/plugins/file.bro | 2 +- testing/btest/plugins/hooks.bro | 2 +- testing/btest/plugins/init-plugin.bro | 2 +- testing/btest/plugins/pktdumper.bro | 2 +- testing/btest/plugins/pktsrc.bro | 2 +- testing/btest/plugins/protocol.bro | 2 +- testing/btest/plugins/reader.bro | 2 +- testing/btest/plugins/writer.bro | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/testing/btest/Baseline/plugins.api-version-mismatch/output b/testing/btest/Baseline/plugins.api-version-mismatch/output index 1e4dae5e65..04f3cdd3a2 100644 --- a/testing/btest/Baseline/plugins.api-version-mismatch/output +++ b/testing/btest/Baseline/plugins.api-version-mismatch/output @@ -1 +1 @@ -fatal error in /home/robin/bro/master/scripts/base/init-bare.bro, line 1: plugin's API version does not match Bro (expected 2, got 42 in /home/robin/bro/master/testing/btest/.tmp/plugins.api-version-mismatch//lib/XXX) +fatal error in /home/robin/bro/plugins/scripts/base/init-bare.bro, line 1: plugin's API version does not match Bro (expected 2, got 42 in /home/robin/bro/plugins/testing/btest/.tmp/plugins.api-version-mismatch/build//lib/XXX) diff --git a/testing/btest/plugins/api-version-mismatch.sh b/testing/btest/plugins/api-version-mismatch.sh index cfb4269946..2483582359 100644 --- a/testing/btest/plugins/api-version-mismatch.sh +++ b/testing/btest/plugins/api-version-mismatch.sh @@ -1,4 +1,4 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Foo # @TEST-EXEC: bash %INPUT # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC-FAIL: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >tmp 2>&1 diff --git a/testing/btest/plugins/bifs-and-scripts-install.sh b/testing/btest/plugins/bifs-and-scripts-install.sh index 627eb0f2c5..60c754f8ff 100644 --- a/testing/btest/plugins/bifs-and-scripts-install.sh +++ b/testing/btest/plugins/bifs-and-scripts-install.sh @@ -1,10 +1,10 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Foo # @TEST-EXEC: bash %INPUT # @TEST-EXEC: ./configure --bro-dist=${DIST} --install-root=`pwd`/test-install # @TEST-EXEC: make # @TEST-EXEC: make install # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd`/test-install bro -NN Demo::Foo >>output -# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro demo/foo -r $TRACES/empty.trace >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd`/test-install bro demo/foo -r $TRACES/empty.trace >>output # @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output mkdir -p scripts/demo/foo/base/ diff --git a/testing/btest/plugins/bifs-and-scripts.sh b/testing/btest/plugins/bifs-and-scripts.sh index cf49642766..25f2dbeb5e 100644 --- a/testing/btest/plugins/bifs-and-scripts.sh +++ b/testing/btest/plugins/bifs-and-scripts.sh @@ -1,4 +1,4 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Foo # @TEST-EXEC: bash %INPUT # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output diff --git a/testing/btest/plugins/file.bro b/testing/btest/plugins/file.bro index 7d25cab538..29724aa8a4 100644 --- a/testing/btest/plugins/file.bro +++ b/testing/btest/plugins/file.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Foo # @TEST-EXEC: cp -r %DIR/file-plugin/* . # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output diff --git a/testing/btest/plugins/hooks.bro b/testing/btest/plugins/hooks.bro index 786e6ccc88..c1dec2f4c3 100644 --- a/testing/btest/plugins/hooks.bro +++ b/testing/btest/plugins/hooks.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Hooks +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Hooks # @TEST-EXEC: cp -r %DIR/hooks-plugin/* . # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/http/get.trace %INPUT 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq >output diff --git a/testing/btest/plugins/init-plugin.bro b/testing/btest/plugins/init-plugin.bro index 2fffa88f2c..a4ebf7b00c 100644 --- a/testing/btest/plugins/init-plugin.bro +++ b/testing/btest/plugins/init-plugin.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Foo # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output diff --git a/testing/btest/plugins/pktdumper.bro b/testing/btest/plugins/pktdumper.bro index 29b69acadd..d9bd91a5a6 100644 --- a/testing/btest/plugins/pktdumper.bro +++ b/testing/btest/plugins/pktdumper.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Foo # @TEST-EXEC: cp -r %DIR/pktdumper-plugin/* . # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output diff --git a/testing/btest/plugins/pktsrc.bro b/testing/btest/plugins/pktsrc.bro index 349e361664..a13596e245 100644 --- a/testing/btest/plugins/pktsrc.bro +++ b/testing/btest/plugins/pktsrc.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Foo # @TEST-EXEC: cp -r %DIR/pktsrc-plugin/* . # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output diff --git a/testing/btest/plugins/protocol.bro b/testing/btest/plugins/protocol.bro index 671edb6cf1..8a6c2a6399 100644 --- a/testing/btest/plugins/protocol.bro +++ b/testing/btest/plugins/protocol.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Foo # @TEST-EXEC: cp -r %DIR/protocol-plugin/* . # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output diff --git a/testing/btest/plugins/reader.bro b/testing/btest/plugins/reader.bro index 5065678c2e..ec9b6cf046 100644 --- a/testing/btest/plugins/reader.bro +++ b/testing/btest/plugins/reader.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Foo # @TEST-EXEC: cp -r %DIR/reader-plugin/* . # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output diff --git a/testing/btest/plugins/writer.bro b/testing/btest/plugins/writer.bro index f2e74ad667..8cecff6843 100644 --- a/testing/btest/plugins/writer.bro +++ b/testing/btest/plugins/writer.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin -u . Demo Foo # @TEST-EXEC: cp -r %DIR/writer-plugin/* . # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output From bdb2707a08a5772d9de7371dc1050f0d5db4bb56 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 16 Feb 2015 20:05:30 -0800 Subject: [PATCH 21/37] Updating submodules. --- aux/bro-aux | 2 +- aux/plugins | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/bro-aux b/aux/bro-aux index 63675de3cc..8c37b26823 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 63675de3cc7bc3eb2a3645860224c372d3f7f36a +Subproject commit 8c37b26823ada9c77614b2f8f781c11c8fe3d078 diff --git a/aux/plugins b/aux/plugins index ad600b5bdc..9072e28935 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit ad600b5bdcd56a2723e323c0f2c8e1708956ca4f +Subproject commit 9072e28935ba85b6938fe946d7aa23cb58ee1566 From 9025b425344c150b5c06431b63cf3995c60916bb Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 Feb 2015 12:56:36 -0800 Subject: [PATCH 22/37] Updating submodule. --- aux/bro-aux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/bro-aux b/aux/bro-aux index 8c37b26823..fa145348ab 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 8c37b26823ada9c77614b2f8f781c11c8fe3d078 +Subproject commit fa145348abe15dcd5f8e52cc96e1fb758c092e36 From 818ba9127f4856f1e83c5e42eba0117adfeafdbe Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 Feb 2015 13:59:21 -0800 Subject: [PATCH 23/37] Update submodules. --- aux/bro-aux | 2 +- cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/bro-aux b/aux/bro-aux index fa145348ab..c409d529cf 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit fa145348abe15dcd5f8e52cc96e1fb758c092e36 +Subproject commit c409d529cf0c3fa851d4720914badb04d0c6b7c2 diff --git a/cmake b/cmake index 9623367210..5e4e3507e2 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 962336721040fdf55a6b264f8bbc84153b54d9a5 +Subproject commit 5e4e3507e280c393778fd55fb0124217067e0078 From 093d4069206b9ae7ced009241cdd8e70aec1c4ec Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 Feb 2015 14:03:05 -0800 Subject: [PATCH 24/37] Updating plugin docs to recent changes. --- doc/devel/plugins.rst | 97 ++++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst index c703345891..66ffba101f 100644 --- a/doc/devel/plugins.rst +++ b/doc/devel/plugins.rst @@ -3,7 +3,7 @@ Writing Bro Plugins =================== -Bro is internally moving to a plugin structure that enables extending +Bro internally provides plugin API that enables extending the system dynamically, without modifying the core code base. That way custom code remains self-contained and can be maintained, compiled, and installed independently. Currently, plugins can add the following @@ -42,13 +42,13 @@ certain structure. To get started, Bro's distribution provides a helper script ``aux/bro-aux/plugin-support/init-plugin`` that creates a skeleton plugin that can then be customized. Let's use that:: - # mkdir rot13-plugin - # cd rot13-plugin - # init-plugin Demo Rot13 + # init-plugin ./rot13-plugin Demo Rot13 -As you can see the script takes two arguments. The first is a -namespace the plugin will live in, and the second a descriptive name -for the plugin itself. Bro uses the combination of the two to identify +As you can see the script takes three arguments. The first is a +directory inside which the plugin skeleton will be create; it +shouldn't exist it. The second is namespace the plugin will live in, +and the third a descriptive name for the plugin itself relative to the +namespace. Bro uses the combination of namespace and name to identify a plugin. The namespace serves to avoid naming conflicts between plugins written by independent developers; pick, e.g., the name of your organisation. The namespace ``Bro`` is reserved for functionality @@ -82,18 +82,22 @@ The syntax of this file is just like any other ``*.bif`` file; we won't go into it here. Now we can already compile our plugin, we just need to tell the -configure script put in place by ``init-plugin`` where the Bro source -tree is located (Bro needs to have been built there first):: +configure script that ``init-plugin`` put in place where the Bro +source tree is located (Bro needs to have been built there first):: + # cd rot13-plugin # ./configure --bro-dist=/path/to/bro/dist && make [... cmake output ...] -Now our ``rot13-plugin`` directory has everything that it needs -for Bro to recognize it as a dynamic plugin. Once we point Bro to it, -it will pull it in automatically, as we can check with the ``-N`` +This builds the plugin in a subdirectory ``build/``. In fact, that +subdirectory *becomes* the plugin: when ``make`` finishes, ``build/`` +has everything it needs for Bro to recognize it as a dynamic plugin. + +Let's try that. Once we point Bro to the ``build/`` directory, it will +pull in our new plugin automatically, as we can check with the ``-N`` option:: - # export BRO_PLUGIN_PATH=/path/to/rot13-plugin + # export BRO_PLUGIN_PATH=/path/to/rot13-plugin/build # bro -N [...] Plugin: Demo::Rot13 - (dynamic, version 1) @@ -153,24 +157,28 @@ Once we install it, it works again:: The installed version went into ``/lib/bro/plugins/Demo_Rot13``. -We can distribute the plugin in either source or binary form by using -the Makefile's ``sdist`` and ``bdist`` target, respectively. Both -create corrsponding tarballs:: +One can distribute the plugin independently of Bro for others to use. +To distribute in source form, just remove the ``build/`` (``make +distclean`` does that) and then tar up the whole ``rot13-plugin/`` +directory. Others then follow the same process as above after +unpacking. To distribute the plugin in binary form, the build process +conviniently creates a corresponding tarball in ``build/dist/``. In +this case, it's called ``Demo_Rot13-0.1.tar.gz``, with the version +number coming out of the ``VERSION`` file that ``init-plugin`` put +into place. The binary tarball has everything needed to run the +plugin, but no further source files. Optionally, one can include +further files by specifying them in the plugin's ``CMakeLists.txt`` +through the ``bro_plugin_dist_files`` macro; the skeleton does that +for ``README``, ``VERSION``, ``CHANGES``, and ``COPYING``. To use the +plugin through the binary tarball, just unpack it and point +``BRO_PLUGIN_PATH`` there; or copy it into +``/lib/bro/plugins/`` directly. - # make sdist - [...] - Source distribution in build/sdist/Demo_Rot13.tar.gz - - # make bdist - [...] - Binary distribution in build/Demo_Rot13-darwin-x86_64.tar.gz - -The source archive will contain everything in the plugin directory -except any generated files. The binary archive will contain anything -needed to install and run the plugin, i.e., just what ``make install`` -puts into place as well. As the binary distribution is -platform-dependent, its name includes the OS and architecture the -plugin was built on. +Before distributing your plugin, you should edit some of the meta +files that ``init-plugin`` puts in place. Edit ``README`` and +``VERSION``, and update ``CHANGES`` when you make changes. Also put a +license file in place as ``COPYING``; if BSD is fine, you find a +template in ``COPYING.edit-me``. Plugin Directory Layout ======================= @@ -179,7 +187,7 @@ A plugin's directory needs to follow a set of conventions so that Bro (1) recognizes it as a plugin, and (2) knows what to load. While ``init-plugin`` takes care of most of this, the following is the full story. We'll use ```` to represent a plugin's top-level -directory. +directory. With the skeleton, ```` corresponds to ``build/``. ``/__bro_plugin__`` A file that marks a directory as containing a Bro plugin. The file @@ -205,6 +213,8 @@ directory. Directory with auto-generated Bro scripts that declare the plugin's bif elements. The files here are produced by ``bifcl``. +Any other files in ```` are ignored by Bro. + By convention, a plugin should put its custom scripts into sub folders of ``scripts/``, i.e., ``scripts//