From 51203d71934436c7f30922dfc6e0f8d4800a9f62 Mon Sep 17 00:00:00 2001 From: Mike Smiley Date: Tue, 3 Feb 2015 14:29:34 -0500 Subject: [PATCH 01/11] "id" not defined for debug code "id" not defined for debug code when using -DPROFILE_BRO_FUNCTIONS --- src/Func.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Func.cc b/src/Func.cc index d66e9c71fa..693a4535d4 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -323,7 +323,7 @@ int BroFunc::IsPure() const Val* BroFunc::Call(val_list* args, Frame* parent) const { #ifdef PROFILE_BRO_FUNCTIONS - DEBUG_MSG("Function: %s\n", id->Name()); + DEBUG_MSG("Function: %s\n", Name()); #endif SegmentProfiler(segment_logger, location); From a97cd1f3a24d93e81190cc28ba283d26f31035c8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 5 Feb 2015 09:09:08 -0500 Subject: [PATCH 02/11] Fix a bug in the core files framework with handling the BOF buffer. - Any files where the total size was below the size of the default bof_buffer size couldn't have stream analyzers successfully attached because the bof_buffer never reached the full size and was never flushed. This branch explicitly marks the buf_buffer as full and flushes it when the file is being removed. --- src/file_analysis/File.cc | 18 +++++++++++------- .../files.log | 10 ++++++++++ .../file-analysis/big-bof-buffer.bro | 6 ++++++ 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.file-analysis.big-bof-buffer/files.log create mode 100644 testing/btest/scripts/base/frameworks/file-analysis/big-bof-buffer.bro diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index 50617f27b6..cc1f86412c 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -492,18 +492,22 @@ void File::EndOfFile() if ( done ) return; - if ( ! did_mime_type && - LookupFieldDefaultCount(missing_bytes_idx) == 0 ) - DetectMIME(); - - analyzers.DrainModifications(); - if ( file_reassembler ) { file_reassembler->Flush(); - analyzers.DrainModifications(); } + // Mark the bof_buffer as full in case it isn't yet + // so that the whole thing can be flushed out to + // any stream analyzers. + if ( ! bof_buffer.full ) + { + bof_buffer.full = true; + DeliverStream((const u_char*) "", 0); + } + + analyzers.DrainModifications(); + done = true; file_analysis::Analyzer* a = 0; diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.big-bof-buffer/files.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.big-bof-buffer/files.log new file mode 100644 index 0000000000..cebe140bda --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.big-bof-buffer/files.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path files +#open 2015-02-05-13-55-41 +#fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted +#types time string set[addr] set[addr] set[string] string count set[string] string string interval bool bool count count count count bool string string string string string +1362692527.009512 FakNcS1Jfe01uljb3 192.150.187.43 141.142.228.5 CXWv6p3arKYeMETxOg HTTP 0 MD5,SHA1 text/plain - 0.000263 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 - - +#close 2015-02-05-13-55-41 diff --git a/testing/btest/scripts/base/frameworks/file-analysis/big-bof-buffer.bro b/testing/btest/scripts/base/frameworks/file-analysis/big-bof-buffer.bro new file mode 100644 index 0000000000..0f7e23ddcf --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/big-bof-buffer.bro @@ -0,0 +1,6 @@ +# @TEST-EXEC: bro -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: btest-diff files.log + +@load frameworks/files/hash-all-files + +redef default_file_bof_buffer_size=5000; From 8859c73bde5d392be6081bade798e26b79e0e56e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 5 Feb 2015 10:04:04 -0600 Subject: [PATCH 03/11] Add/fix log fields in x509 diff canonifier. --- testing/scripts/diff-remove-x509-names | 30 +++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/testing/scripts/diff-remove-x509-names b/testing/scripts/diff-remove-x509-names index 4863efc990..4534cb7d87 100755 --- a/testing/scripts/diff-remove-x509-names +++ b/testing/scripts/diff-remove-x509-names @@ -3,19 +3,25 @@ # A diff canonifier that removes all X.509 Distinguished Name subject fields # because that output can differ depending on installed OpenSSL version. -BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1; cs_col = -1; ci_col = -1 } +BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1; is_col = -1; cs_col = -1; ci_col = -1; cert_subj_col = -1; cert_issuer_col = -1 } /^#fields/ { for ( i = 2; i < NF; ++i ) { if ( $i == "subject" ) s_col = i-1; - if ( $i == "issuer_subject" ) + if ( $i == "issuer" ) i_col = i-1; + if ( $i == "issuer_subject" ) + is_col = i-1; if ( $i == "client_subject" ) cs_col = i-1; - if ( $i == "client_issuer_subject" ) + if ( $i == "client_issuer" ) ci_col = i-1; + if ( $i == "certificate.subject" ) + cert_subj_col = i-1; + if ( $i == "certificate.issuer" ) + cert_issuer_col = i-1; } } @@ -31,6 +37,12 @@ i_col >= 0 { $i_col = "+"; } +is_col >= 0 { + if ( $is_col != "-" ) + # Mark that it's set, but ignore content. + $is_col = "+"; +} + cs_col >= 0 { if ( $cs_col != "-" ) # Mark that it's set, but ignore content. @@ -43,6 +55,18 @@ ci_col >= 0 { $ci_col = "+"; } +cert_subj_col >= 0 { + if ( $cert_subj_col != "-" ) + # Mark that it's set, but ignore content. + $cert_subj_col = "+"; +} + +cert_issuer_col >= 0 { + if ( $cert_issuer_col != "-" ) + # Mark that it's set, but ignore content. + $cert_issuer_col = "+"; +} + { print; } From 9592f6422530aff4873d31453954acacd6034e43 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 5 Feb 2015 12:44:10 -0500 Subject: [PATCH 04/11] Update the SOCKS analyzer to support user/pass login. - This addresses BIT-1011 - Add a new field to socks.log; "password". - Two new events; socks_login_userpass and socks_login_reply. - One new weird for unsupported authentication method. - A new test for authenticated socks traffic. - Credit to Nicolas Retrain for the initial patch. Thanks! --- scripts/base/protocols/socks/main.bro | 22 ++++++- src/analyzer/protocol/socks/SOCKS.cc | 3 +- src/analyzer/protocol/socks/events.bif | 16 +++++ .../protocol/socks/socks-analyzer.pac | 40 ++++++++++++ .../protocol/socks/socks-protocol.pac | 57 +++++++++++++++--- .../socks.log | 10 +++ .../tunnel.log | 10 +++ .../socks.log | 10 +-- .../socks.log | 10 +-- testing/btest/Traces/socks-auth.pcap | Bin 0 -> 1326 bytes .../base/protocols/socks/socks-auth.bro | 5 ++ 11 files changed, 162 insertions(+), 21 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/socks.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/tunnel.log create mode 100644 testing/btest/Traces/socks-auth.pcap create mode 100644 testing/btest/scripts/base/protocols/socks/socks-auth.bro diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro index 713161d442..f60c3ce41c 100644 --- a/scripts/base/protocols/socks/main.bro +++ b/scripts/base/protocols/socks/main.bro @@ -16,8 +16,10 @@ export { id: conn_id &log; ## Protocol version of SOCKS. version: count &log; - ## Username for the proxy if extracted from the network. + ## Username used to request a login to the proxy. user: string &log &optional; + ## Password used to request a login to the proxy. + password: string &log &optional; ## Server status for the attempt at using the proxy. status: string &log &optional; ## Client requested SOCKS address. Could be an address, a name @@ -91,3 +93,21 @@ event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Addres if ( "SOCKS" in c$service ) Log::write(SOCKS::LOG, c$socks); } + +event socks_login_userpass(c: connection, user: string, password: string) &priority=5 + { + # Authentication only possible with the version 5. + set_session(c, 5); + + c$socks$user = user; + c$socks$password = password; + } + +event socks_login_reply(c: connection, code: count) &priority=5 + { + # Authentication only possible with the version 5. + set_session(c, 5); + + c$socks$status = v5_status[code]; + } + diff --git a/src/analyzer/protocol/socks/SOCKS.cc b/src/analyzer/protocol/socks/SOCKS.cc index e678528f35..ec1e85653b 100644 --- a/src/analyzer/protocol/socks/SOCKS.cc +++ b/src/analyzer/protocol/socks/SOCKS.cc @@ -57,8 +57,7 @@ void SOCKS_Analyzer::DeliverStream(int len, const u_char* data, bool orig) // with the rest of the conneciton. // // Note that we assume that no payload data arrives before both endpoints - // are done with there part of the SOCKS protocol. - + // are done with their part of the SOCKS protocol. if ( ! pia ) { pia = new pia::PIA_TCP(Conn()); diff --git a/src/analyzer/protocol/socks/events.bif b/src/analyzer/protocol/socks/events.bif index 4f1f8ad1cd..ece69140a1 100644 --- a/src/analyzer/protocol/socks/events.bif +++ b/src/analyzer/protocol/socks/events.bif @@ -27,3 +27,19 @@ event socks_request%(c: connection, version: count, request_type: count, sa: SOC ## p: The destination port for the proxied traffic. event socks_reply%(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port%); +## Generated when a SOCKS client performs username and password based login. +## +## c: The parent connection of the proxy. +## +## user: The given username. +## +## password: The given password. +event socks_login_userpass%(c: connection, user: string, password: string%); + +## Generated when a SOCKS server replies to a login attempt. +## +## c: The parent connection of the proxy. +## +## code: The response code for the attempted login. +event socks_login_reply%(c: connection, code: count%); + diff --git a/src/analyzer/protocol/socks/socks-analyzer.pac b/src/analyzer/protocol/socks/socks-analyzer.pac index db98b3f4b3..7d634e2f46 100644 --- a/src/analyzer/protocol/socks/socks-analyzer.pac +++ b/src/analyzer/protocol/socks/socks-analyzer.pac @@ -148,6 +148,31 @@ refine connection SOCKS_Conn += { return true; %} + function socks5_auth_request_userpass(request: SOCKS5_Auth_Request_UserPass): bool + %{ + StringVal* user = new StringVal(${request.username}.length(), (const char*) ${request.username}.begin()); + StringVal* pass = new StringVal(${request.password}.length(), (const char*) ${request.password}.begin()); + + BifEvent::generate_socks_login_userpass(bro_analyzer(), + bro_analyzer()->Conn(), + user, pass); + return true; + %} + + function socks5_unsupported_authentication(auth_method: uint8): bool + %{ + reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_%d", auth_method)); + return true; + %} + + function socks5_auth_reply(reply: SOCKS5_Auth_Reply): bool + %{ + BifEvent::generate_socks_login_reply(bro_analyzer(), + bro_analyzer()->Conn(), + ${reply.code}); + return true; + %} + function version_error(version: uint8): bool %{ bro_analyzer()->ProtocolViolation(fmt("unsupported/unknown SOCKS version %d", version)); @@ -176,3 +201,18 @@ refine typeattr SOCKS5_Request += &let { refine typeattr SOCKS5_Reply += &let { proc: bool = $context.connection.socks5_reply(this); }; + +refine typeattr SOCKS5_Auth_Negotiation_Reply += &let { +}; + +refine typeattr SOCKS5_Auth_Request_UserPass += &let { + proc: bool = $context.connection.socks5_auth_request_userpass(this); +}; + +refine typeattr SOCKS5_Auth_Reply += &let { + proc: bool = $context.connection.socks5_auth_reply(this); +}; + +refine typeattr SOCKS5_Unsupported_Authentication += &let { + proc: bool = $context.connection.socks5_unsupported_authentication($context.connection.v5_auth_method()); +}; diff --git a/src/analyzer/protocol/socks/socks-protocol.pac b/src/analyzer/protocol/socks/socks-protocol.pac index 05ca4bc861..4e48ea0672 100644 --- a/src/analyzer/protocol/socks/socks-protocol.pac +++ b/src/analyzer/protocol/socks/socks-protocol.pac @@ -2,9 +2,10 @@ type SOCKS_Version(is_orig: bool) = record { version: uint8; msg: case version of { - 4 -> socks4_msg: SOCKS4_Message(is_orig); - 5 -> socks5_msg: SOCKS5_Message(is_orig); - default -> socks_msg_fail: SOCKS_Version_Error(version); + 1 -> socks5_auth_msg: SOCKS5_Auth_Message(is_orig); + 4 -> socks4_msg: SOCKS4_Message(is_orig); + 5 -> socks5_msg: SOCKS5_Message(is_orig); + default -> socks_msg_fail: SOCKS_Version_Error(version); }; }; @@ -14,10 +15,11 @@ type SOCKS_Version_Error(version: uint8) = record { # SOCKS5 Implementation type SOCKS5_Message(is_orig: bool) = case $context.connection.v5_past_authentication() of { - true -> msg: SOCKS5_Real_Message(is_orig); false -> auth: SOCKS5_Auth_Negotiation(is_orig); + true -> msg: SOCKS5_Real_Message(is_orig); }; + type SOCKS5_Auth_Negotiation(is_orig: bool) = case is_orig of { true -> req: SOCKS5_Auth_Negotiation_Request; false -> rep: SOCKS5_Auth_Negotiation_Reply; @@ -32,6 +34,32 @@ type SOCKS5_Auth_Negotiation_Reply = record { selected_auth_method: uint8; } &let { past_auth = $context.connection.set_v5_past_authentication(); + set_auth = $context.connection.set_v5_auth_method(selected_auth_method); +}; + +type SOCKS5_Auth_Message(is_orig: bool) = case is_orig of { + true -> req: SOCKS5_Auth_Request; + false -> rep: SOCKS5_Auth_Reply; +}; + +type SOCKS5_Auth_Request = case $context.connection.v5_auth_method() of { + 0x02 -> userpass : SOCKS5_Auth_Request_UserPass; + default -> unsupported : SOCKS5_Unsupported_Authentication; +}; + +type SOCKS5_Unsupported_Authentication = record { + crap: bytestring &restofdata; +}; + +type SOCKS5_Auth_Request_UserPass = record { + ulen : uint8; + username : bytestring &length=ulen; + plen : uint8; + password : bytestring &length=plen; +}; + +type SOCKS5_Auth_Reply = record { + code : uint8; }; type SOCKS5_Real_Message(is_orig: bool) = case is_orig of { @@ -55,10 +83,10 @@ type SOCKS5_Address = record { } &byteorder = bigendian; type SOCKS5_Request = record { - command: uint8; - reserved: uint8; - remote_name: SOCKS5_Address; - port: uint16; + command : uint8; + reserved : uint8; + remote_name : SOCKS5_Address; + port : uint16; } &byteorder = bigendian; type SOCKS5_Reply = record { @@ -99,10 +127,12 @@ type SOCKS4_Reply = record { refine connection SOCKS_Conn += { %member{ bool v5_authenticated_; + uint8 selected_auth_method_; %} %init{ v5_authenticated_ = false; + selected_auth_method_ = 255; %} function v5_past_authentication(): bool @@ -115,5 +145,16 @@ refine connection SOCKS_Conn += { v5_authenticated_ = true; return true; %} + + function set_v5_auth_method(method: uint8): bool + %{ + selected_auth_method_ = method; + return true; + %} + + function v5_auth_method(): uint8 + %{ + return selected_auth_method_; + %} }; diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/socks.log new file mode 100644 index 0000000000..cc5fa80191 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/socks.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path socks +#open 2015-02-05-16-13-12 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user password status request.host request.name request_p bound.host bound.name bound_p +#types time string addr port addr port count string string string addr string port addr string port +1368517392.724989 CXWv6p3arKYeMETxOg 192.168.0.2 55951 192.168.0.1 1080 5 bob alice succeeded 192.168.0.2 - 22 192.168.0.1 - 55951 +#close 2015-02-05-16-13-12 diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/tunnel.log b/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/tunnel.log new file mode 100644 index 0000000000..d53238df93 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.socks.socks-auth/tunnel.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path tunnel +#open 2015-02-05-16-13-12 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action +#types time string addr port addr port enum enum +1368517392.728523 - 192.168.0.2 0 192.168.0.1 1080 Tunnel::SOCKS Tunnel::DISCOVER +#close 2015-02-05-16-13-12 diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log index 148e4adf02..f69df31b66 100644 --- a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log +++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path socks -#open 2013-08-26-19-04-20 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user status request.host request.name request_p bound.host bound.name bound_p -#types time string addr port addr port count string string addr string port addr string port -1340213015.276495 CjhGID4nQcgTWjvg4c 10.0.0.55 53994 60.190.189.214 8124 5 - succeeded - www.osnews.com 80 192.168.0.31 - 2688 -#close 2013-08-26-19-04-20 +#open 2015-02-05-17-39-14 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user password status request.host request.name request_p bound.host bound.name bound_p +#types time string addr port addr port count string string string addr string port addr string port +1340213015.276495 CjhGID4nQcgTWjvg4c 10.0.0.55 53994 60.190.189.214 8124 5 - - succeeded - www.osnews.com 80 192.168.0.31 - 2688 +#close 2015-02-05-17-39-14 diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log index d706a11da3..de7b26f875 100644 --- a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log +++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path socks -#open 2013-08-26-19-04-20 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user status request.host request.name request_p bound.host bound.name bound_p -#types time string addr port addr port count string string addr string port addr string port -1340113261.914619 CXWv6p3arKYeMETxOg 10.0.0.50 59580 85.194.84.197 1080 5 - succeeded - www.google.com 443 0.0.0.0 - 443 -#close 2013-08-26-19-04-20 +#open 2015-02-05-17-39-29 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user password status request.host request.name request_p bound.host bound.name bound_p +#types time string addr port addr port count string string string addr string port addr string port +1340113261.914619 CXWv6p3arKYeMETxOg 10.0.0.50 59580 85.194.84.197 1080 5 - - succeeded - www.google.com 443 0.0.0.0 - 443 +#close 2015-02-05-17-39-29 diff --git a/testing/btest/Traces/socks-auth.pcap b/testing/btest/Traces/socks-auth.pcap new file mode 100644 index 0000000000000000000000000000000000000000..1570e229473da081b50bd3906a3263503896de6a GIT binary patch literal 1326 zcmaKsPiWIn9LIkzFX_UfGfpY9D5wV$MAW)E5-sZxrA#(N@HTb2xuGEQ<~hefgf^Ff z?y`ypWlTXt@T54QNb%3Wfs`J-dGe;Xrv84frsSp7!mlJDeDZyt_xrss>(46+s1W^H zEdYi$=J(*@yAOSE9;2`oPSwl-kr!CAid?8JV@=wjO=e-!>&?Rx1mE3X;fWA*S{i& z13igDJP3bxhz&rZ#(^t@okwgesH`BDbqKrYBowqz!IEb><FdsoPSj;elW54rQ$+{kxuPd#Iyhx zzeOpVI}Dh_gj8KR3;whqrg{=lr+G;xGCp%>M!SCd*6bX?cbQV$q|{SF!HBFN!%%8P zNPQH+ZB$^1etNR^zfw;N;&4wQO8s=dLu?X}31)4d0TZ}q12|v+EW5)GH~I95uaJvKz{QEvA^*sZ@mQR#e>ZU7(~phT976yr03CO%@0dt zxfS9wD^3GQ*GmUBeoi2LnPztVZk%i>m^_S5gV#v@_b6=6iOl_p%*#lh^h{U(QH^}q qAvzXq>*gfGvW3xXUhLZ8h}b?~G93G0)x%voJ}wcNhPU)?z4Ql$s7wg} literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/protocols/socks/socks-auth.bro b/testing/btest/scripts/base/protocols/socks/socks-auth.bro new file mode 100644 index 0000000000..2123dc1d45 --- /dev/null +++ b/testing/btest/scripts/base/protocols/socks/socks-auth.bro @@ -0,0 +1,5 @@ +# @TEST-EXEC: bro -r $TRACES/socks-auth.pcap %INPUT +# @TEST-EXEC: btest-diff socks.log +# @TEST-EXEC: btest-diff tunnel.log + +@load base/protocols/socks From 530c3c0c6b19fe0e708a076a79567f9fa8334216 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 8 Feb 2015 18:20:38 -0800 Subject: [PATCH 05/11] Changing load order for plugin scripts. This can be need if they depends on each other. --- src/plugin/Manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 2ca34d94f3..ab0b85676b 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -172,7 +172,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ // Load {bif,scripts}/__load__.bro automatically. - string init = dir + "scripts/__load__.bro"; + string init = dir + "lib/bif/__load__.bro"; if ( is_file(init) ) { @@ -180,7 +180,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ scripts_to_load.push_back(init); } - init = dir + "lib/bif/__load__.bro"; + init = dir + "scripts/__load__.bro"; if ( is_file(init) ) { From 23b9705a7bbd6333b767b8908be2ddcf7017b5a5 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 8 Feb 2015 18:21:23 -0800 Subject: [PATCH 06/11] Fixing analyzer tag types for some Files::* functions. --- CHANGES | 6 ++++++ VERSION | 2 +- scripts/base/frameworks/files/main.bro | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/CHANGES b/CHANGES index d1031765cc..3367c878cd 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ +2.3-413 | 2015-02-08 18:23:05 -0800 + + * Fixing analyzer tag types for some Files::* functions. (Robin Sommer) + + * Changing load order for plugin scripts. (Robin Sommer) + 2.3-411 | 2015-02-05 10:05:48 -0600 * Fix file analysis of files with total size below the bof_buffer size diff --git a/VERSION b/VERSION index defa33cc31..fca56f2eeb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-411 +2.3-413 diff --git a/scripts/base/frameworks/files/main.bro b/scripts/base/frameworks/files/main.bro index e335d4be9d..94a46578c0 100644 --- a/scripts/base/frameworks/files/main.bro +++ b/scripts/base/frameworks/files/main.bro @@ -267,7 +267,7 @@ export { ## mts: The set of MIME types, each in the form "foo/bar" (case-insensitive). ## ## Returns: True if the MIME types were successfully registered. - global register_for_mime_types: function(tag: Analyzer::Tag, mts: set[string]) : bool; + global register_for_mime_types: function(tag: Files::Tag, mts: set[string]) : bool; ## Registers a MIME type for an analyzer. If a future file with this type is seen, ## the analyzer will be automatically assigned to parsing it. The function *adds* @@ -278,20 +278,20 @@ export { ## mt: The MIME type in the form "foo/bar" (case-insensitive). ## ## Returns: True if the MIME type was successfully registered. - global register_for_mime_type: function(tag: Analyzer::Tag, mt: string) : bool; + global register_for_mime_type: function(tag: Files::Tag, mt: string) : bool; ## Returns a set of all MIME types currently registered for a specific analyzer. ## ## tag: The tag of the analyzer. ## ## Returns: The set of MIME types. - global registered_mime_types: function(tag: Analyzer::Tag) : set[string]; + global registered_mime_types: function(tag: Files::Tag) : set[string]; ## Returns a table of all MIME-type-to-analyzer mappings currently registered. ## ## Returns: A table mapping each analyzer to the set of MIME types ## registered for it. - global all_registered_mime_types: function() : table[Analyzer::Tag] of set[string]; + global all_registered_mime_types: function() : table[Files::Tag] of set[string]; ## Event that can be handled to access the Info record as it is sent on ## to the logging framework. @@ -306,8 +306,8 @@ redef record fa_file += { global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table(); # Store the MIME type to analyzer mappings. -global mime_types: table[Analyzer::Tag] of set[string]; -global mime_type_to_analyzers: table[string] of set[Analyzer::Tag]; +global mime_types: table[Files::Tag] of set[string]; +global mime_type_to_analyzers: table[string] of set[Files::Tag]; global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table(); @@ -401,7 +401,7 @@ function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool return result; } -function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : bool +function register_for_mime_types(tag: Files::Tag, mime_types: set[string]) : bool { local rc = T; @@ -414,7 +414,7 @@ function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : return rc; } -function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool +function register_for_mime_type(tag: Files::Tag, mt: string) : bool { if ( tag !in mime_types ) { @@ -431,12 +431,12 @@ function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool return T; } -function registered_mime_types(tag: Analyzer::Tag) : set[string] +function registered_mime_types(tag: Files::Tag) : set[string] { return tag in mime_types ? mime_types[tag] : set(); } -function all_registered_mime_types(): table[Analyzer::Tag] of set[string] +function all_registered_mime_types(): table[Files::Tag] of set[string] { return mime_types; } @@ -451,7 +451,7 @@ function describe(f: fa_file): string return handler$describe(f); } -event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5 +event get_file_handle(tag: Files::Tag, c: connection, is_orig: bool) &priority=5 { if ( tag !in registered_protocols ) return; From 5f0a27ca31443ee3c308e49ff5b6e6b1c2fec963 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 9 Feb 2015 12:10:49 -0800 Subject: [PATCH 07/11] Submodule update - newest sqlite version --- src/3rdparty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/3rdparty b/src/3rdparty index 7e15efe9d2..f2e34d731e 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 7e15efe9d28d46bfa662fcdd1cbb15ce1db285c9 +Subproject commit f2e34d731ed29bb993fbb065846faa342a8c824f From 88af106b6b5de8497499e1f6be9d317b6fbb4707 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 11 Feb 2015 13:56:34 -0600 Subject: [PATCH 08/11] Fix use of deprecated gperftools headers. As of gperftools 2.0 (Feb. 2012), they've been renamed in to gperftools/ instead of google/, and as of gperftools 2.2, including the later emits deprecation warnings. --- src/util.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util.h b/src/util.h index db77888c16..50c33d5608 100644 --- a/src/util.h +++ b/src/util.h @@ -48,8 +48,8 @@ #endif #ifdef USE_PERFTOOLS_DEBUG -#include -#include +#include +#include extern HeapLeakChecker* heap_checker; #endif From 961fd06cad004f1f167ebbf65f241349a2ea9b63 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 12 Feb 2015 17:06:38 -0600 Subject: [PATCH 09/11] Refactor SOCKS5 user/pass authentication support. - Rename event "socks_login_userpass" to "socks_login_userpass_request" - Rename event "socks_login_reply" to "socks_login_userpass_reply" - Split unsupported authN weird into 2 types: method vs. version Addresses BIT-1011 --- scripts/base/protocols/socks/main.bro | 4 +- src/analyzer/protocol/socks/events.bif | 6 +-- .../protocol/socks/socks-analyzer.pac | 40 ++++++++------ .../protocol/socks/socks-protocol.pac | 53 +++++++++++++++++-- src/analyzer/protocol/socks/socks.pac | 2 +- testing/btest/Baseline/plugins.writer/output | 4 +- 6 files changed, 82 insertions(+), 27 deletions(-) diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro index f60c3ce41c..e052962888 100644 --- a/scripts/base/protocols/socks/main.bro +++ b/scripts/base/protocols/socks/main.bro @@ -94,7 +94,7 @@ event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Addres Log::write(SOCKS::LOG, c$socks); } -event socks_login_userpass(c: connection, user: string, password: string) &priority=5 +event socks_login_userpass_request(c: connection, user: string, password: string) &priority=5 { # Authentication only possible with the version 5. set_session(c, 5); @@ -103,7 +103,7 @@ event socks_login_userpass(c: connection, user: string, password: string) &prior c$socks$password = password; } -event socks_login_reply(c: connection, code: count) &priority=5 +event socks_login_userpass_reply(c: connection, code: count) &priority=5 { # Authentication only possible with the version 5. set_session(c, 5); diff --git a/src/analyzer/protocol/socks/events.bif b/src/analyzer/protocol/socks/events.bif index ece69140a1..224f570817 100644 --- a/src/analyzer/protocol/socks/events.bif +++ b/src/analyzer/protocol/socks/events.bif @@ -34,12 +34,12 @@ event socks_reply%(c: connection, version: count, reply: count, sa: SOCKS::Addre ## user: The given username. ## ## password: The given password. -event socks_login_userpass%(c: connection, user: string, password: string%); +event socks_login_userpass_request%(c: connection, user: string, password: string%); -## Generated when a SOCKS server replies to a login attempt. +## Generated when a SOCKS server replies to a username/password login attempt. ## ## c: The parent connection of the proxy. ## ## code: The response code for the attempted login. -event socks_login_reply%(c: connection, code: count%); +event socks_login_userpass_reply%(c: connection, code: count%); diff --git a/src/analyzer/protocol/socks/socks-analyzer.pac b/src/analyzer/protocol/socks/socks-analyzer.pac index 7d634e2f46..b8c4165a54 100644 --- a/src/analyzer/protocol/socks/socks-analyzer.pac +++ b/src/analyzer/protocol/socks/socks-analyzer.pac @@ -148,28 +148,34 @@ refine connection SOCKS_Conn += { return true; %} - function socks5_auth_request_userpass(request: SOCKS5_Auth_Request_UserPass): bool + function socks5_auth_request_userpass(request: SOCKS5_Auth_Request_UserPass_v1): bool %{ StringVal* user = new StringVal(${request.username}.length(), (const char*) ${request.username}.begin()); StringVal* pass = new StringVal(${request.password}.length(), (const char*) ${request.password}.begin()); - BifEvent::generate_socks_login_userpass(bro_analyzer(), - bro_analyzer()->Conn(), - user, pass); + BifEvent::generate_socks_login_userpass_request(bro_analyzer(), + bro_analyzer()->Conn(), + user, pass); return true; %} - function socks5_unsupported_authentication(auth_method: uint8): bool + function socks5_unsupported_authentication_method(auth_method: uint8): bool %{ - reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_%d", auth_method)); + reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_method_%d", auth_method)); + return true; + %} + + function socks5_unsupported_authentication_version(auth_method: uint8, version: uint8): bool + %{ + reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_%d_%d", auth_method, version)); return true; %} - function socks5_auth_reply(reply: SOCKS5_Auth_Reply): bool + function socks5_auth_reply_userpass(reply: SOCKS5_Auth_Reply_UserPass_v1): bool %{ - BifEvent::generate_socks_login_reply(bro_analyzer(), - bro_analyzer()->Conn(), - ${reply.code}); + BifEvent::generate_socks_login_userpass_reply(bro_analyzer(), + bro_analyzer()->Conn(), + ${reply.code}); return true; %} @@ -205,14 +211,18 @@ refine typeattr SOCKS5_Reply += &let { refine typeattr SOCKS5_Auth_Negotiation_Reply += &let { }; -refine typeattr SOCKS5_Auth_Request_UserPass += &let { +refine typeattr SOCKS5_Auth_Request_UserPass_v1 += &let { proc: bool = $context.connection.socks5_auth_request_userpass(this); }; -refine typeattr SOCKS5_Auth_Reply += &let { - proc: bool = $context.connection.socks5_auth_reply(this); +refine typeattr SOCKS5_Auth_Reply_UserPass_v1 += &let { + proc: bool = $context.connection.socks5_auth_reply_userpass(this); }; -refine typeattr SOCKS5_Unsupported_Authentication += &let { - proc: bool = $context.connection.socks5_unsupported_authentication($context.connection.v5_auth_method()); +refine typeattr SOCKS5_Unsupported_Authentication_Method += &let { + proc: bool = $context.connection.socks5_unsupported_authentication_method($context.connection.v5_auth_method()); +}; + +refine typeattr SOCKS5_Unsupported_Authentication_Version += &let { + proc: bool = $context.connection.socks5_unsupported_authentication_version($context.connection.v5_auth_method(), version); }; diff --git a/src/analyzer/protocol/socks/socks-protocol.pac b/src/analyzer/protocol/socks/socks-protocol.pac index 4e48ea0672..d9c31d2377 100644 --- a/src/analyzer/protocol/socks/socks-protocol.pac +++ b/src/analyzer/protocol/socks/socks-protocol.pac @@ -1,8 +1,12 @@ +type SOCKS_Message(is_orig: bool) = case $context.connection.v5_in_auth_sub_negotiation() of { + true -> auth: SOCKS5_Auth_Message(is_orig); + false -> msg: SOCKS_Version(is_orig); +}; + type SOCKS_Version(is_orig: bool) = record { version: uint8; msg: case version of { - 1 -> socks5_auth_msg: SOCKS5_Auth_Message(is_orig); 4 -> socks4_msg: SOCKS4_Message(is_orig); 5 -> socks5_msg: SOCKS5_Message(is_orig); default -> socks_msg_fail: SOCKS_Version_Error(version); @@ -33,6 +37,7 @@ type SOCKS5_Auth_Negotiation_Request = record { type SOCKS5_Auth_Negotiation_Reply = record { selected_auth_method: uint8; } &let { + in_auth_sub_neg = $context.connection.set_v5_in_auth_sub_negotiation(selected_auth_method == 0 || selected_auth_method == 0xff ? false : true); past_auth = $context.connection.set_v5_past_authentication(); set_auth = $context.connection.set_v5_auth_method(selected_auth_method); }; @@ -44,21 +49,48 @@ type SOCKS5_Auth_Message(is_orig: bool) = case is_orig of { type SOCKS5_Auth_Request = case $context.connection.v5_auth_method() of { 0x02 -> userpass : SOCKS5_Auth_Request_UserPass; - default -> unsupported : SOCKS5_Unsupported_Authentication; + default -> unsupported : SOCKS5_Unsupported_Authentication_Method; }; -type SOCKS5_Unsupported_Authentication = record { +type SOCKS5_Unsupported_Authentication_Method = record { + crap: bytestring &restofdata; +}; + +type SOCKS5_Unsupported_Authentication_Version(version: uint8) = record { crap: bytestring &restofdata; }; type SOCKS5_Auth_Request_UserPass = record { + version: uint8; + msg: case version of { + 1 -> v1: SOCKS5_Auth_Request_UserPass_v1; + default -> unsupported: SOCKS5_Unsupported_Authentication_Version(version); + }; +}; + +type SOCKS5_Auth_Request_UserPass_v1 = record { ulen : uint8; username : bytestring &length=ulen; plen : uint8; password : bytestring &length=plen; }; -type SOCKS5_Auth_Reply = record { +type SOCKS5_Auth_Reply = case $context.connection.v5_auth_method() of { + 0x02 -> userpass : SOCKS5_Auth_Reply_UserPass; + default -> unsupported : SOCKS5_Unsupported_Authentication_Method; +} &let { + in_auth_sub_neg = $context.connection.set_v5_in_auth_sub_negotiation(false); +}; + +type SOCKS5_Auth_Reply_UserPass = record { + version: uint8; + msg: case version of { + 1 -> v1: SOCKS5_Auth_Reply_UserPass_v1; + default -> unsupported: SOCKS5_Unsupported_Authentication_Version(version); + }; +}; + +type SOCKS5_Auth_Reply_UserPass_v1 = record { code : uint8; }; @@ -126,15 +158,28 @@ type SOCKS4_Reply = record { refine connection SOCKS_Conn += { %member{ + bool v5_in_auth_sub_negotiation_; bool v5_authenticated_; uint8 selected_auth_method_; %} %init{ + v5_in_auth_sub_negotiation_ = false; v5_authenticated_ = false; selected_auth_method_ = 255; %} + function v5_in_auth_sub_negotiation(): bool + %{ + return v5_in_auth_sub_negotiation_; + %} + + function set_v5_in_auth_sub_negotiation(b: bool): bool + %{ + v5_in_auth_sub_negotiation_ = b; + return true; + %} + function v5_past_authentication(): bool %{ return v5_authenticated_; diff --git a/src/analyzer/protocol/socks/socks.pac b/src/analyzer/protocol/socks/socks.pac index a9c4099508..9aed2820af 100644 --- a/src/analyzer/protocol/socks/socks.pac +++ b/src/analyzer/protocol/socks/socks.pac @@ -20,7 +20,7 @@ connection SOCKS_Conn(bro_analyzer: BroAnalyzer) { %include socks-protocol.pac flow SOCKS_Flow(is_orig: bool) { - datagram = SOCKS_Version(is_orig) withcontext(connection, this); + datagram = SOCKS_Message(is_orig) withcontext(connection, this); }; %include socks-analyzer.pac diff --git a/testing/btest/Baseline/plugins.writer/output b/testing/btest/Baseline/plugins.writer/output index 0882718f03..f7b33992ea 100644 --- a/testing/btest/Baseline/plugins.writer/output +++ b/testing/btest/Baseline/plugins.writer/output @@ -17,6 +17,6 @@ Demo::Foo - A Foo test logging writer (dynamic, version 1.0) [http] 1340213020.732963|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|GET|www.osnews.com|/images/icons/17.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- [http] 1340213021.300269|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|6|GET|www.osnews.com|/images/left.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- [http] 1340213021.861584|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|7|GET|www.osnews.com|/images/icons/32.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- -[packet_filter] 1412721099.419280|bro|ip or not ip|T|T -[socks] 1340213015.276495|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|-|succeeded|-|www.osnews.com|80|192.168.0.31|-|2688 +[packet_filter] 1423781675.402129|bro|ip or not ip|T|T +[socks] 1340213015.276495|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|-|-|succeeded|-|www.osnews.com|80|192.168.0.31|-|2688 [tunnel] 1340213015.276495|-|10.0.0.55|0|60.190.189.214|8124|Tunnel::SOCKS|Tunnel::DISCOVER From 8e4f4b46f7591a3779adc8e21d688f81ee436721 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Feb 2015 16:23:43 -0600 Subject: [PATCH 10/11] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index d43cc790e5..9b6dd56242 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit d43cc790e5b8709b5e032e52ad0e00936494739b +Subproject commit 9b6dd5624254de9d18618562887979da1158da43 From 4bcb9d2d920862660feba80cfb13356952843201 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Feb 2015 18:04:17 -0600 Subject: [PATCH 11/11] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index 9b6dd56242..420c5b42c0 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 9b6dd5624254de9d18618562887979da1158da43 +Subproject commit 420c5b42c0c90f22fc7a862fc491c8e554d05381