From eab80c8834a5c60786b563c5eba3d33fd31cb93b Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 4 Aug 2017 07:04:28 -0700 Subject: [PATCH] HTTP: Recognize and skip upgrade/websocket connections. This adds a slight patch to the HTTP analyzer, which recognizez when a connection is upgraded to a different protocol (using a 101 reply with a few specific headers being set). In this case, the analyzer stops further processing of the connection (which will result in DPD errors) and raises a new event: event http_connection_upgrade(c: connection, protocol: string); Protocol contains the name of the protocol that is being upgraded to, as specified in one of the header values. --- src/analyzer/protocol/http/HTTP.cc | 42 +++++++++++++-- src/analyzer/protocol/http/HTTP.h | 49 ++++++++++-------- src/analyzer/protocol/http/events.bif | 31 ++++++++--- src/analyzer/protocol/mime/MIME.h | 18 +++---- .../.stdout | 1 + .../http.log | 10 ++++ testing/btest/Traces/http/websocket.pcap | Bin 0 -> 4224 bytes .../http/101-switching-protocols.bro | 13 +++++ 8 files changed, 122 insertions(+), 42 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.http.101-switching-protocols/.stdout create mode 100644 testing/btest/Baseline/scripts.base.protocols.http.101-switching-protocols/http.log create mode 100644 testing/btest/Traces/http/websocket.pcap create mode 100644 testing/btest/scripts/base/protocols/http/101-switching-protocols.bro diff --git a/src/analyzer/protocol/http/HTTP.cc b/src/analyzer/protocol/http/HTTP.cc index badf8c1b54..65a511b8cb 100644 --- a/src/analyzer/protocol/http/HTTP.cc +++ b/src/analyzer/protocol/http/HTTP.cc @@ -822,6 +822,9 @@ HTTP_Analyzer::HTTP_Analyzer(Connection* conn) connect_request = false; pia = 0; + upgraded = false; + upgrade_connection = false; + upgrade_protocol.clear(); content_line_orig = new tcp::ContentLine_Analyzer(conn, true); AddSupportAnalyzer(content_line_orig); @@ -879,6 +882,9 @@ void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) if ( TCP() && TCP()->IsPartial() ) return; + if ( upgraded ) + return; + if ( pia ) { // There will be a PIA instance if this connection has been identified @@ -1468,15 +1474,35 @@ void HTTP_Analyzer::ReplyMade(const int interrupted, const char* msg) unanswered_requests.pop(); } - reply_code = 0; - if ( reply_reason_phrase ) { Unref(reply_reason_phrase); reply_reason_phrase = 0; } - if ( interrupted ) + // unanswered requests = 1 because there is no pop after 101. + if ( reply_code == 101 && unanswered_requests.size() == 1 && upgrade_connection && + upgrade_protocol.size() ) + { + // Upgraded connection that switches immediately - e.g. websocket. + upgraded = true; + RemoveSupportAnalyzer(content_line_orig); + RemoveSupportAnalyzer(content_line_resp); + + if ( http_connection_upgrade ) + { + val_list* vl = new val_list(); + vl->append(BuildConnVal()); + vl->append(new StringVal(upgrade_protocol)); + ConnectionEvent(http_connection_upgrade, vl); + } + } + + reply_code = 0; + upgrade_connection = false; + upgrade_protocol.clear(); + + if ( interrupted || upgraded ) reply_state = EXPECT_REPLY_NOTHING; else reply_state = EXPECT_REPLY_LINE; @@ -1611,11 +1637,17 @@ void HTTP_Analyzer::HTTP_Header(int is_orig, mime::MIME_Header* h) if ( ! is_orig && mime::strcasecmp_n(h->get_name(), "connection") == 0 ) - { + { if ( mime::strcasecmp_n(h->get_value_token(), "close") == 0 ) - connection_close = 1; + connection_close = 1; + else if ( mime::strcasecmp_n(h->get_value_token(), "upgrade") == 0 ) + upgrade_connection = true; } + if ( ! is_orig && + mime::strcasecmp_n(h->get_name(), "upgrade") == 0 ) + upgrade_protocol.assign(h->get_value_token().data, h->get_value_token().length); + if ( http_header ) { Rule::PatternType rule = diff --git a/src/analyzer/protocol/http/HTTP.h b/src/analyzer/protocol/http/HTTP.h index d55c10c4c1..bfc079187f 100644 --- a/src/analyzer/protocol/http/HTTP.h +++ b/src/analyzer/protocol/http/HTTP.h @@ -40,8 +40,8 @@ public: { zip->Done(); delete zip; } } - void EndOfData(); - void Deliver(int len, const char* data, int trailing_CRLF); + void EndOfData() override; + void Deliver(int len, const char* data, int trailing_CRLF) override; int Undelivered(int64_t len); int64_t BodyLength() const { return body_length; } int64_t HeaderLength() const { return header_length; } @@ -68,17 +68,17 @@ protected: bool send_size; // whether to send size indication to FAF std::string precomputed_file_id; - MIME_Entity* NewChildEntity() { return new HTTP_Entity(http_message, this, 1); } + MIME_Entity* NewChildEntity() override { return new HTTP_Entity(http_message, this, 1); } void DeliverBody(int len, const char* data, int trailing_CRLF); void DeliverBodyClear(int len, const char* data, int trailing_CRLF); - void SubmitData(int len, const char* buf); + void SubmitData(int len, const char* buf) override; void SetPlainDelivery(int64_t length); - void SubmitHeader(mime::MIME_Header* h); - void SubmitAllHeaders(); + void SubmitHeader(mime::MIME_Header* h) override; + void SubmitAllHeaders() override; }; enum { @@ -106,18 +106,18 @@ public: bool is_orig, int expect_body, int64_t init_header_length); ~HTTP_Message(); void Done(const int interrupted, const char* msg); - void Done() { Done(0, "message ends normally"); } + void Done() override { Done(0, "message ends normally"); } int Undelivered(int64_t len); - void BeginEntity(mime::MIME_Entity* /* entity */); - void EndEntity(mime::MIME_Entity* entity); - void SubmitHeader(mime::MIME_Header* h); - void SubmitAllHeaders(mime::MIME_HeaderList& /* hlist */); - void SubmitData(int len, const char* buf); - int RequestBuffer(int* plen, char** pbuf); + void BeginEntity(mime::MIME_Entity* /* entity */) override; + void EndEntity(mime::MIME_Entity* entity) override; + void SubmitHeader(mime::MIME_Header* h) override; + void SubmitAllHeaders(mime::MIME_HeaderList& /* hlist */) override; + void SubmitData(int len, const char* buf) override; + int RequestBuffer(int* plen, char** pbuf) override; void SubmitAllData(); - void SubmitEvent(int event_type, const char* detail); + void SubmitEvent(int event_type, const char* detail) override; void SubmitTrailingHeaders(mime::MIME_HeaderList& /* hlist */); void SetPlainDelivery(int64_t length); @@ -169,15 +169,15 @@ public: int HTTP_ReplyCode() const { return reply_code; }; // Overriden from Analyzer. - virtual void Done(); - virtual void DeliverStream(int len, const u_char* data, bool orig); - virtual void Undelivered(uint64 seq, int len, bool orig); + void Done() override; + void DeliverStream(int len, const u_char* data, bool orig) override; + void Undelivered(uint64 seq, int len, bool orig) override; // Overriden from tcp::TCP_ApplicationAnalyzer - virtual void EndpointEOF(bool is_orig); - virtual void ConnectionFinished(int half_finished); - virtual void ConnectionReset(); - virtual void PacketWithRST(); + void EndpointEOF(bool is_orig) override; + void ConnectionFinished(int half_finished) override; + void ConnectionReset() override; + void PacketWithRST() override; static analyzer::Analyzer* Instantiate(Connection* conn) { return new HTTP_Analyzer(conn); } @@ -234,6 +234,13 @@ protected: bool connect_request; pia::PIA_TCP *pia; + // set to true after a connection was upgraded + bool upgraded; + // set to true when encountering an "connection" header in a reply. + bool upgrade_connection; + // set to the protocol string when encountering an "upgrade" header + // in a reply. + std::string upgrade_protocol; Val* request_method; diff --git a/src/analyzer/protocol/http/events.bif b/src/analyzer/protocol/http/events.bif index 7a509c6d54..ab005ba8d6 100644 --- a/src/analyzer/protocol/http/events.bif +++ b/src/analyzer/protocol/http/events.bif @@ -19,7 +19,7 @@ ## ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity ## http_entity_data http_event http_header http_message_done http_reply http_stats -## truncate_http_URI +## truncate_http_URI http_connection_upgrade event http_request%(c: connection, method: string, original_URI: string, unescaped_URI: string, version: string%); ## Generated for HTTP replies. Bro supports persistent and pipelined HTTP @@ -40,7 +40,7 @@ event http_request%(c: connection, method: string, original_URI: string, unescap ## ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity ## http_entity_data http_event http_header http_message_done http_request -## http_stats +## http_stats http_connection_upgrade event http_reply%(c: connection, version: string, code: count, reason: string%); ## Generated for HTTP headers. Bro supports persistent and pipelined HTTP @@ -60,7 +60,7 @@ event http_reply%(c: connection, version: string, code: count, reason: string%); ## ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity ## http_entity_data http_event http_message_done http_reply http_request -## http_stats +## http_stats http_connection_upgrade ## ## .. note:: This event is also raised for headers found in nested body ## entities. @@ -83,6 +83,7 @@ event http_header%(c: connection, is_orig: bool, name: string, value: string%); ## ## .. bro:see:: http_begin_entity http_content_type http_end_entity http_entity_data ## http_event http_header http_message_done http_reply http_request http_stats +## http_connection_upgrade ## ## .. note:: This event is also raised for headers found in nested body ## entities. @@ -104,7 +105,7 @@ event http_all_headers%(c: connection, is_orig: bool, hlist: mime_header_list%); ## ## .. bro:see:: http_all_headers http_content_type http_end_entity http_entity_data ## http_event http_header http_message_done http_reply http_request http_stats -## mime_begin_entity +## mime_begin_entity http_connection_upgrade event http_begin_entity%(c: connection, is_orig: bool%); ## Generated when finishing parsing an HTTP body entity. This event is generated @@ -123,7 +124,7 @@ event http_begin_entity%(c: connection, is_orig: bool%); ## ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_entity_data ## http_event http_header http_message_done http_reply http_request -## http_stats mime_end_entity +## http_stats mime_end_entity http_connection_upgrade event http_end_entity%(c: connection, is_orig: bool%); ## Generated when parsing an HTTP body entity, passing on the data. This event @@ -152,6 +153,7 @@ event http_end_entity%(c: connection, is_orig: bool%); ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity ## http_event http_header http_message_done http_reply http_request http_stats ## mime_entity_data http_entity_data_delivery_size skip_http_data +## http_connection_upgrade event http_entity_data%(c: connection, is_orig: bool, length: count, data: string%); ## Generated for reporting an HTTP body's content type. This event is @@ -173,6 +175,7 @@ event http_entity_data%(c: connection, is_orig: bool, length: count, data: strin ## ## .. bro:see:: http_all_headers http_begin_entity http_end_entity http_entity_data ## http_event http_header http_message_done http_reply http_request http_stats +## http_connection_upgrade ## ## .. note:: This event is also raised for headers found in nested body ## entities. @@ -198,6 +201,7 @@ event http_content_type%(c: connection, is_orig: bool, ty: string, subty: string ## ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity ## http_entity_data http_event http_header http_reply http_request http_stats +## http_connection_upgrade event http_message_done%(c: connection, is_orig: bool, stat: http_message_stat%); ## Generated for errors found when decoding HTTP requests or replies. @@ -214,7 +218,7 @@ event http_message_done%(c: connection, is_orig: bool, stat: http_message_stat%) ## ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity ## http_entity_data http_header http_message_done http_reply http_request -## http_stats mime_event +## http_stats mime_event http_connection_upgrade event http_event%(c: connection, event_type: string, detail: string%); ## Generated at the end of an HTTP session to report statistics about it. This @@ -228,5 +232,18 @@ event http_event%(c: connection, event_type: string, detail: string%); ## ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity ## http_entity_data http_event http_header http_message_done http_reply -## http_request +## http_request http_connection_upgrade event http_stats%(c: connection, stats: http_stats_rec%); + +## Generated when a HTTP session is upgraded to a different protocol (e.g. websocket). +## This event is raised when a server replies with a HTTP 101 reply. No more HTTP events +## will be raised after this event. +## +## c: The connection. +## +## protocol: The protocol to which the connection is switching. +## +## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity +## http_entity_data http_event http_header http_message_done http_reply +## http_request +event http_connection_upgrade%(c: connection, protocol: string%); diff --git a/src/analyzer/protocol/mime/MIME.h b/src/analyzer/protocol/mime/MIME.h index 8c7fdd4326..fcc921993d 100644 --- a/src/analyzer/protocol/mime/MIME.h +++ b/src/analyzer/protocol/mime/MIME.h @@ -232,16 +232,16 @@ class MIME_Mail : public MIME_Message { public: MIME_Mail(analyzer::Analyzer* mail_conn, bool is_orig, int buf_size = 0); ~MIME_Mail(); - void Done(); + void Done() override; - void BeginEntity(MIME_Entity* entity); - void EndEntity(MIME_Entity* entity); - void SubmitHeader(MIME_Header* h); - void SubmitAllHeaders(MIME_HeaderList& hlist); - void SubmitData(int len, const char* buf); - int RequestBuffer(int* plen, char** pbuf); + void BeginEntity(MIME_Entity* entity) override; + void EndEntity(MIME_Entity* entity) override; + void SubmitHeader(MIME_Header* h) override; + void SubmitAllHeaders(MIME_HeaderList& hlist) override; + void SubmitData(int len, const char* buf) override; + int RequestBuffer(int* plen, char** pbuf) override; void SubmitAllData(); - void SubmitEvent(int event_type, const char* detail); + void SubmitEvent(int event_type, const char* detail) override; void Undelivered(int len); protected: @@ -283,6 +283,6 @@ extern int MIME_get_value(int len, const char* data, BroString*& buf, extern int MIME_get_field_name(int len, const char* data, data_chunk_t* name); extern BroString* MIME_decode_quoted_pairs(data_chunk_t buf); -} } // namespace analyzer::* +} } // namespace analyzer::* #endif diff --git a/testing/btest/Baseline/scripts.base.protocols.http.101-switching-protocols/.stdout b/testing/btest/Baseline/scripts.base.protocols.http.101-switching-protocols/.stdout new file mode 100644 index 0000000000..0e9c739b00 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.101-switching-protocols/.stdout @@ -0,0 +1 @@ +Connection upgraded to websocket diff --git a/testing/btest/Baseline/scripts.base.protocols.http.101-switching-protocols/http.log b/testing/btest/Baseline/scripts.base.protocols.http.101-switching-protocols/http.log new file mode 100644 index 0000000000..a1e96de25e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.101-switching-protocols/http.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path http +#open 2017-08-04-00-45-31 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types +#types time string addr port addr port count string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] +1501770877.501001 CHhAvVGS1DHFjwGM9 192.168.0.5 50798 54.148.114.85 80 1 GET sandbox.kaazing.net /echo?.kl=Y - 1.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:54.0) Gecko/20100101 Firefox/54.0 0 0 101 Web Socket Protocol Handshake 101 Web Socket Protocol Handshake (empty) - - - - - - - - - +#close 2017-08-04-00-45-31 diff --git a/testing/btest/Traces/http/websocket.pcap b/testing/btest/Traces/http/websocket.pcap new file mode 100644 index 0000000000000000000000000000000000000000..0a71c8a77dc92fabb9b265487d1b716ce72cee66 GIT binary patch literal 4224 zcmeH~eNYr-9LFCH^xy^z3yG|zN2#fdy*EzO6+*9?5|({jMGB-A*%E4WDs*k@wk!d7p+I@13eSkx%LfSuP!2 zPL|K??0ffRGNWq0(tb)Jt*7_34GI{HQ9?|4GhWNd&7F~xlW!@E&VLT?<(cC2 zbBE339{7$U&W-N@bV{mW1H#N4*U7j8X+0(Fj_@;KhaHEm_MIK`$LV>de9Oq5u|lN7 zPG+=w!HAjM1ymeqcDtIsMZ0&p4R-GUx_O|uc#iiMLMH5e6Uj&{c*#r#!EPKzqDfmk zio~Z^W+avs&pBpECDzxqQ5)~^$u-s(AL&!b66ckmIhhXSgLYTSEn4d_HbNO zki1rj*NQCdDz7lLUS5I;&2m%_Mb5U+YNxj~a~?s`WYxEnu97rfq;P?@xam6Tuv;C@ zrBn&B3td+G?X;fv^vE`+-C>762VE&Be7797;msmTt;fUrfvxdj&F0g3#R|^v7X=Te z39@7hynBEwgvj9#R(A8#7W)&=jB<-OH8}Ff|v9O3zHM-B#+z$ z?y=Mx75o*ni|-aWjW4pec@NXUce-Uy53e!z@`?%;u+&jmWNA?ZuK=HYn&xM1wnTtr z#cN#Fs>qt`kwuncSc?;Vvpm{k+o4t14K|Ok+_aA7N>*yRy?Q zhPSL*vUZ^?tM_z=nr(~MEL&D&Ss_aj@4=|Cw1?;Y3?~XfzJj*lAze z>M@_**Ku;7n0%+5iB}}l_cfrLH<7p|J8_Oqyl^GNXK6lp7!-rUXrn0-D@EeQV;PBM zUwLV?rfR)(#5wLE{BN6PczBlj+4`R@FpAnOaelL(B?=?*jP|iyz+ontPX^>RQF~XiDi~MAl`Qd3C}Ns5 z*`zhKq?2WQH;9#`ZN5MSwO7*GfR{pNSwtN!wsH~caMJqbHlrvLJ;uI4tO_bst5;~U zQY-ZaSE`Y!#ZsqR3)X>vQuUum@3cZzAQmibjQ2`f4_1Y^2=vRi+%ib#43s6x28Xek z6Ibqw*vvnBG_{#`oZKDpHo@Z|*VCG8=HGyPN>DoQ%*!VYK*M1ybi&52)unSjwa~py zx)`C*YmcQC`e5E4JC3hL?zh05-cqIpEj|2u220oKmco;D;z5?Qv^rKmz62|Ar1}0) z(-Zomu^=O{tT^}F#ssn4iN0^YL>@bT^{g9S@7Pd%Hq@&PJ6AT0+%=E+p***AG$ghR z_wkXPHJha2Zf@J~R#)}KvilH2t^%-Hgmq=KF!9h>5b=*As=f`?_1X~P)5eLj3x8oER2>r2e+Em{NH@0w5q$+lu-ZuV9EmL7MQKr9M zGL)%)W`7cxSz6Ito%x%Ac_hKSbH0I@Y(?Y1oxT-4{0TOWRh_AgV_nDYh%h=(eAW5; z&n6p3D>M!#iu9+f#22<9(e_FP;t`!Vae@JXCW*IQ%4TE18%X>xkb(HBPW&p>#;suE zBf|w`J!A(*TDxo%?MUqUIU}*G7~hg68t|=i(d+GdH-5U`Tyo?>PvzKatxI0{&{s41 zr`nGjotR@G?JyljS}D}WuGiCc){wSWA#D8kM>312tfwwN<1jOg^hPRU5EE?JKqjdm?U*9q-S#4?YVQ1?RKQ3n^ zLVWHxoFLlGNZd8BcBt_tVX*_E{Q=8-+;L?j-ZLQG4VD|Vx9rQxwJysuk1n~ZNdo@yl$ z-=0EDi>GV!AZF