HTTP: Recognize and skip upgrade/websocket connections.

This adds a slight patch to the HTTP analyzer, which recognizez when a connection is
upgraded to a different protocol (using a 101 reply with a few specific headers being
set).

In this case, the analyzer stops further processing of the connection (which will
result in DPD errors) and raises a new event:

event http_connection_upgrade(c: connection, protocol: string);

Protocol contains the name of the protocol that is being upgraded to, as specified in
one of the header values.

src/analyzer/protocol/http/HTTP.cc

@@ -822,6 +822,9 @@ HTTP_Analyzer::HTTP_Analyzer(Connection* conn)
 
 	connect_request = false;
 	pia = 0;
+	upgraded = false;
+	upgrade_connection = false;
+	upgrade_protocol.clear();
 
 	content_line_orig = new tcp::ContentLine_Analyzer(conn, true);
 	AddSupportAnalyzer(content_line_orig);
@@ -879,6 +882,9 @@ void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig)
 	if ( TCP() && TCP()->IsPartial() )
 		return;
 
+	if ( upgraded )
+		return;
+
 	if ( pia )
 		{
 		// There will be a PIA instance if this connection has been identified
@@ -1468,15 +1474,35 @@ void HTTP_Analyzer::ReplyMade(const int interrupted, const char* msg)
 		unanswered_requests.pop();
 		}
 
-	reply_code = 0;
-
 	if ( reply_reason_phrase )
 		{
 		Unref(reply_reason_phrase);
 		reply_reason_phrase = 0;
 		}
 
-	if ( interrupted )
+	// unanswered requests = 1 because there is no pop after 101.
+	if ( reply_code == 101 && unanswered_requests.size() == 1 && upgrade_connection &&
+	     upgrade_protocol.size() )
+		{
+		// Upgraded connection that switches immediately - e.g. websocket.
+		upgraded = true;
+		RemoveSupportAnalyzer(content_line_orig);
+		RemoveSupportAnalyzer(content_line_resp);
+
+		if ( http_connection_upgrade )
+			{
+			val_list* vl = new val_list();
+			vl->append(BuildConnVal());
+			vl->append(new StringVal(upgrade_protocol));
+			ConnectionEvent(http_connection_upgrade, vl);
+			}
+		}
+
+	reply_code = 0;
+	upgrade_connection = false;
+	upgrade_protocol.clear();
+
+	if ( interrupted || upgraded )
 		reply_state = EXPECT_REPLY_NOTHING;
 	else
 		reply_state = EXPECT_REPLY_LINE;
@@ -1614,8 +1640,14 @@ void HTTP_Analyzer::HTTP_Header(int is_orig, mime::MIME_Header* h)
 		{
 		if ( mime::strcasecmp_n(h->get_value_token(), "close") == 0 )
 			connection_close = 1;
+		else if ( mime::strcasecmp_n(h->get_value_token(), "upgrade") == 0 )
+			upgrade_connection = true;
 		}
 
+	if ( ! is_orig &&
+	     mime::strcasecmp_n(h->get_name(), "upgrade") == 0 )
+	     upgrade_protocol.assign(h->get_value_token().data, h->get_value_token().length);
+
 	if ( http_header )
 		{
 		Rule::PatternType rule =

src/analyzer/protocol/http/HTTP.h

@@ -40,8 +40,8 @@ public:
 			{ zip->Done(); delete zip; }
 		}
 
-	void EndOfData();
+	void EndOfData() override;
-	void Deliver(int len, const char* data, int trailing_CRLF);
+	void Deliver(int len, const char* data, int trailing_CRLF) override;
 	int Undelivered(int64_t len);
 	int64_t BodyLength() const 		{ return body_length; }
 	int64_t HeaderLength() const 	{ return header_length; }
@@ -68,17 +68,17 @@ protected:
 	bool send_size; // whether to send size indication to FAF
 	std::string precomputed_file_id;
 
-	MIME_Entity* NewChildEntity() { return new HTTP_Entity(http_message, this, 1); }
+	MIME_Entity* NewChildEntity() override { return new HTTP_Entity(http_message, this, 1); }
 
 	void DeliverBody(int len, const char* data, int trailing_CRLF);
 	void DeliverBodyClear(int len, const char* data, int trailing_CRLF);
 
-	void SubmitData(int len, const char* buf);
+	void SubmitData(int len, const char* buf) override;
 
 	void SetPlainDelivery(int64_t length);
 
-	void SubmitHeader(mime::MIME_Header* h);
+	void SubmitHeader(mime::MIME_Header* h) override;
-	void SubmitAllHeaders();
+	void SubmitAllHeaders() override;
 };
 
 enum {
@@ -106,18 +106,18 @@ public:
 			 bool is_orig, int expect_body, int64_t init_header_length);
 	~HTTP_Message();
 	void Done(const int interrupted, const char* msg);
-	void Done() { Done(0, "message ends normally"); }
+	void Done() override { Done(0, "message ends normally"); }
 
 	int Undelivered(int64_t len);
 
-	void BeginEntity(mime::MIME_Entity* /* entity */);
+	void BeginEntity(mime::MIME_Entity* /* entity */) override;
-	void EndEntity(mime::MIME_Entity* entity);
+	void EndEntity(mime::MIME_Entity* entity) override;
-	void SubmitHeader(mime::MIME_Header* h);
+	void SubmitHeader(mime::MIME_Header* h) override;
-	void SubmitAllHeaders(mime::MIME_HeaderList& /* hlist */);
+	void SubmitAllHeaders(mime::MIME_HeaderList& /* hlist */) override;
-	void SubmitData(int len, const char* buf);
+	void SubmitData(int len, const char* buf) override;
-	int RequestBuffer(int* plen, char** pbuf);
+	int RequestBuffer(int* plen, char** pbuf) override;
 	void SubmitAllData();
-	void SubmitEvent(int event_type, const char* detail);
+	void SubmitEvent(int event_type, const char* detail) override;
 
 	void SubmitTrailingHeaders(mime::MIME_HeaderList& /* hlist */);
 	void SetPlainDelivery(int64_t length);
@@ -169,15 +169,15 @@ public:
 	int HTTP_ReplyCode() const { return reply_code; };
 
 	// Overriden from Analyzer.
-	virtual void Done();
+	void Done() override;
-	virtual void DeliverStream(int len, const u_char* data, bool orig);
+	void DeliverStream(int len, const u_char* data, bool orig) override;
-	virtual void Undelivered(uint64 seq, int len, bool orig);
+	void Undelivered(uint64 seq, int len, bool orig) override;
 
 	// Overriden from tcp::TCP_ApplicationAnalyzer
-	virtual void EndpointEOF(bool is_orig);
+	void EndpointEOF(bool is_orig) override;
-	virtual void ConnectionFinished(int half_finished);
+	void ConnectionFinished(int half_finished) override;
-	virtual void ConnectionReset();
+	void ConnectionReset() override;
-	virtual void PacketWithRST();
+	void PacketWithRST() override;
 
 	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new HTTP_Analyzer(conn); }
@@ -234,6 +234,13 @@ protected:
 
 	bool connect_request;
 	pia::PIA_TCP *pia;
+	// set to true after a connection was upgraded
+	bool upgraded;
+	// set to true when encountering an "connection" header in a reply.
+	bool upgrade_connection;
+	// set to the protocol string when encountering an "upgrade" header
+	// in a reply.
+	std::string upgrade_protocol;
 
 	Val* request_method;
 

src/analyzer/protocol/http/events.bif

@@ -19,7 +19,7 @@
 ##
 ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity
 ##    http_entity_data http_event http_header http_message_done http_reply http_stats
-##    truncate_http_URI
+##    truncate_http_URI http_connection_upgrade
 event http_request%(c: connection, method: string, original_URI: string, unescaped_URI: string, version: string%);
 
 ## Generated for HTTP replies. Bro supports persistent and pipelined HTTP
@@ -40,7 +40,7 @@ event http_request%(c: connection, method: string, original_URI: string, unescap
 ##
 ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity
 ##    http_entity_data http_event http_header http_message_done http_request
-##    http_stats
+##    http_stats http_connection_upgrade
 event http_reply%(c: connection, version: string, code: count, reason: string%);
 
 ## Generated for HTTP headers. Bro supports persistent and pipelined HTTP
@@ -60,7 +60,7 @@ event http_reply%(c: connection, version: string, code: count, reason: string%);
 ##
 ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity
 ##    http_entity_data http_event  http_message_done http_reply http_request
-##    http_stats
+##    http_stats http_connection_upgrade
 ##
 ## .. note:: This event is also raised for headers found in nested body
 ##    entities.
@@ -83,6 +83,7 @@ event http_header%(c: connection, is_orig: bool, name: string, value: string%);
 ##
 ## .. bro:see::  http_begin_entity http_content_type http_end_entity http_entity_data
 ##    http_event http_header http_message_done http_reply http_request http_stats
+##    http_connection_upgrade
 ##
 ## .. note:: This event is also raised for headers found in nested body
 ##    entities.
@@ -104,7 +105,7 @@ event http_all_headers%(c: connection, is_orig: bool, hlist: mime_header_list%);
 ##
 ## .. bro:see:: http_all_headers  http_content_type http_end_entity http_entity_data
 ##    http_event http_header http_message_done http_reply http_request http_stats
-##    mime_begin_entity
+##    mime_begin_entity http_connection_upgrade
 event http_begin_entity%(c: connection, is_orig: bool%);
 
 ## Generated when finishing parsing an HTTP body entity. This event is generated
@@ -123,7 +124,7 @@ event http_begin_entity%(c: connection, is_orig: bool%);
 ##
 ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_entity_data
 ##    http_event http_header http_message_done http_reply http_request
-##    http_stats   mime_end_entity
+##    http_stats mime_end_entity http_connection_upgrade
 event http_end_entity%(c: connection, is_orig: bool%);
 
 ## Generated when parsing an HTTP body entity, passing on the data. This event
@@ -152,6 +153,7 @@ event http_end_entity%(c: connection, is_orig: bool%);
 ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity
 ##    http_event http_header http_message_done http_reply http_request http_stats
 ##    mime_entity_data http_entity_data_delivery_size skip_http_data
+##    http_connection_upgrade
 event http_entity_data%(c: connection, is_orig: bool, length: count, data: string%);
 
 ## Generated for reporting an HTTP body's content type.  This event is
@@ -173,6 +175,7 @@ event http_entity_data%(c: connection, is_orig: bool, length: count, data: strin
 ##
 ## .. bro:see:: http_all_headers http_begin_entity  http_end_entity http_entity_data
 ##    http_event http_header http_message_done http_reply http_request http_stats
+##    http_connection_upgrade
 ##
 ## .. note:: This event is also raised for headers found in nested body
 ##    entities.
@@ -198,6 +201,7 @@ event http_content_type%(c: connection, is_orig: bool, ty: string, subty: string
 ##
 ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity
 ##    http_entity_data http_event http_header  http_reply http_request http_stats
+##    http_connection_upgrade
 event http_message_done%(c: connection, is_orig: bool, stat: http_message_stat%);
 
 ## Generated for errors found when decoding HTTP requests or replies.
@@ -214,7 +218,7 @@ event http_message_done%(c: connection, is_orig: bool, stat: http_message_stat%)
 ##
 ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity
 ##    http_entity_data  http_header http_message_done http_reply http_request
-##    http_stats mime_event
+##    http_stats mime_event http_connection_upgrade
 event http_event%(c: connection, event_type: string, detail: string%);
 
 ## Generated at the end of an HTTP session to report statistics about it. This
@@ -228,5 +232,18 @@ event http_event%(c: connection, event_type: string, detail: string%);
 ##
 ## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity
 ##    http_entity_data http_event http_header http_message_done http_reply
-##    http_request
+##    http_request http_connection_upgrade
 event http_stats%(c: connection, stats: http_stats_rec%);
+
+## Generated when a HTTP session is upgraded to a different protocol (e.g. websocket).
+## This event is raised when a server replies with a HTTP 101 reply. No more HTTP events
+## will be raised after this event.
+##
+## c: The connection.
+##
+## protocol: The protocol to which the connection is switching.
+##
+## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity
+##    http_entity_data http_event http_header http_message_done http_reply
+##    http_request
+event http_connection_upgrade%(c: connection, protocol: string%);

src/analyzer/protocol/mime/MIME.h

@@ -232,16 +232,16 @@ class MIME_Mail : public MIME_Message {
 public:
 	MIME_Mail(analyzer::Analyzer* mail_conn, bool is_orig, int buf_size = 0);
 	~MIME_Mail();
-	void Done();
+	void Done() override;
 
-	void BeginEntity(MIME_Entity* entity);
+	void BeginEntity(MIME_Entity* entity) override;
-	void EndEntity(MIME_Entity* entity);
+	void EndEntity(MIME_Entity* entity) override;
-	void SubmitHeader(MIME_Header* h);
+	void SubmitHeader(MIME_Header* h) override;
-	void SubmitAllHeaders(MIME_HeaderList& hlist);
+	void SubmitAllHeaders(MIME_HeaderList& hlist) override;
-	void SubmitData(int len, const char* buf);
+	void SubmitData(int len, const char* buf) override;
-	int RequestBuffer(int* plen, char** pbuf);
+	int RequestBuffer(int* plen, char** pbuf) override;
 	void SubmitAllData();
-	void SubmitEvent(int event_type, const char* detail);
+	void SubmitEvent(int event_type, const char* detail) override;
 	void Undelivered(int len);
 
 protected:

testing/btest/Baseline/scripts.base.protocols.http.101-switching-protocols/.stdout

@@ -0,0 +1 @@
+Connection upgraded to websocket

testing/btest/Baseline/scripts.base.protocols.http.101-switching-protocols/http.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#open	2017-08-04-00-45-31
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	version	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	tags	username	password	proxied	orig_fuids	orig_filenames	orig_mime_types	resp_fuids	resp_filenames	resp_mime_types
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	string	count	count	count	string	count	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]
+1501770877.501001	CHhAvVGS1DHFjwGM9	192.168.0.5	50798	54.148.114.85	80	1	GET	sandbox.kaazing.net	/echo?.kl=Y	-	1.1	Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:54.0) Gecko/20100101 Firefox/54.0	0	0	101	Web Socket Protocol Handshake	101	Web Socket Protocol Handshake	(empty)	-	-	-	-	-	-	-	-	-
+#close	2017-08-04-00-45-31

testing/btest/scripts/base/protocols/http/101-switching-protocols.bro

@@ -0,0 +1,13 @@
+# This tests that the HTTP analyzer does not generate a dpd error as a
+# result of seeing an upgraded connection.
+#
+# @TEST-EXEC: bro -r $TRACES/http/websocket.pcap %INPUT
+# @TEST-EXEC: test ! -f dpd.log
+# @TEST-EXEC: test ! -f weird.log
+# @TEST-EXEC: btest-diff http.log
+# @TEST-EXEC: btest-diff .stdout
+
+event http_connection_upgrade(c: connection, protocol: string)
+	{
+	print fmt("Connection upgraded to %s", protocol);
+	}