HTTP CONNECT proxy support.

- The HTTP analyzer now supports handling HTTP CONNECT proxies
   same as the SOCKS analyzer handles proxying.

scripts/base/protocols/http/main.bro

@@ -4,6 +4,7 @@
 
 @load base/utils/numbers
 @load base/utils/files
+@load base/frameworks/tunnels
 
 module HTTP;
 
@@ -217,6 +218,16 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p
 		c$http$info_code = code;
 		c$http$info_msg = reason;
 		}
+	
+	if ( c$http?$method && c$http$method == "CONNECT" && code == 200 )
+		{
+		# Copy this conn_id and set the orig_p to zero because in the case of CONNECT proxies there will
+		# be potentially many source ports since a new proxy connection is established for each
+		# proxied connection.  We treat this as a singular "tunnel".
+		local tid = copy(c$id);
+		tid$orig_p = 0/tcp;
+		Tunnel::register([$cid=tid, $tunnel_type=Tunnel::HTTP]);
+		}
 	}
 
 event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=5

src/analyzer/protocol/http/HTTP.cc

@@ -889,6 +889,9 @@ HTTP_Analyzer::HTTP_Analyzer(Connection* conn)
 	reply_code = 0;
 	reply_reason_phrase = 0;
 
+	connect_request = false;
+	pia = 0;
+
 	content_line_orig = new tcp::ContentLine_Analyzer(conn, true);
 	AddSupportAnalyzer(content_line_orig);
 
@@ -945,6 +948,14 @@ void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig)
 	if ( TCP() && TCP()->IsPartial() )
 		return;
 
+	if ( pia )
+		{
+		// There will be a PIA instance if this connection has been identified 
+		// as a connect proxy.
+		ForwardStream(len, data, is_orig);
+		return;
+		}
+
 	const char* line = reinterpret_cast<const char*>(data);
 	const char* end_of_line = line + len;
 
@@ -1059,6 +1070,27 @@ void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig)
 						reply_message, is_orig,
 						ExpectReplyMessageBody(),
 						len);
+
+				if ( connect_request && reply_code == 200 )
+					{
+					pia = new pia::PIA_TCP(Conn());
+					if ( AddChildAnalyzer(pia) )
+						{
+						pia->FirstPacket(true, 0);
+						pia->FirstPacket(false, 0);
+
+						// This connection has transitioned to no longer
+						// being http and the content line support analyzers
+						// need to be removed.
+						RemoveSupportAnalyzer(content_line_orig);
+						RemoveSupportAnalyzer(content_line_resp);
+						}
+					else
+						{
+						pia = 0;
+						}
+					}
+
 				}
 			else
 				{
@@ -1404,6 +1436,9 @@ void HTTP_Analyzer::HTTP_Request()
 		// DEBUG_MSG("%.6f http_request\n", network_time);
 		ConnectionEvent(http_request, vl);
 		}
+
+	if ( strcasecmp_n(request_method->AsString()->Len(), (const char*) (request_method->AsString()->Bytes()), "CONNECT") == 0 )
+		connect_request = true;
 	}
 
 void HTTP_Analyzer::HTTP_Reply()

src/analyzer/protocol/http/HTTP.h

@@ -5,6 +5,7 @@
 
 #include "analyzer/protocol/tcp/TCP.h"
 #include "analyzer/protocol/tcp/ContentLine.h"
+#include "analyzer/protocol/pia/PIA.h"
 #include "analyzer/protocol/zip/ZIP.h"
 #include "analyzer/protocol/mime/MIME.h"
 #include "binpac_bro.h"
@@ -237,6 +238,9 @@ protected:
 	int connection_close;
 	int request_ongoing, reply_ongoing;
 
+	bool connect_request;
+	pia::PIA_TCP *pia;
+
 	Val* request_method;
 
 	// request_URI is in the original form (may contain '%<hex><hex>'

src/types.bif

@@ -186,6 +186,7 @@ enum Type %{
 	TEREDO,
 	SOCKS,
 	GTPv1,
+	HTTP,
 %}
 
 type EncapsulatingConn: record;

testing/btest/Baseline/scripts.base.protocols.http.http-connect/conn.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2014-02-13-03-37-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1078232251.833846	CXWv6p3arKYeMETxOg	79.26.245.236	3378	254.228.86.79	8240	tcp	http,smtp	6.722274	1685	223	SF	-	0	ShADadfF	14	2257	16	944	(empty)
+#close	2014-02-13-03-37-02

testing/btest/Baseline/scripts.base.protocols.http.http-connect/http.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#open	2014-02-13-03-37-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	orig_fuids	orig_mime_types	resp_fuids	resp_mime_types
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	vector[string]	vector[string]	vector[string]	vector[string]
+1078232252.284420	CXWv6p3arKYeMETxOg	79.26.245.236	3378	254.228.86.79	8240	1	CONNECT	-	mailin03.sul.t-online.de:25 /	-	-	0	0	200	Connection established	-	-	-	(empty)	-	-	-	-	-	-	-
+#close	2014-02-13-03-37-02

testing/btest/Baseline/scripts.base.protocols.http.http-connect/smtp.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	smtp
+#open	2014-02-13-03-37-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	helo	mailfrom	rcptto	date	from	to	reply_to	msg_id	in_reply_to	subject	x_originating_ip	first_received	second_received	last_reply	path	user_agent	fuids
+#types	time	string	addr	port	addr	port	count	string	string	table[string]	string	string	table[string]	string	string	string	string	addr	string	string	string	vector[addr]	string	vector[string]
+1078232255.642953	CXWv6p3arKYeMETxOg	79.26.245.236	3378	254.228.86.79	8240	1	208.191.73.21	<nhfjenna_neumann@lycos.com>	<thenightwatch@t-online.de>	Tue, 2 Mar 2004 13:57:49 +0100	Sybille Ostermann <nhfjenna_neumann@lycos.com>	thenightwatch@t-online.de	-	-	-	Hier sind die dicken Girls hemmungloser denn je.. grcu	-	from mail.iosphere.net (mail.iosphere.net [216.58.97.33]) by mail.netsync.net with esmtp; Mrz, 02 2004 12:55:34 -0700	-	250 Message accepted.	254.228.86.79,79.26.245.236,216.58.97.33	Microsoft Outlook Build 10.0.2616	FVS9k93PUgScEUCOjd
+#close	2014-02-13-03-37-02

testing/btest/Baseline/scripts.base.protocols.http.http-connect/tunnel.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	tunnel
+#open	2014-02-13-03-37-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
+#types	time	string	addr	port	addr	port	enum	enum
+1078232252.284420	-	79.26.245.236	0	254.228.86.79	8240	Tunnel::HTTP	Tunnel::DISCOVER
+#close	2014-02-13-03-37-02

testing/btest/scripts/base/protocols/http/http-connect.bro

@@ -0,0 +1,11 @@
+# This tests that the HTTP analyzer handles HTTP CONNECT proxying correctly.
+#
+# @TEST-EXEC: bro -r $TRACES/http/connect-with-smtp.trace %INPUT
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff http.log
+# @TEST-EXEC: btest-diff smtp.log
+# @TEST-EXEC: btest-diff tunnel.log
+
+# The base analysis scripts are loaded by default.
+#@load base/protocols/http
+