Add smtp starttls support

scripts/base/protocols/smtp/files.bro

@@ -41,13 +41,13 @@ function describe_file(f: fa_file): string
 
 event bro_init() &priority=5
 	{
-	Files::register_protocol(Analyzer::ANALYZER_SMTP, 
+	Files::register_protocol(Analyzer::ANALYZER_SMTP,
 	                         [$get_file_handle = SMTP::get_file_handle,
 	                          $describe        = SMTP::describe_file]);
 	}
 
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
 	{
-	if ( c?$smtp )
+	if ( c?$smtp && !c$smtp$tls )
 		c$smtp$fuids[|c$smtp$fuids|] = f$id;
 	}

scripts/base/protocols/smtp/main.bro

@@ -49,7 +49,10 @@ export {
 		path:              vector of addr  &log &optional;
 		## Value of the User-Agent header from the client.
 		user_agent:        string          &log &optional;
-		
+
+		## Indicates that the connection has switched to using TLS
+		tls:               bool            &default=F;
+
 		## Indicates if the "Received: from" headers should still be
 		## processed.
 		process_received_from: bool        &default=T;
@@ -276,6 +279,12 @@ event connection_state_remove(c: connection) &priority=-5
 		smtp_message(c);
 	}
 
+event smtp_starttls(c: connection) &priority=5
+	{
+	if ( c?$smtp )
+		c$smtp$tls = T;
+	}
+
 function describe(rec: Info): string
 	{
 	if ( rec?$mailfrom && rec?$rcptto )

src/analyzer/protocol/smtp/SMTP.cc

@@ -8,7 +8,7 @@
 #include "SMTP.h"
 #include "Event.h"
 #include "Reporter.h"
-#include "analyzer/protocol/tcp/ContentLine.h"
+#include "analyzer/Manager.h"
 
 #include "events.bif.h"
 
@@ -44,12 +44,12 @@ SMTP_Analyzer::SMTP_Analyzer(Connection* conn)
 	line_after_gap = 0;
 	mail = 0;
 	UpdateState(first_cmd, 0);
-	tcp::ContentLine_Analyzer* cl_orig = new tcp::ContentLine_Analyzer(conn, true);
+	cl_orig = new tcp::ContentLine_Analyzer(conn, true);
 	cl_orig->SetIsNULSensitive(true);
 	cl_orig->SetSkipPartial(true);
 	AddSupportAnalyzer(cl_orig);
 
-	tcp::ContentLine_Analyzer* cl_resp = new tcp::ContentLine_Analyzer(conn, false);
+	cl_resp = new tcp::ContentLine_Analyzer(conn, false);
 	cl_resp->SetIsNULSensitive(true);
 	cl_resp->SetSkipPartial(true);
 	AddSupportAnalyzer(cl_resp);
@@ -118,6 +118,13 @@ void SMTP_Analyzer::DeliverStream(int length, const u_char* line, bool orig)
 	{
 	tcp::TCP_ApplicationAnalyzer::DeliverStream(length, line, orig);
 
+	// If an TLS transaction has been initiated, forward to child and abort.
+	if ( state == SMTP_IN_TLS )
+		{
+		ForwardStream(length, line, orig);
+		return;
+		}
+
 	// NOTE: do not use IsOrig() here, because of TURN command.
 	int is_sender = orig_is_sender ? orig : ! orig;
 
@@ -152,10 +159,6 @@ void SMTP_Analyzer::DeliverStream(int length, const u_char* line, bool orig)
 void SMTP_Analyzer::ProcessLine(int length, const char* line, bool orig)
 	{
 	const char* end_of_line = line + length;
-	if ( state == SMTP_IN_TLS )
-		// Do not try to parse contents after STARTTLS/220.
-		return;
-
 	int cmd_len = 0;
 	const char* cmd = "";
 
@@ -380,6 +383,25 @@ void SMTP_Analyzer::NewCmd(const int cmd_code)
 		first_cmd = cmd_code;
 	}
 
+void SMTP_Analyzer::StartTLS()
+	{
+	// STARTTLS was succesful. Remove SMTP support analyzers, add SSL
+	// analyzer and throw event signifying the change.
+	state = SMTP_IN_TLS;
+	expect_sender = expect_recver = 1;
+	RemoveSupportAnalyzer(cl_orig);
+	RemoveSupportAnalyzer(cl_resp);
+	Analyzer* ssl = analyzer_mgr->InstantiateAnalyzer("SSL", Conn());
+	if ( ssl )
+		AddChildAnalyzer(ssl);
+
+	val_list* vl = new val_list;
+	vl->append(BuildConnVal());
+
+	ConnectionEvent(smtp_starttls, vl);
+	}
+
+
 
 // Here we keep a SMTP state machine and update it on each reply.
 // However, the purpose is NOT to check correctness of SMTP commands
@@ -740,8 +762,7 @@ void SMTP_Analyzer::UpdateState(const int cmd_code, const int reply_code)
 				break;
 
 			case 220:
-				state = SMTP_IN_TLS;
+				StartTLS();
-				expect_sender = expect_recver = 1;
 				break;
 
 			case 454:

src/analyzer/protocol/smtp/SMTP.h

@@ -7,6 +7,7 @@
 using namespace std;
 
 #include "analyzer/protocol/tcp/TCP.h"
+#include "analyzer/protocol/tcp/ContentLine.h"
 #include "analyzer/protocol/mime/MIME.h"
 
 #undef SMTP_CMD_DEF
@@ -74,6 +75,7 @@ protected:
 				int detail_len, const char* detail);
 	void UnexpectedCommand(const int cmd_code, const int reply_code);
 	void UnexpectedReply(const int cmd_code, const int reply_code);
+	void StartTLS();
 
 	bool orig_is_sender;
 	int expect_sender, expect_recver;
@@ -88,6 +90,10 @@ protected:
 					// after a gap
 
 	mime::MIME_Mail* mail;
+
+private:
+	tcp::ContentLine_Analyzer* cl_orig;
+	tcp::ContentLine_Analyzer* cl_resp;
 };
 
 } } // namespace analyzer::* 

src/analyzer/protocol/smtp/events.bif

@@ -98,3 +98,11 @@ event smtp_data%(c: connection, is_orig: bool, data: string%);
 ##
 ## .. bro:see:: smtp_data  smtp_request smtp_reply
 event smtp_unexpected%(c: connection, is_orig: bool, msg: string, detail: string%);
+
+## Generated if a connection switched to using TLS using STARTTLS. After this event
+## no more SMTP events will be raised for the connection. See the SSL analyzer for
+## related SSL events.
+##
+## c: The connection
+##
+event smtp_starttls%(c: connection%);

testing/btest/Baseline/scripts.base.protocols.smtp.starttls/smtp.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	smtp
+#open	2014-05-15-16-56-36
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	helo	mailfrom	rcptto	date	from	to	reply_to	msg_id	in_reply_to	subject	x_originating_ip	first_received	second_received	last_reply	path	user_agent	fuids
+#types	time	string	addr	port	addr	port	count	string	string	set[string]	string	string	set[string]	string	string	string	string	addr	string	string	string	vector[addr]	string	vector[string]
+1400168396.898137	CXWv6p3arKYeMETxOg	192.168.4.149	54170	74.125.142.26	25	1	openssl.client.net	-	-	-	-	-	-	-	-	-	-	-	-	220 2.0.0 Ready to start TLS	74.125.142.26,192.168.4.149	-	(empty)
+#close	2014-05-15-16-56-36

testing/btest/Baseline/scripts.base.protocols.smtp.starttls/ssl.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open	2014-05-15-16-56-36
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	session_id	last_alert	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	bool	vector[string]	vector[string]	string	string	string	string
+1400168397.019290	CXWv6p3arKYeMETxOg	192.168.4.149	54170	74.125.142.26	25	TLSv12	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256	secp256r1	-	-	-	T	FUE0dj3SWjQASC0bk3,FbPkr51wrSMIUT5Hib,FVW1o23Jjs8yenOhzb	(empty)	CN=mx.google.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority G2,O=Google Inc,C=US	-	-
+#close	2014-05-15-16-56-36

testing/btest/Baseline/scripts.base.protocols.smtp.starttls/x509.log

@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	x509
+#open	2014-05-15-16-56-36
+#fields	ts	id	certificate.version	certificate.serial	certificate.subject	certificate.issuer	certificate.not_valid_before	certificate.not_valid_after	certificate.key_alg	certificate.sig_alg	certificate.key_type	certificate.key_length	certificate.exponent	certificate.curve	san.dns	san.uri	san.email	san.ip	basic_constraints.ca	basic_constraints.path_len
+#types	time	string	count	string	string	string	time	time	string	string	string	count	string	string	vector[string]	vector[string]	vector[string]	vector[addr]	bool	count
+1400168397.069811	FUE0dj3SWjQASC0bk3	3	325D8297987D50B0	CN=mx.google.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority G2,O=Google Inc,C=US	1378726355.000000	1410262355.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	aspmx.l.google.com,alt1.aspmx.l.google.com,alt2.aspmx.l.google.com,alt3.aspmx.l.google.com,alt4.aspmx.l.google.com,gmail-smtp-in.l.google.com,alt1.gmail-smtp-in.l.google.com,alt2.gmail-smtp-in.l.google.com,alt3.gmail-smtp-in.l.google.com,alt4.gmail-smtp-in.l.google.com,gmr-smtp-in.l.google.com,alt1.gmr-smtp-in.l.google.com,alt2.gmr-smtp-in.l.google.com,alt3.gmr-smtp-in.l.google.com,alt4.gmr-smtp-in.l.google.com,mx.google.com,aspmx2.googlemail.com,aspmx3.googlemail.com,aspmx4.googlemail.com,aspmx5.googlemail.com	-	-	-	F	-
+1400168397.069811	FbPkr51wrSMIUT5Hib	3	023A69	CN=Google Internet Authority G2,O=Google Inc,C=US	CN=GeoTrust Global CA,O=GeoTrust Inc.,C=US	1365174955.000000	1428160555.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	0
+1400168397.069811	FVW1o23Jjs8yenOhzb	3	12BBE6	CN=GeoTrust Global CA,O=GeoTrust Inc.,C=US	OU=Equifax Secure Certificate Authority,O=Equifax,C=US	1021953600.000000	1534824000.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	-
+#close	2014-05-15-16-56-36

testing/btest/scripts/base/protocols/smtp/starttls.test

@@ -0,0 +1,6 @@
+# @TEST-EXEC: bro -C -r $TRACES/tls/smtp-starttls.pcap %INPUT
+# @TEST-EXEC: btest-diff smtp.log
+# @TEST-EXEC: btest-diff ssl.log
+# @TEST-EXEC: btest-diff x509.log
+
+@load base/protocols/smtp