mysql: Recognize when client/server negotiate SSL

This instantiates the SSL analyzer when the client requests SSL
so that Zeek now has a bit more visibility into encrypted MySQL
connections.

The pattern used is the same as in the IMAP, POP or XMPP analyzer.

NEWS

@@ -165,6 +165,11 @@ New Functionality
   of new analyzers as well as for collecting operational data in production
   environments.
 
+- The MySQL analyzer has been extended to detect when client and server negotiate
+  to use a SSL encrypted session. This allows analysis of the subsequent SSL
+  handshake. The service field for encrypted MySQL connections in the conn.log
+  will have entries for both, mysql and ssl.
+
 Changed Functionality
 ---------------------
 

src/analyzer/protocol/mysql/MySQL.cc

@@ -3,6 +3,7 @@
 #include "zeek/analyzer/protocol/mysql/MySQL.h"
 
 #include "zeek/Reporter.h"
+#include "zeek/analyzer/Manager.h"
 #include "zeek/analyzer/protocol/mysql/events.bif.h"
 #include "zeek/analyzer/protocol/tcp/TCP_Reassembler.h"
 
@@ -13,6 +14,7 @@ MySQL_Analyzer::MySQL_Analyzer(Connection* c) : analyzer::tcp::TCP_ApplicationAn
 	{
 	interp = new binpac::MySQL::MySQL_Conn(this);
 	had_gap = false;
+	tls_active = false;
 	}
 
 MySQL_Analyzer::~MySQL_Analyzer()
@@ -31,13 +33,34 @@ void MySQL_Analyzer::Done()
 void MySQL_Analyzer::EndpointEOF(bool is_orig)
 	{
 	analyzer::tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+
+	if ( tls_active )
+		ForwardEndOfData(is_orig);
+
 	interp->FlowEOF(is_orig);
 	}
 
+void MySQL_Analyzer::StartTLS()
+	{
+	tls_active = true;
+
+	Analyzer* ssl = analyzer_mgr->InstantiateAnalyzer("SSL", Conn());
+	if ( ssl )
+		AddChildAnalyzer(ssl);
+	}
+
 void MySQL_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 	{
 	analyzer::tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig);
 
+	if ( tls_active )
+		{
+		// If TLS has been initiated, forward to child and
+		// short-circuit further processing
+		ForwardStream(len, data, orig);
+		return;
+		}
+
 	if ( TCP() && TCP()->IsPartial() )
 		return;
 
@@ -60,6 +83,10 @@ void MySQL_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 void MySQL_Analyzer::Undelivered(uint64_t seq, int len, bool orig)
 	{
 	analyzer::tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig);
+
+	if ( tls_active )
+		ForwardUndelivered(seq, len, orig);
+
 	had_gap = true;
 	interp->NewGap(orig, len);
 	}

src/analyzer/protocol/mysql/MySQL.h

@@ -25,11 +25,14 @@ public:
 	// Overridden from analyzer::tcp::TCP_ApplicationAnalyzer.
 	void EndpointEOF(bool is_orig) override;
 
+	void StartTLS();
+
 	static analyzer::Analyzer* Instantiate(Connection* conn) { return new MySQL_Analyzer(conn); }
 
 protected:
 	binpac::MySQL::MySQL_Conn* interp;
 	bool had_gap;
+	bool tls_active;
 	};
 
 	} // namespace zeek::analyzer::mysql

src/analyzer/protocol/mysql/mysql-analyzer.pac

@@ -22,12 +22,19 @@ refine flow MySQL_Flow += {
 		if ( ${msg.version} == 9 || ${msg.version == 10} )
 			connection()->zeek_analyzer()->AnalyzerConfirmation();
 
+		// If the client requested SSL and didn't provide credentials, switch to SSL
+		if ( ${msg.version} == 10 && ( ${msg.v10_response.cap_flags} & CLIENT_SSL ) && ${msg.v10_response.credentials}->empty() )
+			{
+			connection()->zeek_analyzer()->StartTLS();
+			return true;
+			}
+
 		if ( mysql_handshake )
 			{
-			if ( ${msg.version} == 10 )
+			if ( ${msg.version} == 10 && ${msg.v10_response.credentials}->size() > 0 )
 				zeek::BifEvent::enqueue_mysql_handshake(connection()->zeek_analyzer(),
 				                                  connection()->zeek_analyzer()->Conn(),
-				                                  zeek::make_intrusive<zeek::StringVal>(c_str(${msg.v10_response.username})));
+				                                  zeek::make_intrusive<zeek::StringVal>(c_str(${msg.v10_response.credentials[0].username})));
 			if ( ${msg.version} == 9 )
 				zeek::BifEvent::enqueue_mysql_handshake(connection()->zeek_analyzer(),
 				                                  connection()->zeek_analyzer()->Conn(),

src/analyzer/protocol/mysql/mysql-protocol.pac

@@ -158,7 +158,8 @@ enum EOFType {
 };
 
 enum Client_Capabilities {
-	# Expects an OK (instead of EOF) after the resultset rows of a Text Resultset. 
+	CLIENT_SSL           = 0x00000800,
+	# Expects an OK (instead of EOF) after the resultset rows of a Text Resultset.
 	CLIENT_DEPRECATE_EOF = 0x01000000,
 };
 
@@ -237,13 +238,17 @@ type Handshake_Response_Packet = case $context.connection.get_version() of {
 	version: uint8 = $context.connection.get_version();
 };
 
+type Handshake_Credentials_v10 = record {
+	username : NUL_String;
+	password : bytestring &restofdata;
+};
+
 type Handshake_Response_Packet_v10 = record {
 	cap_flags   : uint32;
 	max_pkt_size: uint32;
 	char_set    : uint8;
 	pad         : padding[23];
-	username    : NUL_String;
+	credentials : Handshake_Credentials_v10[] &until($input.length() == 0);
-	password    : bytestring &restofdata;
 } &let {
 	deprecate_eof: bool = $context.connection.set_deprecate_eof(cap_flags & CLIENT_DEPRECATE_EOF);
 };

src/analyzer/protocol/mysql/mysql.pac

@@ -8,16 +8,26 @@
 %include zeek.pac
 
 %extern{
-	#include "zeek/analyzer/protocol/mysql/events.bif.h"
+
+namespace zeek::analyzer::mysql { class MySQL_Analyzer; }
+namespace binpac { namespace MySQL { class MySQL_Conn; } }
+using MySQLAnalyzer = zeek::analyzer::mysql::MySQL_Analyzer*;
+
+#include "zeek/analyzer/protocol/mysql/MySQL.h"
+
+#include "zeek/analyzer/protocol/mysql/events.bif.h"
+
 %}
 
+extern type MySQLAnalyzer;
+
 analyzer MySQL withcontext {
 	connection: MySQL_Conn;
 	flow:       MySQL_Flow;
 };
 
 # Our connection consists of two flows, one in each direction.
-connection MySQL_Conn(zeek_analyzer: ZeekAnalyzer) {
+connection MySQL_Conn(zeek_analyzer: MySQLAnalyzer) {
 	upflow   = MySQL_Flow(true);
 	downflow = MySQL_Flow(false);
 };

testing/btest/Baseline/scripts.base.protocols.mysql.encrypted-aws-rds/tls-12.conn.log

@@ -7,5 +7,5 @@
 #open XXXX-XX-XX-XX-XX-XX
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]
-XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	82.239.87.25	58132	79.107.90.25	3306	tcp	-	2.043921	724	3255	SF	-	-	0	ShAdDaFf	14	1460	11	3835	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	82.239.87.25	58132	79.107.90.25	3306	tcp	ssl,mysql	2.043921	724	3255	SF	-	-	0	ShAdDaFf	14	1460	11	3835	-
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.mysql.encrypted-aws-rds/tls-12.ssl.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	ssl_history	cert_chain_fps	client_cert_chain_fps	sni_matches_cert
+#types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	string	vector[string]	vector[string]	bool
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	82.239.87.25	58132	79.107.90.25	3306	TLSv12	TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384	x25519	-	F	-	-	T	CsxkrnXGIti	297800fb2627e156b3b70cb2dd41c568a8e43e6a689c84d222441f18f608c1a2,c0ff201aeea68f5eac779595305fa277eb06e98c78b83507e0ae945f7678094a	(empty)	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.mysql.encrypted-aws-rds/tls-12.x509.log

@@ -0,0 +1,12 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	x509
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	fingerprint	certificate.version	certificate.serial	certificate.subject	certificate.issuer	certificate.not_valid_before	certificate.not_valid_after	certificate.key_alg	certificate.sig_alg	certificate.key_type	certificate.key_length	certificate.exponent	certificate.curve	san.dns	san.uri	san.email	san.ip	basic_constraints.ca	basic_constraints.path_len	host_cert	client_cert
+#types	time	string	count	string	string	string	time	time	string	string	string	count	string	string	vector[string]	vector[string]	vector[string]	vector[addr]	bool	count	bool	bool
+XXXXXXXXXX.XXXXXX	297800fb2627e156b3b70cb2dd41c568a8e43e6a689c84d222441f18f608c1a2	3	E8B1F48FD24F222323FE70099E948D	C=US,ST=Washington,L=Seattle,O=Amazon.com,OU=RDS,CN=free-mysql-database.cyx4x7yvdoay.us-east-1.rds.amazonaws.com	CN=Amazon RDS us-east-1 2019 CA,OU=Amazon RDS,O=Amazon Web Services\\, Inc.,L=Seattle,ST=Washington,C=US	XXXXXXXXXX.XXXXXX	XXXXXXXXXX.XXXXXX	rsaEncryption	sha256WithRSAEncryption	rsa	2048	65537	-	free-mysql-database.cyx4x7yvdoay.us-east-1.rds.amazonaws.com	-	-	-	-	-	T	F
+XXXXXXXXXX.XXXXXX	c0ff201aeea68f5eac779595305fa277eb06e98c78b83507e0ae945f7678094a	3	2555	CN=Amazon RDS us-east-1 2019 CA,OU=Amazon RDS,O=Amazon Web Services\\, Inc.,L=Seattle,ST=Washington,C=US	CN=Amazon RDS Root 2019 CA,OU=Amazon RDS,O=Amazon Web Services\\, Inc.,ST=Washington,L=Seattle,C=US	XXXXXXXXXX.XXXXXX	XXXXXXXXXX.XXXXXX	rsaEncryption	sha256WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	0	F	F
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.mysql.encrypted-aws-rds/tls-13.conn.log

@@ -7,5 +7,5 @@
 #open XXXX-XX-XX-XX-XX-XX
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]
-XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	82.239.87.25	57902	79.107.90.25	3306	tcp	-	6.756360	1076	3776	SF	-	-	0	ShAdDaFf	19	2072	14	4512	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	82.239.87.25	57902	79.107.90.25	3306	tcp	ssl,mysql	6.756360	1076	3776	SF	-	-	0	ShAdDaFf	19	2072	14	4512	-
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.mysql.encrypted-aws-rds/tls-13.ssl.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	ssl_history	cert_chain_fps	client_cert_chain_fps	sni_matches_cert
+#types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	string	vector[string]	vector[string]	bool
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	82.239.87.25	57902	79.107.90.25	3306	TLSv13	TLS_AES_256_GCM_SHA384	x25519	-	F	-	-	T	CsiI	-	-	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.mysql.encrypted/conn.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	59272	127.0.0.1	3306	tcp	ssl,mysql	0.021783	713	1959	SF	-	-	0	ShAdDaFf	10	1241	8	2383	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.mysql.encrypted/ssl.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	ssl_history	cert_chain_fps	client_cert_chain_fps	sni_matches_cert
+#types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	string	vector[string]	vector[string]	bool
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	59272	127.0.0.1	3306	TLSv11	TLS_DHE_RSA_WITH_AES_256_CBC_SHA	-	-	F	-	-	T	CsxkrnXGIi	e3803fed72de6742148784b568c223771e91a6a2720849973d30995d627bc4f0	(empty)	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.mysql.encrypted/x509.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	x509
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	fingerprint	certificate.version	certificate.serial	certificate.subject	certificate.issuer	certificate.not_valid_before	certificate.not_valid_after	certificate.key_alg	certificate.sig_alg	certificate.key_type	certificate.key_length	certificate.exponent	certificate.curve	san.dns	san.uri	san.email	san.ip	basic_constraints.ca	basic_constraints.path_len	host_cert	client_cert
+#types	time	string	count	string	string	string	time	time	string	string	string	count	string	string	vector[string]	vector[string]	vector[string]	vector[addr]	bool	count	bool	bool
+XXXXXXXXXX.XXXXXX	e3803fed72de6742148784b568c223771e91a6a2720849973d30995d627bc4f0	1	01	O=Internet Widgits Pty Ltd,ST=Some-State,C=AU	O=Internet Widgits Pty Ltd,ST=Some-State,C=AU	XXXXXXXXXX.XXXXXX	XXXXXXXXXX.XXXXXX	rsaEncryption	sha256WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	-	-	T	F
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/scripts/base/protocols/mysql/encrypted-aws-rds.test

@@ -7,12 +7,12 @@
 # @TEST-EXEC: mkdir tls-13 && mv *log tls-13
 #
 # @TEST-EXEC: btest-diff tls-12/conn.log
-# #TEST-EXEC: btest-diff tls-12/ssl.log
+# @TEST-EXEC: btest-diff tls-12/ssl.log
-# #TEST-EXEC: btest-diff tls-12/x509.log
+# @TEST-EXEC: btest-diff tls-12/x509.log
 #
 # @TEST-EXEC: btest-diff tls-13/conn.log
-# #TEST-EXEC: btest-diff tls-13/ssl.log
+# @TEST-EXEC: btest-diff tls-13/ssl.log
-# #TEST-EXEC: ! test -f tls-13/x509.log
+# @TEST-EXEC: ! test -f tls-13/x509.log
 
 @load base/protocols/conn
 @load base/protocols/mysql

testing/btest/scripts/base/protocols/mysql/encrypted.test

@@ -5,5 +5,12 @@
 # @TEST-EXEC: touch mysql.log
 # @TEST-EXEC: zeek -b -r $TRACES/mysql/encrypted.trace %INPUT
 # @TEST-EXEC: btest-diff mysql.log
+#
+# Ensure the connection was handed off by peaking into some other logs.
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff ssl.log
+# @TEST-EXEC: btest-diff x509.log
 
+@load base/protocols/conn
 @load base/protocols/mysql
+@load base/protocols/ssl