Add support for "auth switch" and "query attrs"

Also fix the issue where Resultset could not correctly distinguish between EOF_Packet and OK_Packet.

src/analyzer/protocol/mysql/mysql-analyzer.pac

@@ -83,8 +83,8 @@ refine flow MySQL_Flow += {
 
 	function proc_resultset(msg: Resultset): bool
 		%{
-		if ( ${msg.is_eof} )
+		if ( ${msg.is_eof_or_ok} )
-			return true;  // Raised through proc_eof_packet()
+			return true;  // Raised through proc_eof_packet() or proc_ok_packet()
 
 		if ( ! mysql_result_row )
 			return true;

src/analyzer/protocol/mysql/mysql-protocol.pac

@@ -96,11 +96,6 @@ type LengthEncodedStringArg(first_byte: uint8) = record {
 		};
 %}
 
-%code{
-	const char* PLUGIN_CACHING_SHA2_PASSWORD = "caching_sha2_password";
-%}
-
-extern type PLUGIN_CACHING_SHA2_PASSWORD;
 extern type to_int;
 
 # Enums
@@ -146,6 +141,7 @@ enum state {
 	SHA2_AUTH_PHASE        = 2,
 	PUB_KEY_PHASE          = 3,
 	SHA2_AUTH_RESP_PHASE   = 4,
+	AUTH_SWITCH_RESP_PHASE = 5,
 };
 
 enum Expected {
@@ -173,9 +169,10 @@ enum Client_Capabilities {
 	# Expects an OK (instead of EOF) after the resultset rows of a Text Resultset.
 	CLIENT_DEPRECATE_EOF = 0x01000000,
 	CLIENT_ZSTD_COMPRESSION_ALGORITHM = 0x04000000,
+	CLIENT_QUERY_ATTRIBUTES = 0x08000000,
 };
 
-enum SHA2_Atuh_State {
+enum SHA2_Auth_State {
 	REQUEST_PUBLIC_KEY          = 2,
 	FAST_AUTH_SUCCESS           = 3,
 	PERFORM_FULL_AUTHENTICATION = 4,
@@ -217,6 +214,7 @@ type Client_Message(state: int) = case state of {
 	SHA2_AUTH_PHASE  -> sha2_auth_phase : SHA2_Auth_Packet;
 	PUB_KEY_PHASE    -> pub_key_phase   : Public_Key_Packet;
 	SHA2_AUTH_RESP_PHASE -> sha2_auth_resp_phase : SHA2_Auth_Response_Packet;
+	AUTH_SWITCH_RESP_PHASE -> auth_switch_resp_phase : Auth_Switch_Response_Packet;
 };
 
 # Handshake Request
@@ -244,10 +242,14 @@ type Handshake_v10 = record {
 	auth_plugin_data_len   : uint8;
 	reserved               : padding[10]; 
 	auth_plugin_data_part_2: bytestring &length=13;
-	have_plugin : case ( ( capability_flags_2 << 4 ) & CLIENT_PLUGIN_AUTH ) of {
+	have_plugin : case ( ( capability_flags_2 << 16 ) & CLIENT_PLUGIN_AUTH ) of {
-		CLIENT_PLUGIN_AUTH -> auth_plugin_name: NUL_String;
+		CLIENT_PLUGIN_AUTH -> auth_plugin: NUL_String;
 		0x0 -> none    : empty;
 	};
+} &let {
+	update_auth_plugin: bool = $context.connection.set_auth_plugin(auth_plugin)
+		&if( ( capability_flags_2 << 16 ) & CLIENT_PLUGIN_AUTH );
+	server_query_attrs: bool = $context.connection.set_server_query_attrs(( capability_flags_2 << 16 ) & CLIENT_QUERY_ATTRIBUTES);
 };
 
 type Handshake_v9 = record {
@@ -287,7 +289,7 @@ type Handshake_Plain_v10(cap_flags: uint32) = record {
 		0x0 -> none_1    : empty;
 	};
 	have_plugin : case ( cap_flags & CLIENT_PLUGIN_AUTH ) of {
-		CLIENT_PLUGIN_AUTH -> auth_plugin_name: EmptyOrNUL_String;
+		CLIENT_PLUGIN_AUTH -> auth_plugin: EmptyOrNUL_String;
 		0x0 -> none_2    : empty;
 	};
 	have_attrs  : case ( cap_flags & CLIENT_CONNECT_ATTRS ) of {
@@ -299,8 +301,10 @@ type Handshake_Plain_v10(cap_flags: uint32) = record {
 		0x0 -> none_4    : empty;
 	};
 } &let {
-	update_state: bool = $context.connection.update_state(SHA2_AUTH_PHASE)
+	update_auth_plugin: bool = $context.connection.set_auth_plugin(auth_plugin)
-		&if(( cap_flags & CLIENT_PLUGIN_AUTH ) && auth_plugin_name==PLUGIN_CACHING_SHA2_PASSWORD);
+		&if( cap_flags & CLIENT_PLUGIN_AUTH );
+	update_state: bool = $context.connection.update_state_from_auth()
+		&if( cap_flags & CLIENT_PLUGIN_AUTH );
 };
 
 type Handshake_Response_Packet_v10 = record {
@@ -314,6 +318,7 @@ type Handshake_Response_Packet_v10 = record {
 	};
 } &let {
 	deprecate_eof: bool = $context.connection.set_deprecate_eof(cap_flags & CLIENT_DEPRECATE_EOF);
+	client_query_attrs: bool = $context.connection.set_client_query_attrs(cap_flags & CLIENT_QUERY_ATTRIBUTES);
 };
 
 type Handshake_Response_Packet_v9 = record {
@@ -352,10 +357,43 @@ type SHA2_Auth_Response_Packet = record {
 	update_state: bool = $context.connection.update_state(COMMAND_PHASE);
 };
 
+# Auth Switch
+
+type Auth_Switch_Response_Packet = record {
+	data  : bytestring &restofdata;
+} &let {
+	update_state: bool = $context.connection.update_state_from_auth();
+};
+
 # Command Request
 
+type AttributeTypeAndName = record {
+	type: uint16;
+	name: LengthEncodedString;
+};
+
+type Attributes(count: uint8) = record {
+	unused              : uint8;
+	send_types_to_server: uint8; # Always 1.
+	names               : AttributeTypeAndName[count];
+	values              : LengthEncodedString[count];
+};
+
+type Query_Attributes = record {
+	count     : uint8;
+	set_coun  : uint8;
+	have_attr : case ( count > 0 ) of {
+		true  -> attrs: Attributes(count);
+		false -> none: empty;
+	};
+};
+
 type Command_Request_Packet = record {
 	command: uint8;
+	attrs  : case ( command == COM_QUERY && $context.connection.get_client_query_attrs() && $context.connection.get_server_query_attrs() ) of {
+		true  -> query_attrs: Query_Attributes;
+		false -> none: empty;
+	};
 	arg    : bytestring &restofdata;
 } &let {
 	update_expectation: bool = $context.connection.set_next_expected_from_command(command);
@@ -413,22 +451,22 @@ type ColumnDefinition = record {
 };
 
 # Only used to indicate the end of a result, no intermediate eofs here.
-type EOFOrOK = case $context.connection.get_deprecate_eof() of {
+# MySQL spec says "You must check whether the packet length is less than 9
+# to make sure that it is a EOF_Packet packet" so the value of 13 here
+# comes from that 9, plus a 4-byte header.
+type EOFOrOK(pkt_len: uint32) = case ( $context.connection.get_deprecate_eof() || pkt_len > 13 ) of {
 	false -> eof: EOF_Packet(EOF_END);
 	true  -> ok: OK_Packet;
 };
 
 type ColumnDefinitionOrEOF(pkt_len: uint32) = record {
 	marker    : uint8;
-	def_or_eof: case is_eof of {
+	def_or_eof: case is_eof_or_ok of {
-		true  -> eof: EOFOrOK;
+		true  -> eof: EOFOrOK(pkt_len);
 		false -> def: ColumnDefinition41(marker);
-	} &requires(is_eof);
+	} &requires(is_eof_or_ok);
 } &let {
-	# MySQL spec says "You must check whether the packet length is less than 9
+	is_eof_or_ok: bool = (marker == 0xfe);
-	# to make sure that it is a EOF_Packet packet" so the value of 13 here
-	# comes from that 9, plus a 4-byte header.
-	is_eof: bool = (marker == 0xfe && pkt_len < 13);
 };
 
 
@@ -442,17 +480,14 @@ type EOFIfLegacyThenResultset(pkt_len: uint32) = case $context.connection.get_de
 
 type Resultset(pkt_len: uint32) = record {
 	marker    : uint8;
-	row_or_eof: case is_eof of {
+	row_or_eof: case is_eof_or_ok of {
-		true  -> eof: EOFOrOK;
+		true  -> eof: EOFOrOK(pkt_len);
 		false -> row: ResultsetRow(marker);
-	} &requires(is_eof);
+	} &requires(is_eof_or_ok);
 } &let {
-	# MySQL spec says "You must check whether the packet length is less than 9
+	is_eof_or_ok      : bool = (marker == 0xfe);
-	# to make sure that it is a EOF_Packet packet" so the value of 13 here
-	# comes from that 9, plus a 4-byte header.
-	is_eof            : bool = (marker == 0xfe && pkt_len < 13);
 	update_result_seen: bool = $context.connection.inc_results_seen();
-	update_expectation: bool = $context.connection.set_next_expected(is_eof ? NO_EXPECTATION : EXPECT_RESULTSET);
+	update_expectation: bool = $context.connection.set_next_expected(is_eof_or_ok ? NO_EXPECTATION : EXPECT_RESULTSET);
 };
 
 type ResultsetRow(first_byte: uint8) = record {
@@ -480,6 +515,9 @@ type AuthSwitchRequest = record {
 	status: uint8;
 	name  : NUL_String;
 	data  : bytestring &restofdata;
+} &let {
+	update_auth_plugin: bool = $context.connection.set_auth_plugin(name);
+	update_state: bool = $context.connection.update_state(AUTH_SWITCH_RESP_PHASE);
 };
 
 type ColumnDefinition320 = record {
@@ -531,6 +569,9 @@ refine connection MySQL_Conn += {
 		uint32 remaining_cols_;
 		uint32 results_seen_;
 		bool deprecate_eof_;
+		bool server_query_attrs_;
+		bool client_query_attrs_;
+		bytestring auth_plugin_;
 	%}
 
 	%init{
@@ -542,6 +583,13 @@ refine connection MySQL_Conn += {
 		remaining_cols_ = 0;
 		results_seen_ = 0;
 		deprecate_eof_ = false;
+		server_query_attrs_ = false;
+		client_query_attrs_ = false;
+		auth_plugin_ = bytestring();
+	%}
+
+	%cleanup{
+		auth_plugin_.free();
 	%}
 
 	function get_version(): uint8
@@ -577,6 +625,18 @@ refine connection MySQL_Conn += {
 		return true;
 		%}
 
+	function update_state_from_auth(): bool
+		%{
+		if ( auth_plugin_ == "caching_sha2_password" )
+			{
+			state_ = SHA2_AUTH_PHASE;
+			if ( expected_ == EXPECT_AUTH_SWITCH )
+				expected_ = EXPECT_STATUS;
+			}
+
+		return true;
+		%}
+
 	function get_deprecate_eof(): bool
 		%{
 		return deprecate_eof_;
@@ -588,6 +648,46 @@ refine connection MySQL_Conn += {
 		return true;
 		%}
 
+	function get_server_query_attrs(): bool
+		%{
+		return server_query_attrs_;
+		%}
+
+	function set_server_query_attrs(q: bool): bool
+		%{
+		server_query_attrs_ = q;
+		return true;
+		%}
+
+	function get_client_query_attrs(): bool
+		%{
+		return client_query_attrs_;
+		%}
+
+	function set_client_query_attrs(q: bool): bool
+		%{
+		client_query_attrs_ = q;
+		return true;
+		%}
+
+	function get_auth_plugin(): bytestring
+		%{
+		return auth_plugin_;
+		%}
+
+	function set_auth_plugin(a: bytestring): bool
+		%{
+		if ( auth_plugin_.length() > 0 &&
+		     strncmp(c_str(auth_plugin_), c_str(a), auth_plugin_.length()) != 0 )
+			{
+			expected_ = EXPECT_AUTH_SWITCH;
+			}
+
+		auth_plugin_.free();
+		auth_plugin_.init(a.data(), a.length());
+		return true;
+		%}
+
 	function get_expectation(): Expected
 		%{
 		return expected_;

testing/btest/Baseline/scripts.base.protocols.mysql.caching_sha2_password-after-auth-switch/mysql.log

@@ -0,0 +1,16 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	mysql
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	cmd	arg	success	rows	response
+#types	time	string	addr	port	addr	port	string	string	bool	count	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	35928	127.0.0.1	3306	login	root	T	0	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	35928	127.0.0.1	3306	query	select @@version_comment limit 1	T	0	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	35928	127.0.0.1	3306	query	select DATABASE(), USER() limit 1	T	0	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	35928	127.0.0.1	3306	query	select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1	T	0	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	35928	127.0.0.1	3306	statistics	(empty)	-	-	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	35928	127.0.0.1	3306	quit	(empty)	-	-	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.mysql.caching_sha2_password-after-auth-switch/out

@@ -0,0 +1,14 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+mysql handshake, root
+mysql ok, 0
+mysql request, 3, select @@version_comment limit 1
+mysql result row, [MySQL Community Server - GPL]
+mysql ok, 0
+mysql request, 3, select DATABASE(), USER() limit 1
+mysql result row, [, root@localhost]
+mysql ok, 0
+mysql request, 3, select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1
+mysql result row, [utf8mb4, utf8mb4, utf8mb4, utf8mb4]
+mysql ok, 0
+mysql request, 9, 
+mysql request, 1, 

testing/btest/Baseline/scripts.base.protocols.mysql.caching_sha2_password/mysql.log

@@ -9,15 +9,15 @@
 #types	time	string	addr	port	addr	port	string	string	bool	count	string
 XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	56494	127.0.0.1	3306	login	root	F	-	Got an error reading communication packets
 XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	127.0.0.1	49352	127.0.0.1	3306	login	root	T	0	-
-XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	127.0.0.1	49352	127.0.0.1	3306	query	\x00\x01show databases	T	0	-
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	127.0.0.1	49352	127.0.0.1	3306	query	show databases	T	0	-
-XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	127.0.0.1	49352	127.0.0.1	3306	query	\x00\x01show tables	T	0	-
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	127.0.0.1	49352	127.0.0.1	3306	query	show tables	T	0	-
 XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	127.0.0.1	49352	127.0.0.1	3306	field_list	t	T	0	-
-XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	127.0.0.1	49352	127.0.0.1	3306	query	\x00\x01select @@version_comment limit 1	T	0	-
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	127.0.0.1	49352	127.0.0.1	3306	query	select @@version_comment limit 1	T	0	-
 XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	127.0.0.1	49352	127.0.0.1	3306	quit	(empty)	-	-	-
 XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	127.0.0.1	40950	127.0.0.1	3306	login	root	T	0	-
-XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	127.0.0.1	40950	127.0.0.1	3306	query	\x00\x01show databases	T	0	-
+XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	127.0.0.1	40950	127.0.0.1	3306	query	show databases	T	0	-
-XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	127.0.0.1	40950	127.0.0.1	3306	query	\x00\x01show tables	T	0	-
+XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	127.0.0.1	40950	127.0.0.1	3306	query	show tables	T	0	-
 XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	127.0.0.1	40950	127.0.0.1	3306	field_list	t	T	0	-
-XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	127.0.0.1	40950	127.0.0.1	3306	query	\x00\x01select @@version_comment limit 1	T	0	-
+XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	127.0.0.1	40950	127.0.0.1	3306	query	select @@version_comment limit 1	T	0	-
 XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	127.0.0.1	40950	127.0.0.1	3306	quit	(empty)	-	-	-
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.mysql.caching_sha2_password/out

@@ -3,37 +3,37 @@ mysql handshake, root
 mysql error, 1158, Got an error reading communication packets
 mysql handshake, root
 mysql ok, 0
-mysql request, 3, \x00\x01show databases
+mysql request, 3, show databases
 mysql result row, [information_schema]
 mysql result row, [mysql]
 mysql result row, [performance_schema]
 mysql result row, [sys]
 mysql result row, [test]
 mysql ok, 0
-mysql request, 3, \x00\x01show tables
+mysql request, 3, show tables
 mysql result row, [t]
 mysql ok, 0
 mysql request, 4, t\x00
 mysql ok, 0
-mysql request, 3, \x00\x01select @@version_comment limit 1
+mysql request, 3, select @@version_comment limit 1
 mysql result row, [MySQL Community Server - GPL]
 mysql ok, 0
 mysql request, 1, 
 mysql handshake, root
 mysql ok, 0
-mysql request, 3, \x00\x01show databases
+mysql request, 3, show databases
 mysql result row, [information_schema]
 mysql result row, [mysql]
 mysql result row, [performance_schema]
 mysql result row, [sys]
 mysql result row, [test]
 mysql ok, 0
-mysql request, 3, \x00\x01show tables
+mysql request, 3, show tables
 mysql result row, [t]
 mysql ok, 0
 mysql request, 4, t\x00
 mysql ok, 0
-mysql request, 3, \x00\x01select @@version_comment limit 1
+mysql request, 3, select @@version_comment limit 1
 mysql result row, [MySQL Community Server - GPL]
 mysql ok, 0
 mysql request, 1, 

testing/btest/Baseline/scripts.base.protocols.mysql.query-attr/mysql.log

@@ -0,0 +1,14 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	mysql
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	cmd	arg	success	rows	response
+#types	time	string	addr	port	addr	port	string	string	bool	count	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	51682	127.0.0.1	3306	login	ykg	T	0	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	51682	127.0.0.1	3306	query	select @@version_comment limit 1	T	0	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	51682	127.0.0.1	3306	query	select now()	T	0	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	51682	127.0.0.1	3306	query	select now()	T	0	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.mysql.query-attr/out

@@ -0,0 +1,12 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+mysql handshake, ykg
+mysql ok, 0
+mysql request, 3, select @@version_comment limit 1
+mysql result row, [Source distribution]
+mysql ok, 0
+mysql request, 3, select now()
+mysql result row, [2022-07-13 10:45:41]
+mysql ok, 0
+mysql request, 3, select now()
+mysql result row, [2022-07-13 10:45:43]
+mysql ok, 0

testing/btest/scripts/base/protocols/mysql/caching_sha2_password-after-auth-switch.test

@@ -0,0 +1,35 @@
+# @TEST-EXEC: zeek -b -C -r $TRACES/mysql/caching_sha2_password-after-auth-switch.pcapng %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff mysql.log
+
+@load base/protocols/mysql
+
+event mysql_ok(c: connection, affected_rows: count)
+	{
+	print "mysql ok", affected_rows;
+	}
+
+event mysql_eof(c: connection, is_intermediate: bool)
+	{
+	print "mysql eof", is_intermediate;
+	}
+
+event mysql_result_row(c: connection, row: string_vec)
+	{
+	print "mysql result row", row;
+	}
+
+event mysql_error(c: connection, code: count, msg: string)
+	{
+	print "mysql error", code, msg;
+	}
+
+event mysql_command_request(c: connection, command: count, arg: string)
+	{
+	print "mysql request", command, arg;
+	}
+
+event mysql_handshake(c: connection, username: string)
+	{
+	print "mysql handshake", username;
+	}

testing/btest/scripts/base/protocols/mysql/query-attr.test

@@ -0,0 +1,35 @@
+# @TEST-EXEC: zeek -b -C -r $TRACES/mysql/query-attr.pcapng %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff mysql.log
+
+@load base/protocols/mysql
+
+event mysql_ok(c: connection, affected_rows: count)
+	{
+	print "mysql ok", affected_rows;
+	}
+
+event mysql_eof(c: connection, is_intermediate: bool)
+	{
+	print "mysql eof", is_intermediate;
+	}
+
+event mysql_result_row(c: connection, row: string_vec)
+	{
+	print "mysql result row", row;
+	}
+
+event mysql_error(c: connection, code: count, msg: string)
+	{
+	print "mysql error", code, msg;
+	}
+
+event mysql_command_request(c: connection, command: count, arg: string)
+	{
+	print "mysql request", command, arg;
+	}
+
+event mysql_handshake(c: connection, username: string)
+	{
+	print "mysql handshake", username;
+	}