diff --git a/CHANGES b/CHANGES index d78a5a4c13..683583c389 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,42 @@ +3.2.0-dev.221 | 2020-03-11 11:21:20 -0700 + + * Made additional MySQL fixes. (Vlad Grigorescu) + + 1) There are a couple more places where the new protocol uses and OK + packet instead of the deprecated EOF. + + 2) With > 255 results, we could end up in an situation where the uint8 + sequence number would wrap, and we'd naively think it'd be a new + handshake. + + Now, we track the previous sequence number, and assume overflow if it + was 255 previously and 0 now. + + We also reset the previous sequence number to 0 in various packets + that we'd expect at the end of other commands. + + * Add support to MySQL for deprecation of EOF packets. (Vlad Grigorescu) + + From the docs: "As of MySQL 5.7.5, OK packes are also used to indicate + EOF, and EOF packets are deprecated." + + The client sets a capability flag (CLIENT_DEPRECATE_EOF) to indicate + that it expects an OK instead of an EOF after the resultset rows. + + * MySQL analyzer whitespace cleanup (Vlad Grigorescu) + + * Fix EOF detection in the MySQL protocol analyzer. (Vlad Grigorescu) + + The MySQL documentation + (https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_eof_packet.html) + warns us that "You must check whether the packet length is less than 9 to + make sure that it is a EOF_Packet packet." + + While we were doing this in two places, we were comparing the total + packet length, which includes the 4-byte header. Changed to compare to + 13 instead. + 3.2.0-dev.214 | 2020-03-09 13:35:26 -0700 * Stop running GitHub Actions in forked repos (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index 6063142acb..93b57d86a2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0-dev.214 +3.2.0-dev.221 diff --git a/src/analyzer/protocol/mysql/mysql-protocol.pac b/src/analyzer/protocol/mysql/mysql-protocol.pac index b61cd0c4bc..6b4128f967 100644 --- a/src/analyzer/protocol/mysql/mysql-protocol.pac +++ b/src/analyzer/protocol/mysql/mysql-protocol.pac @@ -7,18 +7,18 @@ # Basic Types type uint24le = record { - byte3 : uint8; - byte2 : uint8; - byte1 : uint8; + byte3: uint8; + byte2: uint8; + byte1: uint8; }; type LengthEncodedInteger = record { length : uint8; - integer : LengthEncodedIntegerLookahead(length); + integer: LengthEncodedIntegerLookahead(length); }; type LengthEncodedIntegerArg(length: uint8) = record { - integer : LengthEncodedIntegerLookahead(length); + integer: LengthEncodedIntegerLookahead(length); }; type LengthEncodedIntegerLookahead(length: uint8) = record { @@ -151,18 +151,23 @@ enum Expected { EXPECT_AUTH_SWITCH, }; +enum Client_Capabilities { + # Expects an OK (instead of EOF) after the resultset rows of a Text Resultset. + CLIENT_DEPRECATE_EOF = 0x01000000, +}; + type NUL_String = RE/[^\0]*\0/; # MySQL PDU type MySQL_PDU(is_orig: bool) = record { - hdr : Header; - msg : case is_orig of { + hdr : Header; + msg : case is_orig of { false -> server_msg: Server_Message(hdr.seq_id, hdr.len); true -> client_msg: Client_Message(state); } &requires(state); } &let { - state : int = $context.connection.get_state(); + state: int = $context.connection.get_state(); } &length=hdr.len &byteorder=bigendian; type Header = record { @@ -172,9 +177,12 @@ type Header = record { len : uint32 = to_int()(le_len) + 4; } &length=4; -type Server_Message(seq_id: uint8, pkt_len: uint32) = case seq_id of { - 0 -> initial_handshake: Initial_Handshake_Packet; - default -> command_response : Command_Response(pkt_len); +type Server_Message(seq_id: uint8, pkt_len: uint32) = case is_initial of { + true -> initial_handshake: Initial_Handshake_Packet; + false -> command_response : Command_Response(pkt_len); +} &let { + is_initial : bool = (seq_id == 0) && ($context.connection.get_previous_seq_id() != 255); + update_seq_id : bool = $context.connection.set_previous_seq_id(seq_id); }; type Client_Message(state: int) = case state of { @@ -185,158 +193,172 @@ type Client_Message(state: int) = case state of { # Handshake Request type Initial_Handshake_Packet = record { - version : uint8; - pkt : case version of { - 10 -> handshake10 : Handshake_v10; - 9 -> handshake9 : Handshake_v9; - default -> error : ERR_Packet; + version : uint8; + pkt : case version of { + 10 -> handshake10: Handshake_v10; + 9 -> handshake9 : Handshake_v9; + default -> error : ERR_Packet; }; } &let { - set_version : bool = $context.connection.set_version(version); + set_version: bool = $context.connection.set_version(version); }; type Handshake_v10 = record { - server_version : NUL_String; - connection_id : uint32; - auth_plugin_data_part_1 : bytestring &length=8; - filler_1 : uint8; - capability_flag_1 : uint16; - character_set : uint8; - status_flags : uint16; - capability_flags_2 : uint16; - auth_plugin_data_len : uint8; - auth_plugin_name : NUL_String; + server_version : NUL_String; + connection_id : uint32; + auth_plugin_data_part_1: bytestring &length=8; + filler_1 : uint8; + capability_flag_1 : uint16; + character_set : uint8; + status_flags : uint16; + capability_flags_2 : uint16; + auth_plugin_data_len : uint8; + auth_plugin_name : NUL_String; }; type Handshake_v9 = record { - server_version : NUL_String; - connection_id : uint32; - scramble : NUL_String; + server_version: NUL_String; + connection_id : uint32; + scramble : NUL_String; }; # Handshake Response type Handshake_Response_Packet = case $context.connection.get_version() of { - 10 -> v10_response : Handshake_Response_Packet_v10; - 9 -> v9_response : Handshake_Response_Packet_v9; + 10 -> v10_response: Handshake_Response_Packet_v10; + 9 -> v9_response : Handshake_Response_Packet_v9; } &let { - version : uint8 = $context.connection.get_version(); + version: uint8 = $context.connection.get_version(); } &byteorder=bigendian; type Handshake_Response_Packet_v10 = record { - cap_flags : uint32; - max_pkt_size : uint32; - char_set : uint8; - pad : padding[23]; - username : NUL_String; - password : bytestring &restofdata; + cap_flags : uint32; + max_pkt_size: uint32; + char_set : uint8; + pad : padding[23]; + username : NUL_String; + password : bytestring &restofdata; +} &let { + deprecate_eof: bool = $context.connection.set_deprecate_eof(cap_flags & CLIENT_DEPRECATE_EOF); }; type Handshake_Response_Packet_v9 = record { - cap_flags : uint16; - max_pkt_size : uint24le; - username : NUL_String; - auth_response : NUL_String; - have_db : case ( cap_flags & 0x8 ) of { - 0x8 -> database : NUL_String; - 0x0 -> none : empty; + cap_flags : uint16; + max_pkt_size : uint24le; + username : NUL_String; + auth_response: NUL_String; + have_db : case ( cap_flags & 0x8 ) of { + 0x8 -> database: NUL_String; + 0x0 -> none : empty; }; - password : bytestring &restofdata; + password : bytestring &restofdata; }; # Command Request type Command_Request_Packet = record { - command : uint8; - arg : bytestring &restofdata; + command: uint8; + arg : bytestring &restofdata; } &let { - update_expectation : bool = $context.connection.set_next_expected_from_command(command); + update_expectation: bool = $context.connection.set_next_expected_from_command(command); }; # Command Response type Command_Response(pkt_len: uint32) = case $context.connection.get_expectation() of { - EXPECT_COLUMN_COUNT -> col_count_meta : ColumnCountMeta; - EXPECT_COLUMN_DEFINITION -> col_def : ColumnDefinition; - EXPECT_COLUMN_DEFINITION_OR_EOF -> def_or_eof : ColumnDefinitionOrEOF(pkt_len); - EXPECT_RESULTSET -> resultset : Resultset(pkt_len); - EXPECT_REST_OF_PACKET -> rest : bytestring &restofdata; - EXPECT_STATUS -> status : Command_Response_Status; - EXPECT_AUTH_SWITCH -> auth_switch : AuthSwitchRequest; - EXPECT_EOF -> eof : EOF1; - default -> unknow : empty; + EXPECT_COLUMN_COUNT -> col_count_meta: ColumnCountMeta; + EXPECT_COLUMN_DEFINITION -> col_def : ColumnDefinition; + EXPECT_COLUMN_DEFINITION_OR_EOF -> def_or_eof : ColumnDefinitionOrEOF(pkt_len); + EXPECT_RESULTSET -> resultset : Resultset(pkt_len); + EXPECT_REST_OF_PACKET -> rest : bytestring &restofdata; + EXPECT_STATUS -> status : Command_Response_Status; + EXPECT_AUTH_SWITCH -> auth_switch : AuthSwitchRequest; + EXPECT_EOF -> eof : EOFIfLegacy; + default -> unknown : empty; }; type Command_Response_Status = record { pkt_type: uint8; response: case pkt_type of { - 0x00 -> data_ok: OK_Packet; - 0xfe -> data_eof: EOF_Packet; - 0xff -> data_err: ERR_Packet; - default -> unknown: empty; + 0x00 -> data_ok: OK_Packet; + 0xfe -> data_eof: EOF_Packet; + 0xff -> data_err: ERR_Packet; + default -> unknown: empty; }; }; type ColumnCountMeta = record { byte : uint8; pkt_type: case byte of { - 0x00 -> ok : OK_Packet; - 0xff -> err : ERR_Packet; + 0x00 -> ok : OK_Packet; + 0xff -> err : ERR_Packet; # 0xfb -> Not implemented default -> col_count: ColumnCount(byte); }; }; type ColumnCount(byte: uint8) = record { - le_column_count : LengthEncodedIntegerLookahead(byte); + le_column_count : LengthEncodedIntegerLookahead(byte); } &let { - col_num : uint32 = to_int()(le_column_count); - update_col_num : bool = $context.connection.set_col_count(col_num); - update_remain : bool = $context.connection.set_remaining_cols(col_num); - update_expectation : bool = $context.connection.set_next_expected(EXPECT_COLUMN_DEFINITION); + col_num : uint32 = to_int()(le_column_count); + update_col_num : bool = $context.connection.set_col_count(col_num); + update_remain : bool = $context.connection.set_remaining_cols(col_num); + update_expectation: bool = $context.connection.set_next_expected(EXPECT_COLUMN_DEFINITION); }; type ColumnDefinition = record { dummy: uint8; - def : ColumnDefinition41(dummy); + def : ColumnDefinition41(dummy); } &let { - update_remain : bool = $context.connection.dec_remaining_cols(); - update_expectation : bool = $context.connection.set_next_expected($context.connection.get_remaining_cols() > 0 ? EXPECT_COLUMN_DEFINITION : EXPECT_EOF); + update_remain : bool = $context.connection.dec_remaining_cols(); + update_expectation: bool = $context.connection.set_next_expected($context.connection.get_remaining_cols() > 0 ? EXPECT_COLUMN_DEFINITION : EXPECT_EOF); +}; + +type EOFOrOK = case $context.connection.get_deprecate_eof() of { + false -> eof: EOF_Packet; + true -> ok: OK_Packet; }; type ColumnDefinitionOrEOF(pkt_len: uint32) = record { - marker: uint8; + marker : uint8; def_or_eof: case is_eof of { - true -> eof: EOF_Packet; + true -> eof: EOFOrOK; false -> def: ColumnDefinition41(marker); } &requires(is_eof); } &let { - is_eof: bool = (marker == 0xfe && pkt_len <= 9); + # MySQL spec says "You must check whether the packet length is less than 9 + # to make sure that it is a EOF_Packet packet" so the value of 13 here + # comes from that 9, plus a 4-byte header. + is_eof: bool = (marker == 0xfe && pkt_len < 13); }; -type EOF1 = record { - eof : EOF_Packet; +type EOFIfLegacy = case $context.connection.get_deprecate_eof() of { + false -> eof: EOF_Packet; + true -> none: empty; } &let { - update_result_seen : bool = $context.connection.set_results_seen(0); - update_expectation : bool = $context.connection.set_next_expected(EXPECT_RESULTSET); + update_result_seen: bool = $context.connection.set_results_seen(0); + update_expectation: bool = $context.connection.set_next_expected(EXPECT_RESULTSET); }; type Resultset(pkt_len: uint32) = record { - marker: uint8; + marker : uint8; row_or_eof: case is_eof of { - true -> eof: EOF_Packet; + true -> eof: EOFOrOK; false -> row: ResultsetRow(marker); } &requires(is_eof); } &let { - is_eof: bool = (marker == 0xfe && pkt_len <= 9); - update_result_seen : bool = $context.connection.inc_results_seen(); - update_expectation : bool = $context.connection.set_next_expected(is_eof ? NO_EXPECTATION : EXPECT_RESULTSET); + # MySQL spec says "You must check whether the packet length is less than 9 + # to make sure that it is a EOF_Packet packet" so the value of 13 here + # comes from that 9, plus a 4-byte header. + is_eof : bool = (marker == 0xfe && pkt_len < 13); + update_result_seen: bool = $context.connection.inc_results_seen(); + update_expectation: bool = $context.connection.set_next_expected(is_eof ? NO_EXPECTATION : EXPECT_RESULTSET); }; type ResultsetRow(first_byte: uint8) = record { first_field: LengthEncodedStringArg(first_byte); - fields: LengthEncodedString[$context.connection.get_col_count() - 1]; + fields : LengthEncodedString[$context.connection.get_col_count() - 1]; }; type ColumnDefinition41(first_byte: uint8) = record { @@ -357,8 +379,8 @@ type ColumnDefinition41(first_byte: uint8) = record { type AuthSwitchRequest = record { status: uint8; - name: NUL_String; - data: bytestring &restofdata; + name : NUL_String; + data : bytestring &restofdata; }; type ColumnDefinition320 = record { @@ -371,8 +393,8 @@ type ColumnDefinition320 = record { }; type OK_Packet = record { - le_rows : LengthEncodedInteger; - todo : bytestring &restofdata; + le_rows: LengthEncodedInteger; + todo : bytestring &restofdata; } &let { rows : uint32 = to_int()(le_rows); update_state: bool = $context.connection.update_state(COMMAND_PHASE); @@ -398,20 +420,24 @@ type EOF_Packet = record { refine connection MySQL_Conn += { %member{ uint8 version_; + uint8 previous_seq_id_; int state_; Expected expected_; uint32 col_count_; uint32 remaining_cols_; uint32 results_seen_; + bool deprecate_eof_; %} %init{ version_ = 0; + previous_seq_id_ = 0; state_ = CONNECTION_PHASE; expected_ = EXPECT_STATUS; col_count_ = 0; remaining_cols_ = 0; results_seen_ = 0; + deprecate_eof_ = false; %} function get_version(): uint8 @@ -425,6 +451,17 @@ refine connection MySQL_Conn += { return true; %} + function get_previous_seq_id(): uint8 + %{ + return previous_seq_id_; + %} + + function set_previous_seq_id(s: uint8): bool + %{ + previous_seq_id_ = s; + return true; + %} + function get_state(): int %{ return state_; @@ -436,6 +473,17 @@ refine connection MySQL_Conn += { return true; %} + function get_deprecate_eof(): bool + %{ + return deprecate_eof_; + %} + + function set_deprecate_eof(d: bool): bool + %{ + deprecate_eof_ = d; + return true; + %} + function get_expectation(): Expected %{ return expected_; diff --git a/testing/btest/Baseline/scripts.base.protocols.mysql.wireshark/mysql.log b/testing/btest/Baseline/scripts.base.protocols.mysql.wireshark/mysql.log index ac18135111..9354f6b5d3 100644 --- a/testing/btest/Baseline/scripts.base.protocols.mysql.wireshark/mysql.log +++ b/testing/btest/Baseline/scripts.base.protocols.mysql.wireshark/mysql.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path mysql -#open 2018-05-17-04-01-33 +#open 2020-03-07-04-49-02 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd arg success rows response #types time string addr port addr port string string bool count string 1216281025.136728 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 login tfoerste T 0 - @@ -12,7 +12,7 @@ 1216281030.835395 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 init_db test T 0 - 1216281030.835742 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 query show databases T 0 - 1216281030.836349 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 query show tables T 0 - -1216281030.836757 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 field_list agent - - - +1216281030.836757 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 field_list agent T 0 - 1216281048.287657 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 query create table foo (id BIGINT( 10 ) UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, animal VARCHAR(64) NOT NULL, name VARCHAR(64) NULL DEFAULT NULL) ENGINE = MYISAM T 0 - 1216281057.746222 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 query insert into foo (animal, name) values ("dog", "Goofy") T 1 - 1216281061.713980 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 query insert into foo (animal, name) values ("cat", "Garfield") T 1 - @@ -24,4 +24,4 @@ 1216281116.209268 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 query delete from foo T 1 - 1216281122.880561 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 query drop table foo T 0 - 1216281124.418765 CHhAvVGS1DHFjwGM9 192.168.0.254 56162 192.168.0.254 3306 quit (empty) - - - -#close 2018-05-17-04-01-33 +#close 2020-03-07-04-49-02 diff --git a/testing/btest/Baseline/scripts.base.protocols.mysql.wireshark/out b/testing/btest/Baseline/scripts.base.protocols.mysql.wireshark/out index d322011e9d..87c86b0e0f 100644 --- a/testing/btest/Baseline/scripts.base.protocols.mysql.wireshark/out +++ b/testing/btest/Baseline/scripts.base.protocols.mysql.wireshark/out @@ -2,19 +2,24 @@ mysql ok, 0 mysql request, 3, select @@version_comment limit 1 mysql ok, 0 mysql result row, [Gentoo Linux mysql-5.0.54] +mysql ok, 0 mysql request, 3, SELECT DATABASE() mysql ok, 0 mysql result row, [] +mysql ok, 0 mysql request, 2, test mysql ok, 0 mysql request, 3, show databases mysql ok, 0 mysql result row, [information_schema] mysql result row, [test] +mysql ok, 0 mysql request, 3, show tables mysql ok, 0 mysql result row, [agent] +mysql ok, 0 mysql request, 4, agent\x00 +mysql ok, 0 mysql request, 3, create table foo (id BIGINT( 10 ) UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, animal VARCHAR(64) NOT NULL, name VARCHAR(64) NULL DEFAULT NULL) ENGINE = MYISAM mysql ok, 0 mysql request, 3, insert into foo (animal, name) values ("dog", "Goofy") @@ -25,6 +30,7 @@ mysql request, 3, select * from foo mysql ok, 0 mysql result row, [1, dog, Goofy] mysql result row, [2, cat, Garfield] +mysql ok, 0 mysql request, 3, delete from foo where name like '%oo%' mysql ok, 1 mysql request, 3, delete from foo where id = 1 @@ -32,9 +38,11 @@ mysql ok, 0 mysql request, 3, select count(*) from foo mysql ok, 0 mysql result row, [1] +mysql ok, 0 mysql request, 3, select * from foo mysql ok, 0 mysql result row, [2, cat, Garfield] +mysql ok, 0 mysql request, 3, delete from foo mysql ok, 1 mysql request, 3, drop table foo