diff --git a/CHANGES b/CHANGES index bad8abdf23..cdb3fca9d6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,24 @@ +8.0.0-dev.577 | 2025-07-01 14:19:26 -0400 + + * Touchup TODOs in the Redis analyzer (Evan Typanski, Corelight) + + Also renames `KnownCommand` to `RedisCommand` to avoid conflicts. + + * Handle more Redis RESP3 protocol pieces (Evan Typanski, Corelight) + + This passes the "minimum protocol version" along in the reply and adds + support for attributes, which were added relatively recently. + + * Stringify all Redis-RESP serialized data (Evan Typanski, Corelight) + + * GH-4504: Handle Redis protocol `message` separately (Evan Typanski, Corelight) + + Closes #4504 + + Messages are not typical responses, so they need special handling. This + is different between RESP2 and 3, so this is the first instance where + the script layer needs to tell the difference. + 8.0.0-dev.571 | 2025-07-01 11:03:23 +0200 * Bump pre-commit hooks (Benjamin Bannier, Corelight) diff --git a/VERSION b/VERSION index 413be5af09..a3cecee4dd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.0.0-dev.571 +8.0.0-dev.577 diff --git a/scripts/base/protocols/redis/main.zeek b/scripts/base/protocols/redis/main.zeek index ab917df8c9..07b6a6a61d 100644 --- a/scripts/base/protocols/redis/main.zeek +++ b/scripts/base/protocols/redis/main.zeek @@ -41,6 +41,11 @@ export { end: count &optional; }; + type RESPVersion: enum { + RESP2, + RESP3 + }; + type State: record { ## Pending commands. pending: table[count] of Info; @@ -52,14 +57,34 @@ export { ## Each range is one or two elements, one meaning it's unbounded, two meaning ## it begins at one and ends at the second. no_reply_ranges: vector of NoReplyRange; + ## The command indexes (from current_command and current_reply) that will + ## not get responses no matter what. + skip_commands: set[count]; ## We store if this analyzer had a violation to avoid logging if so. ## This should not be super necessary, but worth a shot. violation: bool &default=F; + ## If we are in "subscribed" mode + subscribed_mode: bool &default=F; + ## The RESP version + resp_version: RESPVersion &default=RESP2; }; # Redis specifically mentions 10k commands as a good pipelining threshold, so # we'll piggyback on that. option max_pending_commands = 10000; + + # These commands enter subscribed mode + global enter_subscribed_mode = [RedisCommand_PSUBSCRIBE, + RedisCommand_SSUBSCRIBE, RedisCommand_SUBSCRIBE]; + + # These commands exit subscribed mode + global exit_subscribed_mode = [RedisCommand_RESET, RedisCommand_QUIT]; + + # These commands don't expect a response (ever) - their replies are out of band. + global no_response_commands = [RedisCommand_PSUBSCRIBE, + RedisCommand_PUNSUBSCRIBE, RedisCommand_SSUBSCRIBE, + RedisCommand_SUBSCRIBE, RedisCommand_SUNSUBSCRIBE, + RedisCommand_UNSUBSCRIBE]; } redef record connection += { @@ -122,7 +147,16 @@ function is_last_interval_closed(c: connection): bool c$redis_state$no_reply_ranges[-1]?$end; } -event Redis::command(c: connection, cmd: Command) +event hello_command(c: connection, hello: HelloCommand) + { + if ( ! c?$redis_state ) + make_new_state(c); + + if ( hello?$requested_resp_version && hello$requested_resp_version == "3" ) + c$redis_state$resp_version = RESP3; + } + +event command(c: connection, cmd: Command) { if ( ! c?$redis_state ) make_new_state(c); @@ -139,10 +173,30 @@ event Redis::command(c: connection, cmd: Command) } ++c$redis_state$current_command; + + if ( cmd?$known ) + { + if ( c$redis_state$resp_version == RESP2 ) + { + local should_enter = cmd$known in enter_subscribed_mode; + local should_exit = cmd$known in exit_subscribed_mode; + c$redis_state$subscribed_mode = should_enter && ! should_exit; + + # It's weird if it's in both - in the future users may be able to add that + if ( should_enter && should_exit ) + Reporter::conn_weird("Redis_command_enter_exit_subscribed_mode", c, cat( + cmd$known)); + } + if ( cmd$known in no_response_commands || c$redis_state$subscribed_mode ) + { + add c$redis_state$skip_commands[c$redis_state$current_command]; + } + } + # CLIENT commands can skip a number of replies and may be used with # pipelining. We need special logic in order to track the command/reply # pairs. - if ( cmd?$known && cmd$known == KnownCommand_CLIENT ) + if ( cmd?$known && cmd$known == RedisCommand_CLIENT ) { # All 3 CLIENT commands we care about have 3 elements if ( |cmd$raw| == 3 ) @@ -177,6 +231,7 @@ event Redis::command(c: connection, cmd: Command) } } } + set_state(c, T); c$redis$cmd = cmd; @@ -187,17 +242,24 @@ event Redis::command(c: connection, cmd: Command) function reply_num(c: connection): count { local resp_num = c$redis_state$current_reply + 1; + local result = resp_num; for ( i in c$redis_state$no_reply_ranges ) { local range = c$redis_state$no_reply_ranges[i]; if ( ! range?$end && resp_num > range$begin ) { } # TODO: This is necessary if not using pipelining if ( range?$end && resp_num >= range$begin && resp_num < range$end ) - return range$end; + result = range$end; } - # Default: no disable/enable shenanigans - return resp_num; + # Account for commands that don't expect a response + while ( result in c$redis_state$skip_commands ) + { + delete c$redis_state$skip_commands[result]; + result += 1; + } + + return result; } # Logs up to and including the last seen command from the last reply @@ -229,11 +291,24 @@ function log_from(c: connection, previous_reply_num: count) } } -event Redis::reply(c: connection, data: ReplyData) +event reply(c: connection, data: ReplyData) { if ( ! c?$redis_state ) make_new_state(c); + # If the server is talking in RESP3, mark accordingly, even if we didn't see HELLO + if ( data$min_protocol_version == 3 ) + { + c$redis_state$resp_version = RESP3; + c$redis_state$subscribed_mode = F; + } + + if ( c$redis_state$subscribed_mode ) + { + event server_push(c, data); + return; + } + local previous_reply_num = c$redis_state$current_reply; c$redis_state$current_reply = reply_num(c); set_state(c, F); @@ -241,9 +316,13 @@ event Redis::reply(c: connection, data: ReplyData) c$redis$reply = data; c$redis$success = T; log_from(c, previous_reply_num); + + # Tidy up the skip_commands when it's up to date + if ( c$redis_state$current_command == c$redis_state$current_reply ) + clear_table(c$redis_state$skip_commands); } -event Redis::error(c: connection, data: ReplyData) +event error(c: connection, data: ReplyData) { if ( ! c?$redis_state ) make_new_state(c); diff --git a/scripts/base/protocols/redis/spicy-events.zeek b/scripts/base/protocols/redis/spicy-events.zeek index ad9a500f18..8e0e6c733e 100644 --- a/scripts/base/protocols/redis/spicy-events.zeek +++ b/scripts/base/protocols/redis/spicy-events.zeek @@ -37,6 +37,12 @@ export { password: string; }; + ## The Redis HELLO command (handshake). + type HelloCommand: record { + ## The sent requested RESP version, such as "2" or "3" + requested_resp_version: string &optional; + }; + ## A generic Redis command from the client. type Command: record { ## The raw command, exactly as parsed @@ -49,12 +55,17 @@ export { ## The value, if this command is known to have a value value: string &log &optional; ## The command in an enum if it was known - known: KnownCommand &optional; + known: RedisCommand &optional; }; ## A generic Redis reply from the client. type ReplyData: record { - value: string &log &optional; + ## The RESP3 attributes applied to this, if any + attributes: string &optional; + ## The string version of the reply data + value: string &log; + ## The minimum RESP version that supports this reply type + min_protocol_version: count; }; } @@ -79,6 +90,13 @@ global get_command: event(c: connection, key: string); ## command: The AUTH command sent to the server and its data. global auth_command: event(c: connection, command: AuthCommand); +## Generated for Redis HELLO commands sent to the Redis server. +## +## c: The connection. +## +## command: The HELLO command sent to the server and its data. +global hello_command: event(c: connection, command: HelloCommand); + ## Generated for every command sent by the client to the Redis server. ## ## c: The connection. @@ -87,11 +105,15 @@ global auth_command: event(c: connection, command: AuthCommand); global command: event(c: connection, cmd: Command); ## Generated for every successful response sent by the Redis server to the -## client. +## client. For RESP2, this includes "push" messages, which are out of band. +## These will also raise a server_push event. RESP3 push messages will only +## raise a server_push event. ## ## c: The connection. ## ## data: The server data sent to the client. +## +## .. zeek:see:: Redis::server_push global reply: event(c: connection, data: ReplyData); ## Generated for every error response sent by the Redis server to the @@ -101,3 +123,11 @@ global reply: event(c: connection, data: ReplyData); ## ## data: The server data sent to the client. global error: event(c: connection, data: ReplyData); + +## Generated for out-of-band data, outside of the request-response +## model. +## +## c: The connection. +## +## data: The server data sent to the client. +global server_push: event(c: connection, data: ReplyData); diff --git a/src/analyzer/protocol/redis/redis.spicy b/src/analyzer/protocol/redis/redis.spicy index a2f23d4c6d..c8dc9a8003 100644 --- a/src/analyzer/protocol/redis/redis.spicy +++ b/src/analyzer/protocol/redis/redis.spicy @@ -6,7 +6,7 @@ module Redis; import RESP; -public type KnownCommand = enum { +public type RedisCommand = enum { APPEND, AUTH, BITCOUNT, @@ -34,6 +34,7 @@ public type KnownCommand = enum { GETRANGE, GETSET, HDEL, + HELLO, HGET, HSET, INCR, @@ -43,11 +44,19 @@ public type KnownCommand = enum { MOVE, MSET, PERSIST, + PSUBSCRIBE, + PUNSUBSCRIBE, + QUIT, RENAME, + RESET, SET, STRLEN, + SUBSCRIBE, + SSUBSCRIBE, + SUNSUBSCRIBE, TTL, TYPE, + UNSUBSCRIBE, }; type Command = struct { @@ -55,7 +64,7 @@ type Command = struct { name: bytes; key: optional; value: optional; - known: optional; + known: optional; }; # This just assumes all elements in the array is a bulk string and puts them in a vector @@ -149,44 +158,44 @@ function parse_command(raw: vector): Command { if (|raw| >= 2) { switch (*cmd) { - case KnownCommand::KEYS: + case RedisCommand::KEYS: parsed.key = raw[1]; - case KnownCommand::APPEND, - KnownCommand::BITCOUNT, - KnownCommand::BITFIELD, - KnownCommand::BITFIELD_RO, - KnownCommand::BITPOS, - KnownCommand::BLPOP, - KnownCommand::BRPOP, - KnownCommand::COPY, - KnownCommand::DECR, - KnownCommand::DECRBY, - KnownCommand::DEL, - KnownCommand::DUMP, - KnownCommand::EXISTS, - KnownCommand::EXPIRE, - KnownCommand::EXPIREAT, - KnownCommand::EXPIRETIME, - KnownCommand::GET, - KnownCommand::GETBIT, - KnownCommand::GETDEL, - KnownCommand::GETEX, - KnownCommand::GETRANGE, - KnownCommand::GETSET, - KnownCommand::HDEL, - KnownCommand::HGET, - KnownCommand::HSET, - KnownCommand::INCR, - KnownCommand::INCRBY, - KnownCommand::MGET, - KnownCommand::MOVE, - KnownCommand::MSET, - KnownCommand::PERSIST, - KnownCommand::RENAME, - KnownCommand::SET, - KnownCommand::STRLEN, - KnownCommand::TTL, - KnownCommand::TYPE: + case RedisCommand::APPEND, + RedisCommand::BITCOUNT, + RedisCommand::BITFIELD, + RedisCommand::BITFIELD_RO, + RedisCommand::BITPOS, + RedisCommand::BLPOP, + RedisCommand::BRPOP, + RedisCommand::COPY, + RedisCommand::DECR, + RedisCommand::DECRBY, + RedisCommand::DEL, + RedisCommand::DUMP, + RedisCommand::EXISTS, + RedisCommand::EXPIRE, + RedisCommand::EXPIREAT, + RedisCommand::EXPIRETIME, + RedisCommand::GET, + RedisCommand::GETBIT, + RedisCommand::GETDEL, + RedisCommand::GETEX, + RedisCommand::GETRANGE, + RedisCommand::GETSET, + RedisCommand::HDEL, + RedisCommand::HGET, + RedisCommand::HSET, + RedisCommand::INCR, + RedisCommand::INCRBY, + RedisCommand::MGET, + RedisCommand::MOVE, + RedisCommand::MSET, + RedisCommand::PERSIST, + RedisCommand::RENAME, + RedisCommand::SET, + RedisCommand::STRLEN, + RedisCommand::TTL, + RedisCommand::TYPE: parsed.key = raw[1]; default: (); } @@ -194,22 +203,22 @@ function parse_command(raw: vector): Command { if (|raw| >= 3) { switch (*cmd) { - case KnownCommand::SET, - KnownCommand::APPEND, - KnownCommand::DECRBY, - KnownCommand::EXPIRE, - KnownCommand::EXPIREAT, - KnownCommand::GETBIT, - KnownCommand::GETSET, - KnownCommand::HDEL, - KnownCommand::HGET, - KnownCommand::INCRBY, - KnownCommand::MOVE, - KnownCommand::MSET, - KnownCommand::RENAME: + case RedisCommand::SET, + RedisCommand::APPEND, + RedisCommand::DECRBY, + RedisCommand::EXPIRE, + RedisCommand::EXPIREAT, + RedisCommand::GETBIT, + RedisCommand::GETSET, + RedisCommand::HDEL, + RedisCommand::HGET, + RedisCommand::INCRBY, + RedisCommand::MOVE, + RedisCommand::MSET, + RedisCommand::RENAME: parsed.value = raw[2]; # Op first, destination second, then a list of keys. Just log dest - case KnownCommand::BITOP: parsed.key = raw[2]; + case RedisCommand::BITOP: parsed.key = raw[2]; default: (); } } @@ -217,7 +226,7 @@ function parse_command(raw: vector): Command { if (|raw| >= 4) { switch (*cmd) { # timeout, numkeys, then key - case KnownCommand::BLMPOP: parsed.key = raw[3]; + case RedisCommand::BLMPOP: parsed.key = raw[3]; default: (); } } @@ -225,51 +234,60 @@ function parse_command(raw: vector): Command { return parsed; } -function command_from(cmd_bytes: bytes): optional { - local cmd: optional = Null; +function command_from(cmd_bytes: bytes): optional { + local cmd: optional = Null; switch (cmd_bytes.lower()) { - case b"set": cmd = KnownCommand::SET; - case b"append": cmd = KnownCommand::APPEND; - case b"auth": cmd = KnownCommand::AUTH; - case b"bitcount": cmd = KnownCommand::BITCOUNT; - case b"bitfield": cmd = KnownCommand::BITFIELD; - case b"bitfield_ro": cmd = KnownCommand::BITFIELD_RO; - case b"bitop": cmd = KnownCommand::BITOP; - case b"bitpos": cmd = KnownCommand::BITPOS; - case b"blmpop": cmd = KnownCommand::BLMPOP; - case b"blpop": cmd = KnownCommand::BLPOP; - case b"brpop": cmd = KnownCommand::BRPOP; - case b"client": cmd = KnownCommand::CLIENT; - case b"copy": cmd = KnownCommand::COPY; - case b"decr": cmd = KnownCommand::DECR; - case b"decrby": cmd = KnownCommand::DECRBY; - case b"del": cmd = KnownCommand::DEL; - case b"dump": cmd = KnownCommand::DUMP; - case b"exists": cmd = KnownCommand::EXISTS; - case b"expire": cmd = KnownCommand::EXPIRE; - case b"expireat": cmd = KnownCommand::EXPIREAT; - case b"expiretime": cmd = KnownCommand::EXPIRETIME; - case b"expiretime": cmd = KnownCommand::EXPIRETIME; - case b"get": cmd = KnownCommand::GET; - case b"getbit": cmd = KnownCommand::GETBIT; - case b"getdel": cmd = KnownCommand::GETDEL; - case b"getex": cmd = KnownCommand::GETEX; - case b"getrange": cmd = KnownCommand::GETRANGE; - case b"getset": cmd = KnownCommand::GETSET; - case b"hdel": cmd = KnownCommand::HDEL; - case b"hget": cmd = KnownCommand::HGET; - case b"hset": cmd = KnownCommand::HSET; - case b"incr": cmd = KnownCommand::INCR; - case b"incrby": cmd = KnownCommand::INCRBY; - case b"keys": cmd = KnownCommand::KEYS; - case b"mget": cmd = KnownCommand::MGET; - case b"move": cmd = KnownCommand::MOVE; - case b"mset": cmd = KnownCommand::MSET; - case b"persist": cmd = KnownCommand::PERSIST; - case b"rename": cmd = KnownCommand::RENAME; - case b"strlen": cmd = KnownCommand::STRLEN; - case b"ttl": cmd = KnownCommand::TTL; - case b"type": cmd = KnownCommand::TYPE; + case b"append": cmd = RedisCommand::APPEND; + case b"auth": cmd = RedisCommand::AUTH; + case b"bitcount": cmd = RedisCommand::BITCOUNT; + case b"bitfield": cmd = RedisCommand::BITFIELD; + case b"bitfield_ro": cmd = RedisCommand::BITFIELD_RO; + case b"bitop": cmd = RedisCommand::BITOP; + case b"bitpos": cmd = RedisCommand::BITPOS; + case b"blmpop": cmd = RedisCommand::BLMPOP; + case b"blpop": cmd = RedisCommand::BLPOP; + case b"brpop": cmd = RedisCommand::BRPOP; + case b"client": cmd = RedisCommand::CLIENT; + case b"copy": cmd = RedisCommand::COPY; + case b"decr": cmd = RedisCommand::DECR; + case b"decrby": cmd = RedisCommand::DECRBY; + case b"del": cmd = RedisCommand::DEL; + case b"dump": cmd = RedisCommand::DUMP; + case b"exists": cmd = RedisCommand::EXISTS; + case b"expire": cmd = RedisCommand::EXPIRE; + case b"expireat": cmd = RedisCommand::EXPIREAT; + case b"expiretime": cmd = RedisCommand::EXPIRETIME; + case b"expiretime": cmd = RedisCommand::EXPIRETIME; + case b"get": cmd = RedisCommand::GET; + case b"getbit": cmd = RedisCommand::GETBIT; + case b"getdel": cmd = RedisCommand::GETDEL; + case b"getex": cmd = RedisCommand::GETEX; + case b"getrange": cmd = RedisCommand::GETRANGE; + case b"getset": cmd = RedisCommand::GETSET; + case b"hdel": cmd = RedisCommand::HDEL; + case b"hello": cmd = RedisCommand::HELLO; + case b"hget": cmd = RedisCommand::HGET; + case b"hset": cmd = RedisCommand::HSET; + case b"incr": cmd = RedisCommand::INCR; + case b"incrby": cmd = RedisCommand::INCRBY; + case b"keys": cmd = RedisCommand::KEYS; + case b"mget": cmd = RedisCommand::MGET; + case b"move": cmd = RedisCommand::MOVE; + case b"mset": cmd = RedisCommand::MSET; + case b"persist": cmd = RedisCommand::PERSIST; + case b"psubscribe": cmd = RedisCommand::PSUBSCRIBE; + case b"punsubscribe": cmd = RedisCommand::PUNSUBSCRIBE; + case b"quit": cmd = RedisCommand::QUIT; + case b"rename": cmd = RedisCommand::RENAME; + case b"reset": cmd = RedisCommand::RESET; + case b"set": cmd = RedisCommand::SET; + case b"strlen": cmd = RedisCommand::STRLEN; + case b"ssubscribe": cmd = RedisCommand::SSUBSCRIBE; + case b"subscribe": cmd = RedisCommand::SUBSCRIBE; + case b"sunsubscribe": cmd = RedisCommand::SUNSUBSCRIBE; + case b"ttl": cmd = RedisCommand::TTL; + case b"type": cmd = RedisCommand::TYPE; + case b"unsubscribe": cmd = RedisCommand::UNSUBSCRIBE; default: cmd = Null; } @@ -334,7 +352,7 @@ public function make_set(command: Command): Set { } public function is_set(data: RESP::ClientData): bool { - return data.command.known && *(data.command.known) == KnownCommand::SET && data.command.key && data.command.value; + return data.command.known && *(data.command.known) == RedisCommand::SET && data.command.key && data.command.value; } type Get = struct { @@ -347,7 +365,7 @@ public function make_get(command: Command): Get { } public function is_get(data: RESP::ClientData): bool { - return data.command.known && *(data.command.known) == KnownCommand::GET && |data.command.raw| >= 2; + return data.command.known && *(data.command.known) == RedisCommand::GET && |data.command.raw| >= 2; } type Auth = struct { @@ -365,15 +383,45 @@ public function make_auth(command: Command): Auth { } public function is_auth(data: RESP::ClientData): bool { - return data.command.known && *(data.command.known) == KnownCommand::AUTH && |data.command.raw| >= 2; + return data.command.known && *(data.command.known) == RedisCommand::AUTH && |data.command.raw| >= 2; +} + +type Hello = struct { + requested_resp_version: optional; +}; + +public function make_hello(command: Command): Hello { + local hi: Hello = [$requested_resp_version = Null]; + if (|command.raw| > 1) + hi.requested_resp_version = command.raw[1]; + return hi; +} + +public function is_hello(data: RESP::ClientData): bool { + return data.command.known && *(data.command.known) == RedisCommand::HELLO; } type ReplyData = struct { - value: optional; + attributes: optional; + value: bytes; + min_protocol_version: uint8; }; -public function is_err(server_data: RESP::ServerData): bool { - return server_data.data?.simple_error || server_data.data?.bulk_error; +public type ReplyType = enum { + Reply, # A response to a command + Error, # An error response to a command + Push, # A server message that is not responding to a command +}; + +public function classify(server_data: RESP::ServerData): ReplyType { + if (server_data.data?.simple_error || server_data.data?.bulk_error) + return ReplyType::Error; + + # We can tell with RESP3 this is push here, but RESP2 relies on scripts + if (server_data.data?.push) + return ReplyType::Push; + + return ReplyType::Reply; } function bulk_string_content(bulk: RESP::BulkString): bytes { @@ -383,21 +431,108 @@ function bulk_string_content(bulk: RESP::BulkString): bytes { return b""; } -# Gets the server reply in a simpler form -public function make_server_reply(data: RESP::ServerData): ReplyData { - local res: ReplyData = [$value = Null]; - if (data.data?.simple_error) - res.value = data.data.simple_error.content; - else if (data.data?.bulk_error) - res.value = bulk_string_content(data.data.bulk_error); - else if (data.data?.simple_string) - res.value = data.data.simple_string.content; - else if (data.data?.bulk_string) - res.value = bulk_string_content(data.data.bulk_string); - else if (data.data?.verbatim_string) - res.value = bulk_string_content(data.data.verbatim_string); - else if (data.data?.boolean) - res.value = data.data.boolean.val ? b"T" : b"F"; - +function stringify_map(data: RESP::Map&): bytes { + local res = b"{"; + local first = True; + local i = 0; + # num_elements refers to the number of map entries, each with 2 entries + # in the raw data + while (i < data.num_elements) { + if (!first) + res += b", "; + res += stringify(data.raw_data[i * 2]); + res += b": "; + res += stringify(data.raw_data[(i * 2) + 1]); + i += 1; + first = False; + } + res += b"}"; return res; } + +# Returns the bytes string value of this, or Null if it cannot. +function stringify(data: RESP::Data&): bytes { + if (data?.simple_string) + return data.simple_string.content; + else if (data?.simple_error) + return data.simple_error.content; + else if (data?.integer) + return data.integer.val; + else if (data?.bulk_string) + return bulk_string_content(data.bulk_string); + else if (data?.array) { + local res = b"["; + local first = True; + for (ele in data.array.elements) { + if (!first) + res += b", "; + res += stringify(ele); + first = False; + } + res += b"]"; + return res; + } else if (data?.null) + return b"null"; + else if (data?.boolean) + return data.boolean.val ? b"T" : b"F"; + else if (data?.double) + return data.double.val; + else if (data?.big_num) + return data.big_num.val; + else if (data?.bulk_error) + return bulk_string_content(data.bulk_error); + else if (data?.verbatim_string) + return bulk_string_content(data.verbatim_string); + else if (data?.map_) { + return stringify_map(data.map_); + } else if (data?.set_) { + local res = b"("; + local first = True; + for (ele in data.set_.elements) { + if (!first) + res += b", "; + res += stringify(ele); + first = False; + } + res += b")"; + return res; + } else if (data?.push) { + local res = b"["; + local first = True; + for (ele in data.push.elements) { + if (!first) + res += b", "; + res += stringify(ele); + first = False; + } + res += b"]"; + return res; + } + + throw "unknown RESP type"; +} + +# Gets the server reply in a simpler form +public function make_server_reply(data: RESP::ServerData): ReplyData { + local min_protocol_version: uint8 = 2; + switch (data.data.ty) { + case RESP::DataType::NULL, + RESP::DataType::BOOLEAN, + RESP::DataType::DOUBLE, + RESP::DataType::BIG_NUM, + RESP::DataType::BULK_ERROR, + RESP::DataType::VERBATIM_STRING, + RESP::DataType::MAP, + RESP::DataType::SET, + RESP::DataType::PUSH: min_protocol_version = 3; + default: min_protocol_version = 2; + } + + local attributes: optional = Null; + if (data.data?.attributes) { + min_protocol_version = 3; + attributes = stringify_map(data.data.attributes); + } + + return [$attributes = attributes, $value = stringify(data.data), $min_protocol_version = min_protocol_version]; +} diff --git a/src/analyzer/protocol/redis/resp.evt b/src/analyzer/protocol/redis/resp.evt index f67f8fbc1c..c197ecd4f4 100644 --- a/src/analyzer/protocol/redis/resp.evt +++ b/src/analyzer/protocol/redis/resp.evt @@ -7,14 +7,19 @@ protocol analyzer Redis over TCP: import RESP; import Redis; -export Redis::KnownCommand; +export Redis::RedisCommand; on RESP::ClientData if ( Redis::is_set(self) ) -> event Redis::set_command($conn, Redis::make_set(self.command)); on RESP::ClientData if ( Redis::is_get(self) ) -> event Redis::get_command($conn, Redis::make_get(self.command).key); on RESP::ClientData if ( Redis::is_auth(self) ) -> event Redis::auth_command($conn, Redis::make_auth(self.command)); +on RESP::ClientData if ( Redis::is_hello(self) ) -> event Redis::hello_command($conn, Redis::make_hello(self.command)); # All client data is a command on RESP::ClientData -> event Redis::command($conn, self.command); -on RESP::ServerData if ( ! Redis::is_err(self) ) -> event Redis::reply($conn, Redis::make_server_reply(self)); -on RESP::ServerData if ( Redis::is_err(self) ) -> event Redis::error($conn, Redis::make_server_reply(self)); +on RESP::ServerData if ( Redis::classify(self) == Redis::ReplyType::Reply ) -> + event Redis::reply($conn, Redis::make_server_reply(self)); +on RESP::ServerData if ( Redis::classify(self) == Redis::ReplyType::Error ) -> + event Redis::error($conn, Redis::make_server_reply(self)); +on RESP::ServerData if ( Redis::classify(self) == Redis::ReplyType::Push ) -> + event Redis::server_push($conn, Redis::make_server_reply(self)); diff --git a/src/analyzer/protocol/redis/resp.spicy b/src/analyzer/protocol/redis/resp.spicy index 3216fd6d9b..76899dca36 100644 --- a/src/analyzer/protocol/redis/resp.spicy +++ b/src/analyzer/protocol/redis/resp.spicy @@ -79,11 +79,25 @@ public type ServerData = unit { %synchronize-after = b"\x0d\x0a"; var depth: uint8& = new uint8; data: Data(self.depth); + + var type_: Redis::ReplyType; + on %done { + self.type_ = Redis::classify(self); + } }; type Data = unit(depth: uint8&) { %synchronize-after = b"\x0d\x0a"; ty: uint8 &convert=DataType($$); + + # Attributes are special, they precede the actual data + if (self.ty == DataType::ATTRIBUTE) { + attributes: Map(depth); + : uint8 &convert=DataType($$) { + self.ty = $$; + } + }; + switch (self.ty) { DataType::SIMPLE_STRING -> simple_string: SimpleString(False); DataType::SIMPLE_ERROR -> simple_error: SimpleString(True); @@ -102,7 +116,7 @@ type Data = unit(depth: uint8&) { DataType::MAP -> map_: Map(depth); DataType::SET -> set_: Set(depth); # "Push events are encoded similarly to arrays, differing only in their - # first byte" - TODO: can probably make it more obvious, though + # first byte" DataType::PUSH -> push: Array(depth); }; @@ -130,6 +144,7 @@ type DataType = enum { BULK_ERROR = '!', VERBATIM_STRING = '=', MAP = '%', + ATTRIBUTE = '|', SET = '~', PUSH = '>', }; @@ -144,7 +159,7 @@ type SimpleString = unit(is_error: bool) { }; type Integer = unit { - int: RedisBytes &convert=$$.to_int(10); + val: RedisBytes; }; type BulkString = unit(is_error: bool) { @@ -173,32 +188,20 @@ type Boolean = unit { }; type Double = unit { - val: RedisBytes &convert=$$.to_real(); + val: RedisBytes; }; type BigNum = unit { - # Big num can be very big so leave it in bytes. val: RedisBytes; }; type Map = unit(depth: uint8&) { num_elements: RedisBytes &convert=$$.to_uint(10); - # TODO: How can I make this into a map? Alternatively, how can I do this better? raw_data: Data(depth)[self.num_elements * 2]; - - # TODO: This is broken. See https://github.com/zeek/spicy/issues/2061 - # var key_val_pairs: vector>; - # on raw_data { - # while (local i = 0; i < self.num_elements) { - # self.key_val_pairs.push_back(($$[i], $$[i + 1])); - # i += 2; - # } - # } }; type Set = unit(depth: uint8&) { num_elements: RedisBytes &convert=$$.to_uint(10) &requires=self.num_elements <= MAX_SIZE; - # TODO: This should be a set but doesn't go in the backed C++ set elements: Data(depth)[self.num_elements]; }; diff --git a/testing/btest/Baseline/coverage.record-fields/out.default b/testing/btest/Baseline/coverage.record-fields/out.default index 1be477339c..429564ff96 100644 --- a/testing/btest/Baseline/coverage.record-fields/out.default +++ b/testing/btest/Baseline/coverage.record-fields/out.default @@ -589,7 +589,7 @@ connection { * cmd: record Redis::Command, log=T, optional=F Redis::Command { * key: string, log=T, optional=T - * known: enum Redis::KnownCommand, log=F, optional=T + * known: enum Redis::RedisCommand, log=F, optional=T * name: string, log=T, optional=F * raw: vector of string, log=F, optional=F * value: string, log=T, optional=T @@ -598,7 +598,9 @@ connection { conn_id { ... } * reply: record Redis::ReplyData, log=T, optional=T Redis::ReplyData { - * value: string, log=T, optional=T + * attributes: string, log=F, optional=T + * min_protocol_version: count, log=F, optional=F + * value: string, log=T, optional=F } * success: bool, log=T, optional=T * ts: time, log=T, optional=F @@ -615,6 +617,9 @@ connection { } * pending: table[count] of record Redis::Info, log=F, optional=F Redis::Info { ... } + * resp_version: enum Redis::RESPVersion, log=F, optional=T + * skip_commands: set[count], log=F, optional=F + * subscribed_mode: bool, log=F, optional=T * violation: bool, log=F, optional=T } * removal_hooks: set[func], log=F, optional=T diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.attributes/output b/testing/btest/Baseline/scripts.base.protocols.redis.attributes/output new file mode 100644 index 0000000000..deb4f9e7ab --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.attributes/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Got data:, [1, 2, 3] +Got data:, [2039123, 9543892], with attributes:, {key-popularity: {a: 0.1923, b: 0.0012}} diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.attributes/redis.log b/testing/btest/Baseline/scripts.base.protocols.redis.attributes/redis.log new file mode 100644 index 0000000000..ef077bcd8f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.attributes/redis.log @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path redis +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.name cmd.key cmd.value success reply.value +#types time string addr port addr port string string string bool string +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 62283 127.0.0.1 6379 FAKE - - T [1, 2, 3] +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 127.0.0.1 62286 127.0.0.1 6379 FAKE2 - - T [2039123, 9543892] +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.pubsub-resp3/output b/testing/btest/Baseline/scripts.base.protocols.redis.pubsub-resp3/output new file mode 100644 index 0000000000..230044f840 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.pubsub-resp3/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Got published data!, [subscribe, Foo, 1] +Got published data!, [psubscribe, F*, 2] +Got published data!, [message, Foo, Hi:)] +Got published data!, [pmessage, F*, Foo, Hi:)] +Got published data!, [pmessage, F*, Foobar, Hello!] diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/output b/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/output index 49d861c74c..d2a284b02f 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/output +++ b/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/output @@ -1 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Got published data!, [subscribe, Foo, 1] +Got published data!, [psubscribe, F*, 2] +Got published data!, [message, Foo, Hi there :)] +Got published data!, [pmessage, F*, Foo, Hi there :)] +Got published data!, [pmessage, F*, FeeFooFiiFum, Hello! :)] diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/redis.log b/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/redis.log index 2f23f980ef..e604657317 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/redis.log +++ b/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/redis.log @@ -7,7 +7,11 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.name cmd.key cmd.value success reply.value #types time string addr port addr port string string string bool string -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 56162 127.0.0.1 6379 SUBSCRIBE - - T - -XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 127.0.0.1 56163 127.0.0.1 6379 PUBLISH - - T - -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 56162 127.0.0.1 6379 - - - T - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 127.0.0.1 60833 127.0.0.1 6379 PUBLISH - - T 2 +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 60837 127.0.0.1 6379 PUBLISH - - T 1 +XXXXXXXXXX.XXXXXX CtPZjS20MLrsMUOJi2 127.0.0.1 60838 127.0.0.1 6379 SET sanity_check you_are_sane T OK +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 SUBSCRIBE - - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 PSUBSCRIBE - - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 RESET - - T RESET +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 GET sanity_check - T you_are_sane #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.stream/redis.log b/testing/btest/Baseline/scripts.base.protocols.redis.stream/redis.log index d7ae07655a..47195e73af 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.stream/redis.log +++ b/testing/btest/Baseline/scripts.base.protocols.redis.stream/redis.log @@ -10,5 +10,5 @@ XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622832637-0 XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622836953-0 XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622840530-0 -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XRANGE - - T - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XRANGE - - T [[1729622770972-0, [rider, Castilla, speed, 30.2, position, 1, location_id, 1]], [1729622778221-0, [rider, Norem, speed, 28.8, position, 3, location_id, 1]]] #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Traces/redis/attr.pcap b/testing/btest/Traces/redis/attr.pcap new file mode 100644 index 0000000000..a94a53f72f Binary files /dev/null and b/testing/btest/Traces/redis/attr.pcap differ diff --git a/testing/btest/Traces/redis/pubsub-resp3.pcap b/testing/btest/Traces/redis/pubsub-resp3.pcap new file mode 100644 index 0000000000..de591be1ce Binary files /dev/null and b/testing/btest/Traces/redis/pubsub-resp3.pcap differ diff --git a/testing/btest/Traces/redis/pubsub.pcap b/testing/btest/Traces/redis/pubsub.pcap index 458070f948..d02630a7a4 100644 Binary files a/testing/btest/Traces/redis/pubsub.pcap and b/testing/btest/Traces/redis/pubsub.pcap differ diff --git a/testing/btest/scripts/base/protocols/redis/attributes.zeek b/testing/btest/scripts/base/protocols/redis/attributes.zeek new file mode 100644 index 0000000000..961cae443e --- /dev/null +++ b/testing/btest/scripts/base/protocols/redis/attributes.zeek @@ -0,0 +1,19 @@ +# @TEST-DOC: Test Redis protocol handling with replies with attributes +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: zeek -b -r $TRACES/redis/attr.pcap %INPUT >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff redis.log + +# IMPORTANT: The test data was made synthetically, since real commands that +# return attributes may be version-specific. Real traffic would be better. + +@load base/protocols/redis + +event Redis::reply(c: connection, data: Redis::ReplyData) + { + if ( ! data?$attributes ) + print "Got data:", data$value; + else + print "Got data:", data$value, "with attributes:", data$attributes; + } diff --git a/testing/btest/scripts/base/protocols/redis/pubsub-resp3.zeek b/testing/btest/scripts/base/protocols/redis/pubsub-resp3.zeek new file mode 100644 index 0000000000..6d061b3d79 --- /dev/null +++ b/testing/btest/scripts/base/protocols/redis/pubsub-resp3.zeek @@ -0,0 +1,16 @@ +# @TEST-DOC: Test Zeek parsing pubsub commands in RESP3 +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: zeek -b -r $TRACES/redis/pubsub-resp3.pcap %INPUT >output +# @TEST-EXEC: btest-diff output + +# Test pub/sub from Redis. This has two subscribers, one using a pattern. Then, the +# messages that were published get printed to output. + +@load base/protocols/redis + +event Redis::server_push(c: connection, data: Redis::ReplyData) + { + # The first 2 are SUBSCRIBE replies, the other 3 are message and pmessage + print "Got published data!", data$value; + } diff --git a/testing/btest/scripts/base/protocols/redis/pubsub.zeek b/testing/btest/scripts/base/protocols/redis/pubsub.zeek index 15f6d9bb59..76ab16e806 100644 --- a/testing/btest/scripts/base/protocols/redis/pubsub.zeek +++ b/testing/btest/scripts/base/protocols/redis/pubsub.zeek @@ -5,8 +5,12 @@ # @TEST-EXEC: btest-diff output # @TEST-EXEC: btest-diff redis.log -# Testing the example of pub sub in REDIS docs: -# https://redis.io/docs/latest/develop/interact/pubsub/ -# These are just commands between two different clients, one PUBLISH and one SUBSCRIBE +# Test pub/sub from Redis. This has two subscribers, one using a pattern. Then, the +# messages that were published get printed to output. @load base/protocols/redis + +event Redis::server_push(c: connection, data: Redis::ReplyData) + { + print "Got published data!", data$value; + }