Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/topic/etyp/redis-resp3'

Browse source 
* origin/topic/etyp/redis-resp3:
  Touchup TODOs in the Redis analyzer
  Handle more Redis RESP3 protocol pieces
  Stringify all Redis-RESP serialized data
  Handle Redis protocol `message` separately
  Add Redis analyzer array stringification
This commit is contained in:
Evan Typanski 2025-07-01 14:19:26 -04:00
commit 310a82e7fd
20 changed files with 505 additions and 158 deletions
Download patch file Download diff file Expand all files Collapse all files

21
CHANGES
View file

@ -1,3 +1,24 @@
8.0.0-dev.577 | 2025-07-01 14:19:26 -0400
* Touchup TODOs in the Redis analyzer (Evan Typanski, Corelight)
Also renames `KnownCommand` to `RedisCommand` to avoid conflicts.
* Handle more Redis RESP3 protocol pieces (Evan Typanski, Corelight)
This passes the "minimum protocol version" along in the reply and adds
support for attributes, which were added relatively recently.
* Stringify all Redis-RESP serialized data (Evan Typanski, Corelight)
* GH-4504: Handle Redis protocol `message` separately (Evan Typanski, Corelight)
Closes #4504
Messages are not typical responses, so they need special handling. This
is different between RESP2 and 3, so this is the first instance where
the script layer needs to tell the difference.
8.0.0-dev.571 | 2025-07-01 11:03:23 +0200 8.0.0-dev.571 | 2025-07-01 11:03:23 +0200
* Bump pre-commit hooks (Benjamin Bannier, Corelight) * Bump pre-commit hooks (Benjamin Bannier, Corelight)

2
VERSION
View file

@ -1 +1 @@
8.0.0-dev.571 8.0.0-dev.577

93
scripts/base/protocols/redis/main.zeek
View file

@ -41,6 +41,11 @@ export {
end: count &optional; end: count &optional;
}; };
type RESPVersion: enum {
RESP2,
RESP3
};
type State: record { type State: record {
## Pending commands. ## Pending commands.
pending: table[count] of Info; pending: table[count] of Info;
@ -52,14 +57,34 @@ export {
## Each range is one or two elements, one meaning it's unbounded, two meaning ## Each range is one or two elements, one meaning it's unbounded, two meaning
## it begins at one and ends at the second. ## it begins at one and ends at the second.
no_reply_ranges: vector of NoReplyRange; no_reply_ranges: vector of NoReplyRange;
## The command indexes (from current_command and current_reply) that will
## not get responses no matter what.
skip_commands: set[count];
## We store if this analyzer had a violation to avoid logging if so. ## We store if this analyzer had a violation to avoid logging if so.
## This should not be super necessary, but worth a shot. ## This should not be super necessary, but worth a shot.
violation: bool &default=F; violation: bool &default=F;
## If we are in "subscribed" mode
subscribed_mode: bool &default=F;
## The RESP version
resp_version: RESPVersion &default=RESP2;
}; };
# Redis specifically mentions 10k commands as a good pipelining threshold, so # Redis specifically mentions 10k commands as a good pipelining threshold, so
# we'll piggyback on that. # we'll piggyback on that.
option max_pending_commands = 10000; option max_pending_commands = 10000;
# These commands enter subscribed mode
global enter_subscribed_mode = [RedisCommand_PSUBSCRIBE,
RedisCommand_SSUBSCRIBE, RedisCommand_SUBSCRIBE];
# These commands exit subscribed mode
global exit_subscribed_mode = [RedisCommand_RESET, RedisCommand_QUIT];
# These commands don't expect a response (ever) - their replies are out of band.
global no_response_commands = [RedisCommand_PSUBSCRIBE,
RedisCommand_PUNSUBSCRIBE, RedisCommand_SSUBSCRIBE,
RedisCommand_SUBSCRIBE, RedisCommand_SUNSUBSCRIBE,
RedisCommand_UNSUBSCRIBE];
} }
redef record connection += { redef record connection += {
@ -122,7 +147,16 @@ function is_last_interval_closed(c: connection): bool
c$redis_state$no_reply_ranges[-1]?$end; c$redis_state$no_reply_ranges[-1]?$end;
} }
event Redis::command(c: connection, cmd: Command) event hello_command(c: connection, hello: HelloCommand)
{
if ( ! c?$redis_state )
make_new_state(c);
if ( hello?$requested_resp_version && hello$requested_resp_version == "3" )
c$redis_state$resp_version = RESP3;
}
event command(c: connection, cmd: Command)
{ {
if ( ! c?$redis_state ) if ( ! c?$redis_state )
make_new_state(c); make_new_state(c);
@ -139,10 +173,30 @@ event Redis::command(c: connection, cmd: Command)
} }
++c$redis_state$current_command; ++c$redis_state$current_command;
if ( cmd?$known )
{
if ( c$redis_state$resp_version == RESP2 )
{
local should_enter = cmd$known in enter_subscribed_mode;
local should_exit = cmd$known in exit_subscribed_mode;
c$redis_state$subscribed_mode = should_enter && ! should_exit;
# It's weird if it's in both - in the future users may be able to add that
if ( should_enter && should_exit )
Reporter::conn_weird("Redis_command_enter_exit_subscribed_mode", c, cat(
cmd$known));
}
if ( cmd$known in no_response_commands || c$redis_state$subscribed_mode )
{
add c$redis_state$skip_commands[c$redis_state$current_command];
}
}
# CLIENT commands can skip a number of replies and may be used with # CLIENT commands can skip a number of replies and may be used with
# pipelining. We need special logic in order to track the command/reply # pipelining. We need special logic in order to track the command/reply
# pairs. # pairs.
if ( cmd?$known && cmd$known == KnownCommand_CLIENT ) if ( cmd?$known && cmd$known == RedisCommand_CLIENT )
{ {
# All 3 CLIENT commands we care about have 3 elements # All 3 CLIENT commands we care about have 3 elements
if ( |cmd$raw| == 3 ) if ( |cmd$raw| == 3 )
@ -177,6 +231,7 @@ event Redis::command(c: connection, cmd: Command)
} }
} }
} }
set_state(c, T); set_state(c, T);
c$redis$cmd = cmd; c$redis$cmd = cmd;
@ -187,17 +242,24 @@ event Redis::command(c: connection, cmd: Command)
function reply_num(c: connection): count function reply_num(c: connection): count
{ {
local resp_num = c$redis_state$current_reply + 1; local resp_num = c$redis_state$current_reply + 1;
local result = resp_num;
for ( i in c$redis_state$no_reply_ranges ) for ( i in c$redis_state$no_reply_ranges )
{ {
local range = c$redis_state$no_reply_ranges[i]; local range = c$redis_state$no_reply_ranges[i];
if ( ! range?$end && resp_num > range$begin ) if ( ! range?$end && resp_num > range$begin )
{ } # TODO: This is necessary if not using pipelining { } # TODO: This is necessary if not using pipelining
if ( range?$end && resp_num >= range$begin && resp_num < range$end ) if ( range?$end && resp_num >= range$begin && resp_num < range$end )
return range$end; result = range$end;
} }
# Default: no disable/enable shenanigans # Account for commands that don't expect a response
return resp_num; while ( result in c$redis_state$skip_commands )
{
delete c$redis_state$skip_commands[result];
result += 1;
}
return result;
} }
# Logs up to and including the last seen command from the last reply # Logs up to and including the last seen command from the last reply
@ -229,11 +291,24 @@ function log_from(c: connection, previous_reply_num: count)
} }
} }
event Redis::reply(c: connection, data: ReplyData) event reply(c: connection, data: ReplyData)
{ {
if ( ! c?$redis_state ) if ( ! c?$redis_state )
make_new_state(c); make_new_state(c);
# If the server is talking in RESP3, mark accordingly, even if we didn't see HELLO
if ( data$min_protocol_version == 3 )
{
c$redis_state$resp_version = RESP3;
c$redis_state$subscribed_mode = F;
}
if ( c$redis_state$subscribed_mode )
{
event server_push(c, data);
return;
}
local previous_reply_num = c$redis_state$current_reply; local previous_reply_num = c$redis_state$current_reply;
c$redis_state$current_reply = reply_num(c); c$redis_state$current_reply = reply_num(c);
set_state(c, F); set_state(c, F);
@ -241,9 +316,13 @@ event Redis::reply(c: connection, data: ReplyData)
c$redis$reply = data; c$redis$reply = data;
c$redis$success = T; c$redis$success = T;
log_from(c, previous_reply_num); log_from(c, previous_reply_num);
# Tidy up the skip_commands when it's up to date
if ( c$redis_state$current_command == c$redis_state$current_reply )
clear_table(c$redis_state$skip_commands);
} }
event Redis::error(c: connection, data: ReplyData) event error(c: connection, data: ReplyData)
{ {
if ( ! c?$redis_state ) if ( ! c?$redis_state )
make_new_state(c); make_new_state(c);

36
scripts/base/protocols/redis/spicy-events.zeek
View file

@ -37,6 +37,12 @@ export {
password: string; password: string;
}; };
## The Redis HELLO command (handshake).
type HelloCommand: record {
## The sent requested RESP version, such as "2" or "3"
requested_resp_version: string &optional;
};
## A generic Redis command from the client. ## A generic Redis command from the client.
type Command: record { type Command: record {
## The raw command, exactly as parsed ## The raw command, exactly as parsed
@ -49,12 +55,17 @@ export {
## The value, if this command is known to have a value ## The value, if this command is known to have a value
value: string &log &optional; value: string &log &optional;
## The command in an enum if it was known ## The command in an enum if it was known
known: KnownCommand &optional; known: RedisCommand &optional;
}; };
## A generic Redis reply from the client. ## A generic Redis reply from the client.
type ReplyData: record { type ReplyData: record {
value: string &log &optional; ## The RESP3 attributes applied to this, if any
attributes: string &optional;
## The string version of the reply data
value: string &log;
## The minimum RESP version that supports this reply type
min_protocol_version: count;
}; };
} }
@ -79,6 +90,13 @@ global get_command: event(c: connection, key: string);
## command: The AUTH command sent to the server and its data. ## command: The AUTH command sent to the server and its data.
global auth_command: event(c: connection, command: AuthCommand); global auth_command: event(c: connection, command: AuthCommand);
## Generated for Redis HELLO commands sent to the Redis server.
##
## c: The connection.
##
## command: The HELLO command sent to the server and its data.
global hello_command: event(c: connection, command: HelloCommand);
## Generated for every command sent by the client to the Redis server. ## Generated for every command sent by the client to the Redis server.
## ##
## c: The connection. ## c: The connection.
@ -87,11 +105,15 @@ global auth_command: event(c: connection, command: AuthCommand);
global command: event(c: connection, cmd: Command); global command: event(c: connection, cmd: Command);
## Generated for every successful response sent by the Redis server to the ## Generated for every successful response sent by the Redis server to the
## client. ## client. For RESP2, this includes "push" messages, which are out of band.
## These will also raise a server_push event. RESP3 push messages will only
## raise a server_push event.
## ##
## c: The connection. ## c: The connection.
## ##
## data: The server data sent to the client. ## data: The server data sent to the client.
##
## .. zeek:see:: Redis::server_push
global reply: event(c: connection, data: ReplyData); global reply: event(c: connection, data: ReplyData);
## Generated for every error response sent by the Redis server to the ## Generated for every error response sent by the Redis server to the
@ -101,3 +123,11 @@ global reply: event(c: connection, data: ReplyData);
## ##
## data: The server data sent to the client. ## data: The server data sent to the client.
global error: event(c: connection, data: ReplyData); global error: event(c: connection, data: ReplyData);
## Generated for out-of-band data, outside of the request-response
## model.
##
## c: The connection.
##
## data: The server data sent to the client.
global server_push: event(c: connection, data: ReplyData);

375
src/analyzer/protocol/redis/redis.spicy
View file

@ -6,7 +6,7 @@ module Redis;
import RESP; import RESP;
public type KnownCommand = enum { public type RedisCommand = enum {
APPEND, APPEND,
AUTH, AUTH,
BITCOUNT, BITCOUNT,
@ -34,6 +34,7 @@ public type KnownCommand = enum {
GETRANGE, GETRANGE,
GETSET, GETSET,
HDEL, HDEL,
HELLO,
HGET, HGET,
HSET, HSET,
INCR, INCR,
@ -43,11 +44,19 @@ public type KnownCommand = enum {
MOVE, MOVE,
MSET, MSET,
PERSIST, PERSIST,
PSUBSCRIBE,
PUNSUBSCRIBE,
QUIT,
RENAME, RENAME,
RESET,
SET, SET,
STRLEN, STRLEN,
SUBSCRIBE,
SSUBSCRIBE,
SUNSUBSCRIBE,
TTL, TTL,
TYPE, TYPE,
UNSUBSCRIBE,
}; };
type Command = struct { type Command = struct {
@ -55,7 +64,7 @@ type Command = struct {
name: bytes; name: bytes;
key: optional<bytes>; key: optional<bytes>;
value: optional<bytes>; value: optional<bytes>;
known: optional<KnownCommand>; known: optional<RedisCommand>;
}; };
# This just assumes all elements in the array is a bulk string and puts them in a vector # This just assumes all elements in the array is a bulk string and puts them in a vector
@ -149,44 +158,44 @@ function parse_command(raw: vector<bytes>): Command {
if (|raw| >= 2) { if (|raw| >= 2) {
switch (*cmd) { switch (*cmd) {
case KnownCommand::KEYS: case RedisCommand::KEYS:
parsed.key = raw[1]; parsed.key = raw[1];
case KnownCommand::APPEND, case RedisCommand::APPEND,
KnownCommand::BITCOUNT, RedisCommand::BITCOUNT,
KnownCommand::BITFIELD, RedisCommand::BITFIELD,
KnownCommand::BITFIELD_RO, RedisCommand::BITFIELD_RO,
KnownCommand::BITPOS, RedisCommand::BITPOS,
KnownCommand::BLPOP, RedisCommand::BLPOP,
KnownCommand::BRPOP, RedisCommand::BRPOP,
KnownCommand::COPY, RedisCommand::COPY,
KnownCommand::DECR, RedisCommand::DECR,
KnownCommand::DECRBY, RedisCommand::DECRBY,
KnownCommand::DEL, RedisCommand::DEL,
KnownCommand::DUMP, RedisCommand::DUMP,
KnownCommand::EXISTS, RedisCommand::EXISTS,
KnownCommand::EXPIRE, RedisCommand::EXPIRE,
KnownCommand::EXPIREAT, RedisCommand::EXPIREAT,
KnownCommand::EXPIRETIME, RedisCommand::EXPIRETIME,
KnownCommand::GET, RedisCommand::GET,
KnownCommand::GETBIT, RedisCommand::GETBIT,
KnownCommand::GETDEL, RedisCommand::GETDEL,
KnownCommand::GETEX, RedisCommand::GETEX,
KnownCommand::GETRANGE, RedisCommand::GETRANGE,
KnownCommand::GETSET, RedisCommand::GETSET,
KnownCommand::HDEL, RedisCommand::HDEL,
KnownCommand::HGET, RedisCommand::HGET,
KnownCommand::HSET, RedisCommand::HSET,
KnownCommand::INCR, RedisCommand::INCR,
KnownCommand::INCRBY, RedisCommand::INCRBY,
KnownCommand::MGET, RedisCommand::MGET,
KnownCommand::MOVE, RedisCommand::MOVE,
KnownCommand::MSET, RedisCommand::MSET,
KnownCommand::PERSIST, RedisCommand::PERSIST,
KnownCommand::RENAME, RedisCommand::RENAME,
KnownCommand::SET, RedisCommand::SET,
KnownCommand::STRLEN, RedisCommand::STRLEN,
KnownCommand::TTL, RedisCommand::TTL,
KnownCommand::TYPE: RedisCommand::TYPE:
parsed.key = raw[1]; parsed.key = raw[1];
default: (); default: ();
} }
@ -194,22 +203,22 @@ function parse_command(raw: vector<bytes>): Command {
if (|raw| >= 3) { if (|raw| >= 3) {
switch (*cmd) { switch (*cmd) {
case KnownCommand::SET, case RedisCommand::SET,
KnownCommand::APPEND, RedisCommand::APPEND,
KnownCommand::DECRBY, RedisCommand::DECRBY,
KnownCommand::EXPIRE, RedisCommand::EXPIRE,
KnownCommand::EXPIREAT, RedisCommand::EXPIREAT,
KnownCommand::GETBIT, RedisCommand::GETBIT,
KnownCommand::GETSET, RedisCommand::GETSET,
KnownCommand::HDEL, RedisCommand::HDEL,
KnownCommand::HGET, RedisCommand::HGET,
KnownCommand::INCRBY, RedisCommand::INCRBY,
KnownCommand::MOVE, RedisCommand::MOVE,
KnownCommand::MSET, RedisCommand::MSET,
KnownCommand::RENAME: RedisCommand::RENAME:
parsed.value = raw[2]; parsed.value = raw[2];
# Op first, destination second, then a list of keys. Just log dest # Op first, destination second, then a list of keys. Just log dest
case KnownCommand::BITOP: parsed.key = raw[2]; case RedisCommand::BITOP: parsed.key = raw[2];
default: (); default: ();
} }
} }
@ -217,7 +226,7 @@ function parse_command(raw: vector<bytes>): Command {
if (|raw| >= 4) { if (|raw| >= 4) {
switch (*cmd) { switch (*cmd) {
# timeout, numkeys, then key # timeout, numkeys, then key
case KnownCommand::BLMPOP: parsed.key = raw[3]; case RedisCommand::BLMPOP: parsed.key = raw[3];
default: (); default: ();
} }
} }
@ -225,51 +234,60 @@ function parse_command(raw: vector<bytes>): Command {
return parsed; return parsed;
} }
function command_from(cmd_bytes: bytes): optional<KnownCommand> { function command_from(cmd_bytes: bytes): optional<RedisCommand> {
local cmd: optional<KnownCommand> = Null; local cmd: optional<RedisCommand> = Null;
switch (cmd_bytes.lower()) { switch (cmd_bytes.lower()) {
case b"set": cmd = KnownCommand::SET; case b"append": cmd = RedisCommand::APPEND;
case b"append": cmd = KnownCommand::APPEND; case b"auth": cmd = RedisCommand::AUTH;
case b"auth": cmd = KnownCommand::AUTH; case b"bitcount": cmd = RedisCommand::BITCOUNT;
case b"bitcount": cmd = KnownCommand::BITCOUNT; case b"bitfield": cmd = RedisCommand::BITFIELD;
case b"bitfield": cmd = KnownCommand::BITFIELD; case b"bitfield_ro": cmd = RedisCommand::BITFIELD_RO;
case b"bitfield_ro": cmd = KnownCommand::BITFIELD_RO; case b"bitop": cmd = RedisCommand::BITOP;
case b"bitop": cmd = KnownCommand::BITOP; case b"bitpos": cmd = RedisCommand::BITPOS;
case b"bitpos": cmd = KnownCommand::BITPOS; case b"blmpop": cmd = RedisCommand::BLMPOP;
case b"blmpop": cmd = KnownCommand::BLMPOP; case b"blpop": cmd = RedisCommand::BLPOP;
case b"blpop": cmd = KnownCommand::BLPOP; case b"brpop": cmd = RedisCommand::BRPOP;
case b"brpop": cmd = KnownCommand::BRPOP; case b"client": cmd = RedisCommand::CLIENT;
case b"client": cmd = KnownCommand::CLIENT; case b"copy": cmd = RedisCommand::COPY;
case b"copy": cmd = KnownCommand::COPY; case b"decr": cmd = RedisCommand::DECR;
case b"decr": cmd = KnownCommand::DECR; case b"decrby": cmd = RedisCommand::DECRBY;
case b"decrby": cmd = KnownCommand::DECRBY; case b"del": cmd = RedisCommand::DEL;
case b"del": cmd = KnownCommand::DEL; case b"dump": cmd = RedisCommand::DUMP;
case b"dump": cmd = KnownCommand::DUMP; case b"exists": cmd = RedisCommand::EXISTS;
case b"exists": cmd = KnownCommand::EXISTS; case b"expire": cmd = RedisCommand::EXPIRE;
case b"expire": cmd = KnownCommand::EXPIRE; case b"expireat": cmd = RedisCommand::EXPIREAT;
case b"expireat": cmd = KnownCommand::EXPIREAT; case b"expiretime": cmd = RedisCommand::EXPIRETIME;
case b"expiretime": cmd = KnownCommand::EXPIRETIME; case b"expiretime": cmd = RedisCommand::EXPIRETIME;
case b"expiretime": cmd = KnownCommand::EXPIRETIME; case b"get": cmd = RedisCommand::GET;
case b"get": cmd = KnownCommand::GET; case b"getbit": cmd = RedisCommand::GETBIT;
case b"getbit": cmd = KnownCommand::GETBIT; case b"getdel": cmd = RedisCommand::GETDEL;
case b"getdel": cmd = KnownCommand::GETDEL; case b"getex": cmd = RedisCommand::GETEX;
case b"getex": cmd = KnownCommand::GETEX; case b"getrange": cmd = RedisCommand::GETRANGE;
case b"getrange": cmd = KnownCommand::GETRANGE; case b"getset": cmd = RedisCommand::GETSET;
case b"getset": cmd = KnownCommand::GETSET; case b"hdel": cmd = RedisCommand::HDEL;
case b"hdel": cmd = KnownCommand::HDEL; case b"hello": cmd = RedisCommand::HELLO;
case b"hget": cmd = KnownCommand::HGET; case b"hget": cmd = RedisCommand::HGET;
case b"hset": cmd = KnownCommand::HSET; case b"hset": cmd = RedisCommand::HSET;
case b"incr": cmd = KnownCommand::INCR; case b"incr": cmd = RedisCommand::INCR;
case b"incrby": cmd = KnownCommand::INCRBY; case b"incrby": cmd = RedisCommand::INCRBY;
case b"keys": cmd = KnownCommand::KEYS; case b"keys": cmd = RedisCommand::KEYS;
case b"mget": cmd = KnownCommand::MGET; case b"mget": cmd = RedisCommand::MGET;
case b"move": cmd = KnownCommand::MOVE; case b"move": cmd = RedisCommand::MOVE;
case b"mset": cmd = KnownCommand::MSET; case b"mset": cmd = RedisCommand::MSET;
case b"persist": cmd = KnownCommand::PERSIST; case b"persist": cmd = RedisCommand::PERSIST;
case b"rename": cmd = KnownCommand::RENAME; case b"psubscribe": cmd = RedisCommand::PSUBSCRIBE;
case b"strlen": cmd = KnownCommand::STRLEN; case b"punsubscribe": cmd = RedisCommand::PUNSUBSCRIBE;
case b"ttl": cmd = KnownCommand::TTL; case b"quit": cmd = RedisCommand::QUIT;
case b"type": cmd = KnownCommand::TYPE; case b"rename": cmd = RedisCommand::RENAME;
case b"reset": cmd = RedisCommand::RESET;
case b"set": cmd = RedisCommand::SET;
case b"strlen": cmd = RedisCommand::STRLEN;
case b"ssubscribe": cmd = RedisCommand::SSUBSCRIBE;
case b"subscribe": cmd = RedisCommand::SUBSCRIBE;
case b"sunsubscribe": cmd = RedisCommand::SUNSUBSCRIBE;
case b"ttl": cmd = RedisCommand::TTL;
case b"type": cmd = RedisCommand::TYPE;
case b"unsubscribe": cmd = RedisCommand::UNSUBSCRIBE;
default: cmd = Null; default: cmd = Null;
} }
@ -334,7 +352,7 @@ public function make_set(command: Command): Set {
} }
public function is_set(data: RESP::ClientData): bool { public function is_set(data: RESP::ClientData): bool {
return data.command.known && *(data.command.known) == KnownCommand::SET && data.command.key && data.command.value; return data.command.known && *(data.command.known) == RedisCommand::SET && data.command.key && data.command.value;
} }
type Get = struct { type Get = struct {
@ -347,7 +365,7 @@ public function make_get(command: Command): Get {
} }
public function is_get(data: RESP::ClientData): bool { public function is_get(data: RESP::ClientData): bool {
return data.command.known && *(data.command.known) == KnownCommand::GET && |data.command.raw| >= 2; return data.command.known && *(data.command.known) == RedisCommand::GET && |data.command.raw| >= 2;
} }
type Auth = struct { type Auth = struct {
@ -365,15 +383,45 @@ public function make_auth(command: Command): Auth {
} }
public function is_auth(data: RESP::ClientData): bool { public function is_auth(data: RESP::ClientData): bool {
return data.command.known && *(data.command.known) == KnownCommand::AUTH && |data.command.raw| >= 2; return data.command.known && *(data.command.known) == RedisCommand::AUTH && |data.command.raw| >= 2;
}
type Hello = struct {
requested_resp_version: optional<bytes>;
};
public function make_hello(command: Command): Hello {
local hi: Hello = [$requested_resp_version = Null];
if (|command.raw| > 1)
hi.requested_resp_version = command.raw[1];
return hi;
}
public function is_hello(data: RESP::ClientData): bool {
return data.command.known && *(data.command.known) == RedisCommand::HELLO;
} }
type ReplyData = struct { type ReplyData = struct {
value: optional<bytes>; attributes: optional<bytes>;
value: bytes;
min_protocol_version: uint8;
}; };
public function is_err(server_data: RESP::ServerData): bool { public type ReplyType = enum {
return server_data.data?.simple_error || server_data.data?.bulk_error; Reply, # A response to a command
Error, # An error response to a command
Push, # A server message that is not responding to a command
};
public function classify(server_data: RESP::ServerData): ReplyType {
if (server_data.data?.simple_error || server_data.data?.bulk_error)
return ReplyType::Error;
# We can tell with RESP3 this is push here, but RESP2 relies on scripts
if (server_data.data?.push)
return ReplyType::Push;
return ReplyType::Reply;
} }
function bulk_string_content(bulk: RESP::BulkString): bytes { function bulk_string_content(bulk: RESP::BulkString): bytes {
@ -383,21 +431,108 @@ function bulk_string_content(bulk: RESP::BulkString): bytes {
return b""; return b"";
} }
# Gets the server reply in a simpler form function stringify_map(data: RESP::Map&): bytes {
public function make_server_reply(data: RESP::ServerData): ReplyData { local res = b"{";
local res: ReplyData = [$value = Null]; local first = True;
if (data.data?.simple_error) local i = 0;
res.value = data.data.simple_error.content; # num_elements refers to the number of map entries, each with 2 entries
else if (data.data?.bulk_error) # in the raw data
res.value = bulk_string_content(data.data.bulk_error); while (i < data.num_elements) {
else if (data.data?.simple_string) if (!first)
res.value = data.data.simple_string.content; res += b", ";
else if (data.data?.bulk_string) res += stringify(data.raw_data[i * 2]);
res.value = bulk_string_content(data.data.bulk_string); res += b": ";
else if (data.data?.verbatim_string) res += stringify(data.raw_data[(i * 2) + 1]);
res.value = bulk_string_content(data.data.verbatim_string); i += 1;
else if (data.data?.boolean) first = False;
res.value = data.data.boolean.val ? b"T" : b"F"; }
res += b"}";
return res; return res;
} }
# Returns the bytes string value of this, or Null if it cannot.
function stringify(data: RESP::Data&): bytes {
if (data?.simple_string)
return data.simple_string.content;
else if (data?.simple_error)
return data.simple_error.content;
else if (data?.integer)
return data.integer.val;
else if (data?.bulk_string)
return bulk_string_content(data.bulk_string);
else if (data?.array) {
local res = b"[";
local first = True;
for (ele in data.array.elements) {
if (!first)
res += b", ";
res += stringify(ele);
first = False;
}
res += b"]";
return res;
} else if (data?.null)
return b"null";
else if (data?.boolean)
return data.boolean.val ? b"T" : b"F";
else if (data?.double)
return data.double.val;
else if (data?.big_num)
return data.big_num.val;
else if (data?.bulk_error)
return bulk_string_content(data.bulk_error);
else if (data?.verbatim_string)
return bulk_string_content(data.verbatim_string);
else if (data?.map_) {
return stringify_map(data.map_);
} else if (data?.set_) {
local res = b"(";
local first = True;
for (ele in data.set_.elements) {
if (!first)
res += b", ";
res += stringify(ele);
first = False;
}
res += b")";
return res;
} else if (data?.push) {
local res = b"[";
local first = True;
for (ele in data.push.elements) {
if (!first)
res += b", ";
res += stringify(ele);
first = False;
}
res += b"]";
return res;
}
throw "unknown RESP type";
}
# Gets the server reply in a simpler form
public function make_server_reply(data: RESP::ServerData): ReplyData {
local min_protocol_version: uint8 = 2;
switch (data.data.ty) {
case RESP::DataType::NULL,
RESP::DataType::BOOLEAN,
RESP::DataType::DOUBLE,
RESP::DataType::BIG_NUM,
RESP::DataType::BULK_ERROR,
RESP::DataType::VERBATIM_STRING,
RESP::DataType::MAP,
RESP::DataType::SET,
RESP::DataType::PUSH: min_protocol_version = 3;
default: min_protocol_version = 2;
}
local attributes: optional<bytes> = Null;
if (data.data?.attributes) {
min_protocol_version = 3;
attributes = stringify_map(data.data.attributes);
}
return [$attributes = attributes, $value = stringify(data.data), $min_protocol_version = min_protocol_version];
}

11
src/analyzer/protocol/redis/resp.evt
View file

@ -7,14 +7,19 @@ protocol analyzer Redis over TCP:
import RESP; import RESP;
import Redis; import Redis;
export Redis::KnownCommand; export Redis::RedisCommand;
on RESP::ClientData if ( Redis::is_set(self) ) -> event Redis::set_command($conn, Redis::make_set(self.command)); on RESP::ClientData if ( Redis::is_set(self) ) -> event Redis::set_command($conn, Redis::make_set(self.command));
on RESP::ClientData if ( Redis::is_get(self) ) -> event Redis::get_command($conn, Redis::make_get(self.command).key); on RESP::ClientData if ( Redis::is_get(self) ) -> event Redis::get_command($conn, Redis::make_get(self.command).key);
on RESP::ClientData if ( Redis::is_auth(self) ) -> event Redis::auth_command($conn, Redis::make_auth(self.command)); on RESP::ClientData if ( Redis::is_auth(self) ) -> event Redis::auth_command($conn, Redis::make_auth(self.command));
on RESP::ClientData if ( Redis::is_hello(self) ) -> event Redis::hello_command($conn, Redis::make_hello(self.command));
# All client data is a command # All client data is a command
on RESP::ClientData -> event Redis::command($conn, self.command); on RESP::ClientData -> event Redis::command($conn, self.command);
on RESP::ServerData if ( ! Redis::is_err(self) ) -> event Redis::reply($conn, Redis::make_server_reply(self)); on RESP::ServerData if ( Redis::classify(self) == Redis::ReplyType::Reply ) ->
on RESP::ServerData if ( Redis::is_err(self) ) -> event Redis::error($conn, Redis::make_server_reply(self)); event Redis::reply($conn, Redis::make_server_reply(self));
on RESP::ServerData if ( Redis::classify(self) == Redis::ReplyType::Error ) ->
event Redis::error($conn, Redis::make_server_reply(self));
on RESP::ServerData if ( Redis::classify(self) == Redis::ReplyType::Push ) ->
event Redis::server_push($conn, Redis::make_server_reply(self));

33
src/analyzer/protocol/redis/resp.spicy
View file

@ -79,11 +79,25 @@ public type ServerData = unit {
%synchronize-after = b"\x0d\x0a"; %synchronize-after = b"\x0d\x0a";
var depth: uint8& = new uint8; var depth: uint8& = new uint8;
data: Data(self.depth); data: Data(self.depth);
var type_: Redis::ReplyType;
on %done {
self.type_ = Redis::classify(self);
}
}; };
type Data = unit(depth: uint8&) { type Data = unit(depth: uint8&) {
%synchronize-after = b"\x0d\x0a"; %synchronize-after = b"\x0d\x0a";
ty: uint8 &convert=DataType($$); ty: uint8 &convert=DataType($$);
# Attributes are special, they precede the actual data
if (self.ty == DataType::ATTRIBUTE) {
attributes: Map(depth);
: uint8 &convert=DataType($$) {
self.ty = $$;
}
};
switch (self.ty) { switch (self.ty) {
DataType::SIMPLE_STRING -> simple_string: SimpleString(False); DataType::SIMPLE_STRING -> simple_string: SimpleString(False);
DataType::SIMPLE_ERROR -> simple_error: SimpleString(True); DataType::SIMPLE_ERROR -> simple_error: SimpleString(True);
@ -102,7 +116,7 @@ type Data = unit(depth: uint8&) {
DataType::MAP -> map_: Map(depth); DataType::MAP -> map_: Map(depth);
DataType::SET -> set_: Set(depth); DataType::SET -> set_: Set(depth);
# "Push events are encoded similarly to arrays, differing only in their # "Push events are encoded similarly to arrays, differing only in their
# first byte" - TODO: can probably make it more obvious, though # first byte"
DataType::PUSH -> push: Array(depth); DataType::PUSH -> push: Array(depth);
}; };
@ -130,6 +144,7 @@ type DataType = enum {
BULK_ERROR = '!', BULK_ERROR = '!',
VERBATIM_STRING = '=', VERBATIM_STRING = '=',
MAP = '%', MAP = '%',
ATTRIBUTE = '|',
SET = '~', SET = '~',
PUSH = '>', PUSH = '>',
}; };
@ -144,7 +159,7 @@ type SimpleString = unit(is_error: bool) {
}; };
type Integer = unit { type Integer = unit {
int: RedisBytes &convert=$$.to_int(10); val: RedisBytes;
}; };
type BulkString = unit(is_error: bool) { type BulkString = unit(is_error: bool) {
@ -173,32 +188,20 @@ type Boolean = unit {
}; };
type Double = unit { type Double = unit {
val: RedisBytes &convert=$$.to_real(); val: RedisBytes;
}; };
type BigNum = unit { type BigNum = unit {
# Big num can be very big so leave it in bytes.
val: RedisBytes; val: RedisBytes;
}; };
type Map = unit(depth: uint8&) { type Map = unit(depth: uint8&) {
num_elements: RedisBytes &convert=$$.to_uint(10); num_elements: RedisBytes &convert=$$.to_uint(10);
# TODO: How can I make this into a map? Alternatively, how can I do this better?
raw_data: Data(depth)[self.num_elements * 2]; raw_data: Data(depth)[self.num_elements * 2];
# TODO: This is broken. See https://github.com/zeek/spicy/issues/2061
# var key_val_pairs: vector<tuple<Data, Data>>;
# on raw_data {
# while (local i = 0; i < self.num_elements) {
# self.key_val_pairs.push_back(($$[i], $$[i + 1]));
# i += 2;
# }
# }
}; };
type Set = unit(depth: uint8&) { type Set = unit(depth: uint8&) {
num_elements: RedisBytes &convert=$$.to_uint(10) &requires=self.num_elements <= MAX_SIZE; num_elements: RedisBytes &convert=$$.to_uint(10) &requires=self.num_elements <= MAX_SIZE;
# TODO: This should be a set but doesn't go in the backed C++ set
elements: Data(depth)[self.num_elements]; elements: Data(depth)[self.num_elements];
}; };

9
testing/btest/Baseline/coverage.record-fields/out.default
View file

@ -589,7 +589,7 @@ connection {
* cmd: record Redis::Command, log=T, optional=F * cmd: record Redis::Command, log=T, optional=F
Redis::Command { Redis::Command {
* key: string, log=T, optional=T * key: string, log=T, optional=T
* known: enum Redis::KnownCommand, log=F, optional=T * known: enum Redis::RedisCommand, log=F, optional=T
* name: string, log=T, optional=F * name: string, log=T, optional=F
* raw: vector of string, log=F, optional=F * raw: vector of string, log=F, optional=F
* value: string, log=T, optional=T * value: string, log=T, optional=T
@ -598,7 +598,9 @@ connection {
conn_id { ... } conn_id { ... }
* reply: record Redis::ReplyData, log=T, optional=T * reply: record Redis::ReplyData, log=T, optional=T
Redis::ReplyData { Redis::ReplyData {
* value: string, log=T, optional=T * attributes: string, log=F, optional=T
* min_protocol_version: count, log=F, optional=F
* value: string, log=T, optional=F
} }
* success: bool, log=T, optional=T * success: bool, log=T, optional=T
* ts: time, log=T, optional=F * ts: time, log=T, optional=F
@ -615,6 +617,9 @@ connection {
} }
* pending: table[count] of record Redis::Info, log=F, optional=F * pending: table[count] of record Redis::Info, log=F, optional=F
Redis::Info { ... } Redis::Info { ... }
* resp_version: enum Redis::RESPVersion, log=F, optional=T
* skip_commands: set[count], log=F, optional=F
* subscribed_mode: bool, log=F, optional=T
* violation: bool, log=F, optional=T * violation: bool, log=F, optional=T
} }
* removal_hooks: set[func], log=F, optional=T * removal_hooks: set[func], log=F, optional=T

3
testing/btest/Baseline/scripts.base.protocols.redis.attributes/output Normal file
View file

@ -0,0 +1,3 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Got data:, [1, 2, 3]
Got data:, [2039123, 9543892], with attributes:, {key-popularity: {a: 0.1923, b: 0.0012}}

12
testing/btest/Baseline/scripts.base.protocols.redis.attributes/redis.log Normal file
View file

@ -0,0 +1,12 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path redis
#open XXXX-XX-XX-XX-XX-XX
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.name cmd.key cmd.value success reply.value
#types time string addr port addr port string string string bool string
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 62283 127.0.0.1 6379 FAKE - - T [1, 2, 3]
XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 127.0.0.1 62286 127.0.0.1 6379 FAKE2 - - T [2039123, 9543892]
#close XXXX-XX-XX-XX-XX-XX

6
testing/btest/Baseline/scripts.base.protocols.redis.pubsub-resp3/output Normal file
View file

@ -0,0 +1,6 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Got published data!, [subscribe, Foo, 1]
Got published data!, [psubscribe, F*, 2]
Got published data!, [message, Foo, Hi:)]
Got published data!, [pmessage, F*, Foo, Hi:)]
Got published data!, [pmessage, F*, Foobar, Hello!]

5
testing/btest/Baseline/scripts.base.protocols.redis.pubsub/output
View file

@ -1 +1,6 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Got published data!, [subscribe, Foo, 1]
Got published data!, [psubscribe, F*, 2]
Got published data!, [message, Foo, Hi there :)]
Got published data!, [pmessage, F*, Foo, Hi there :)]
Got published data!, [pmessage, F*, FeeFooFiiFum, Hello! :)]

10
testing/btest/Baseline/scripts.base.protocols.redis.pubsub/redis.log
View file

@ -7,7 +7,11 @@
#open XXXX-XX-XX-XX-XX-XX #open XXXX-XX-XX-XX-XX-XX
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.name cmd.key cmd.value success reply.value #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.name cmd.key cmd.value success reply.value
#types time string addr port addr port string string string bool string #types time string addr port addr port string string string bool string
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 56162 127.0.0.1 6379 SUBSCRIBE - - T - XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 127.0.0.1 60833 127.0.0.1 6379 PUBLISH - - T 2
XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 127.0.0.1 56163 127.0.0.1 6379 PUBLISH - - T - XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 60837 127.0.0.1 6379 PUBLISH - - T 1
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 56162 127.0.0.1 6379 - - - T - XXXXXXXXXX.XXXXXX CtPZjS20MLrsMUOJi2 127.0.0.1 60838 127.0.0.1 6379 SET sanity_check you_are_sane T OK
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 SUBSCRIBE - - - -
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 PSUBSCRIBE - - - -
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 RESET - - T RESET
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 GET sanity_check - T you_are_sane
#close XXXX-XX-XX-XX-XX-XX #close XXXX-XX-XX-XX-XX-XX

2
testing/btest/Baseline/scripts.base.protocols.redis.stream/redis.log
View file

@ -10,5 +10,5 @@
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622832637-0 XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622832637-0
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622836953-0 XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622836953-0
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622840530-0 XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622840530-0
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XRANGE - - T - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XRANGE - - T [[1729622770972-0, [rider, Castilla, speed, 30.2, position, 1, location_id, 1]], [1729622778221-0, [rider, Norem, speed, 28.8, position, 3, location_id, 1]]]
#close XXXX-XX-XX-XX-XX-XX #close XXXX-XX-XX-XX-XX-XX

BIN
testing/btest/Traces/redis/attr.pcap Normal file
View file

Binary file not shown.

BIN
testing/btest/Traces/redis/pubsub-resp3.pcap Normal file
View file

Binary file not shown.

BIN
testing/btest/Traces/redis/pubsub.pcap
View file

Binary file not shown.

19
testing/btest/scripts/base/protocols/redis/attributes.zeek Normal file
View file

@ -0,0 +1,19 @@
# @TEST-DOC: Test Redis protocol handling with replies with attributes
# @TEST-REQUIRES: have-spicy
#
# @TEST-EXEC: zeek -b -r $TRACES/redis/attr.pcap %INPUT >output
# @TEST-EXEC: btest-diff output
# @TEST-EXEC: btest-diff redis.log
# IMPORTANT: The test data was made synthetically, since real commands that
# return attributes may be version-specific. Real traffic would be better.
@load base/protocols/redis
event Redis::reply(c: connection, data: Redis::ReplyData)
{
if ( ! data?$attributes )
print "Got data:", data$value;
else
print "Got data:", data$value, "with attributes:", data$attributes;
}

16
testing/btest/scripts/base/protocols/redis/pubsub-resp3.zeek Normal file
View file

@ -0,0 +1,16 @@
# @TEST-DOC: Test Zeek parsing pubsub commands in RESP3
# @TEST-REQUIRES: have-spicy
#
# @TEST-EXEC: zeek -b -r $TRACES/redis/pubsub-resp3.pcap %INPUT >output
# @TEST-EXEC: btest-diff output
# Test pub/sub from Redis. This has two subscribers, one using a pattern. Then, the
# messages that were published get printed to output.
@load base/protocols/redis
event Redis::server_push(c: connection, data: Redis::ReplyData)
{
# The first 2 are SUBSCRIBE replies, the other 3 are message and pmessage
print "Got published data!", data$value;
}

10
testing/btest/scripts/base/protocols/redis/pubsub.zeek
View file

@ -5,8 +5,12 @@
# @TEST-EXEC: btest-diff output # @TEST-EXEC: btest-diff output
# @TEST-EXEC: btest-diff redis.log # @TEST-EXEC: btest-diff redis.log
# Testing the example of pub sub in REDIS docs: # Test pub/sub from Redis. This has two subscribers, one using a pattern. Then, the
# https://redis.io/docs/latest/develop/interact/pubsub/ # messages that were published get printed to output.
# These are just commands between two different clients, one PUBLISH and one SUBSCRIBE
@load base/protocols/redis @load base/protocols/redis
event Redis::server_push(c: connection, data: Redis::ReplyData)
{
print "Got published data!", data$value;
}