Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/topic/etyp/redis-resp3'

Browse source 
* origin/topic/etyp/redis-resp3:
  Touchup TODOs in the Redis analyzer
  Handle more Redis RESP3 protocol pieces
  Stringify all Redis-RESP serialized data
  Handle Redis protocol `message` separately
  Add Redis analyzer array stringification
This commit is contained in:
Evan Typanski 2025-07-01 14:19:26 -04:00
commit 310a82e7fd
20 changed files with 505 additions and 158 deletions
Download patch file Download diff file Expand all files Collapse all files

21
CHANGES
View file

@ -1,3 +1,24 @@
8.0.0-dev.577 | 2025-07-01 14:19:26 -0400
* Touchup TODOs in the Redis analyzer (Evan Typanski, Corelight)
Also renames `KnownCommand` to `RedisCommand` to avoid conflicts.
* Handle more Redis RESP3 protocol pieces (Evan Typanski, Corelight)
This passes the "minimum protocol version" along in the reply and adds
support for attributes, which were added relatively recently.
* Stringify all Redis-RESP serialized data (Evan Typanski, Corelight)
* GH-4504: Handle Redis protocol `message` separately (Evan Typanski, Corelight)
Closes #4504
Messages are not typical responses, so they need special handling. This
is different between RESP2 and 3, so this is the first instance where
the script layer needs to tell the difference.
8.0.0-dev.571 | 2025-07-01 11:03:23 +0200
* Bump pre-commit hooks (Benjamin Bannier, Corelight)

2
VERSION
View file

@ -1 +1 @@
8.0.0-dev.571
8.0.0-dev.577

93
scripts/base/protocols/redis/main.zeek
View file

@ -41,6 +41,11 @@ export {
end: count &optional;
};
type RESPVersion: enum {
RESP2,
RESP3
};
type State: record {
## Pending commands.
pending: table[count] of Info;
@ -52,14 +57,34 @@ export {
## Each range is one or two elements, one meaning it's unbounded, two meaning
## it begins at one and ends at the second.
no_reply_ranges: vector of NoReplyRange;
## The command indexes (from current_command and current_reply) that will
## not get responses no matter what.
skip_commands: set[count];
## We store if this analyzer had a violation to avoid logging if so.
## This should not be super necessary, but worth a shot.
violation: bool &default=F;
## If we are in "subscribed" mode
subscribed_mode: bool &default=F;
## The RESP version
resp_version: RESPVersion &default=RESP2;
};
# Redis specifically mentions 10k commands as a good pipelining threshold, so
# we'll piggyback on that.
option max_pending_commands = 10000;
# These commands enter subscribed mode
global enter_subscribed_mode = [RedisCommand_PSUBSCRIBE,
RedisCommand_SSUBSCRIBE, RedisCommand_SUBSCRIBE];
# These commands exit subscribed mode
global exit_subscribed_mode = [RedisCommand_RESET, RedisCommand_QUIT];
# These commands don't expect a response (ever) - their replies are out of band.
global no_response_commands = [RedisCommand_PSUBSCRIBE,
RedisCommand_PUNSUBSCRIBE, RedisCommand_SSUBSCRIBE,
RedisCommand_SUBSCRIBE, RedisCommand_SUNSUBSCRIBE,
RedisCommand_UNSUBSCRIBE];
}
redef record connection += {
@ -122,7 +147,16 @@ function is_last_interval_closed(c: connection): bool
c$redis_state$no_reply_ranges[-1]?$end;
}
event Redis::command(c: connection, cmd: Command)
event hello_command(c: connection, hello: HelloCommand)
{
if ( ! c?$redis_state )
make_new_state(c);
if ( hello?$requested_resp_version && hello$requested_resp_version == "3" )
c$redis_state$resp_version = RESP3;
}
event command(c: connection, cmd: Command)
{
if ( ! c?$redis_state )
make_new_state(c);
@ -139,10 +173,30 @@ event Redis::command(c: connection, cmd: Command)
}
++c$redis_state$current_command;
if ( cmd?$known )
{
if ( c$redis_state$resp_version == RESP2 )
{
local should_enter = cmd$known in enter_subscribed_mode;
local should_exit = cmd$known in exit_subscribed_mode;
c$redis_state$subscribed_mode = should_enter && ! should_exit;
# It's weird if it's in both - in the future users may be able to add that
if ( should_enter && should_exit )
Reporter::conn_weird("Redis_command_enter_exit_subscribed_mode", c, cat(
cmd$known));
}
if ( cmd$known in no_response_commands || c$redis_state$subscribed_mode )
{
add c$redis_state$skip_commands[c$redis_state$current_command];
}
}
# CLIENT commands can skip a number of replies and may be used with
# pipelining. We need special logic in order to track the command/reply
# pairs.
if ( cmd?$known && cmd$known == KnownCommand_CLIENT )
if ( cmd?$known && cmd$known == RedisCommand_CLIENT )
{
# All 3 CLIENT commands we care about have 3 elements
if ( |cmd$raw| == 3 )
@ -177,6 +231,7 @@ event Redis::command(c: connection, cmd: Command)
}
}
}
set_state(c, T);
c$redis$cmd = cmd;
@ -187,17 +242,24 @@ event Redis::command(c: connection, cmd: Command)
function reply_num(c: connection): count
{
local resp_num = c$redis_state$current_reply + 1;
local result = resp_num;
for ( i in c$redis_state$no_reply_ranges )
{
local range = c$redis_state$no_reply_ranges[i];
if ( ! range?$end && resp_num > range$begin )
{ } # TODO: This is necessary if not using pipelining
if ( range?$end && resp_num >= range$begin && resp_num < range$end )
return range$end;
result = range$end;
}
# Default: no disable/enable shenanigans
return resp_num;
# Account for commands that don't expect a response
while ( result in c$redis_state$skip_commands )
{
delete c$redis_state$skip_commands[result];
result += 1;
}
return result;
}
# Logs up to and including the last seen command from the last reply
@ -229,11 +291,24 @@ function log_from(c: connection, previous_reply_num: count)
}
}
event Redis::reply(c: connection, data: ReplyData)
event reply(c: connection, data: ReplyData)
{
if ( ! c?$redis_state )
make_new_state(c);
# If the server is talking in RESP3, mark accordingly, even if we didn't see HELLO
if ( data$min_protocol_version == 3 )
{
c$redis_state$resp_version = RESP3;
c$redis_state$subscribed_mode = F;
}
if ( c$redis_state$subscribed_mode )
{
event server_push(c, data);
return;
}
local previous_reply_num = c$redis_state$current_reply;
c$redis_state$current_reply = reply_num(c);
set_state(c, F);
@ -241,9 +316,13 @@ event Redis::reply(c: connection, data: ReplyData)
c$redis$reply = data;
c$redis$success = T;
log_from(c, previous_reply_num);
# Tidy up the skip_commands when it's up to date
if ( c$redis_state$current_command == c$redis_state$current_reply )
clear_table(c$redis_state$skip_commands);
}
event Redis::error(c: connection, data: ReplyData)
event error(c: connection, data: ReplyData)
{
if ( ! c?$redis_state )
make_new_state(c);

36
scripts/base/protocols/redis/spicy-events.zeek
View file

@ -37,6 +37,12 @@ export {
password: string;
};
## The Redis HELLO command (handshake).
type HelloCommand: record {
## The sent requested RESP version, such as "2" or "3"
requested_resp_version: string &optional;
};
## A generic Redis command from the client.
type Command: record {
## The raw command, exactly as parsed
@ -49,12 +55,17 @@ export {
## The value, if this command is known to have a value
value: string &log &optional;
## The command in an enum if it was known
known: KnownCommand &optional;
known: RedisCommand &optional;
};
## A generic Redis reply from the client.
type ReplyData: record {
value: string &log &optional;
## The RESP3 attributes applied to this, if any
attributes: string &optional;
## The string version of the reply data
value: string &log;
## The minimum RESP version that supports this reply type
min_protocol_version: count;
};
}
@ -79,6 +90,13 @@ global get_command: event(c: connection, key: string);
## command: The AUTH command sent to the server and its data.
global auth_command: event(c: connection, command: AuthCommand);
## Generated for Redis HELLO commands sent to the Redis server.
##
## c: The connection.
##
## command: The HELLO command sent to the server and its data.
global hello_command: event(c: connection, command: HelloCommand);
## Generated for every command sent by the client to the Redis server.
##
## c: The connection.
@ -87,11 +105,15 @@ global auth_command: event(c: connection, command: AuthCommand);
global command: event(c: connection, cmd: Command);
## Generated for every successful response sent by the Redis server to the
## client.
## client. For RESP2, this includes "push" messages, which are out of band.
## These will also raise a server_push event. RESP3 push messages will only
## raise a server_push event.
##
## c: The connection.
##
## data: The server data sent to the client.
##
## .. zeek:see:: Redis::server_push
global reply: event(c: connection, data: ReplyData);
## Generated for every error response sent by the Redis server to the
@ -101,3 +123,11 @@ global reply: event(c: connection, data: ReplyData);
##
## data: The server data sent to the client.
global error: event(c: connection, data: ReplyData);
## Generated for out-of-band data, outside of the request-response
## model.
##
## c: The connection.
##
## data: The server data sent to the client.
global server_push: event(c: connection, data: ReplyData);

375
src/analyzer/protocol/redis/redis.spicy
View file

@ -6,7 +6,7 @@ module Redis;
import RESP;
public type KnownCommand = enum {
public type RedisCommand = enum {
APPEND,
AUTH,
BITCOUNT,
@ -34,6 +34,7 @@ public type KnownCommand = enum {
GETRANGE,
GETSET,
HDEL,
HELLO,
HGET,
HSET,
INCR,
@ -43,11 +44,19 @@ public type KnownCommand = enum {
MOVE,
MSET,
PERSIST,
PSUBSCRIBE,
PUNSUBSCRIBE,
QUIT,
RENAME,
RESET,
SET,
STRLEN,
SUBSCRIBE,
SSUBSCRIBE,
SUNSUBSCRIBE,
TTL,
TYPE,
UNSUBSCRIBE,
};
type Command = struct {
@ -55,7 +64,7 @@ type Command = struct {
name: bytes;
key: optional<bytes>;
value: optional<bytes>;
known: optional<KnownCommand>;
known: optional<RedisCommand>;
};
# This just assumes all elements in the array is a bulk string and puts them in a vector
@ -149,44 +158,44 @@ function parse_command(raw: vector<bytes>): Command {
if (|raw| >= 2) {
switch (*cmd) {
case KnownCommand::KEYS:
case RedisCommand::KEYS:
parsed.key = raw[1];
case KnownCommand::APPEND,
KnownCommand::BITCOUNT,
KnownCommand::BITFIELD,
KnownCommand::BITFIELD_RO,
KnownCommand::BITPOS,
KnownCommand::BLPOP,
KnownCommand::BRPOP,
KnownCommand::COPY,
KnownCommand::DECR,
KnownCommand::DECRBY,
KnownCommand::DEL,
KnownCommand::DUMP,
KnownCommand::EXISTS,
KnownCommand::EXPIRE,
KnownCommand::EXPIREAT,
KnownCommand::EXPIRETIME,
KnownCommand::GET,
KnownCommand::GETBIT,
KnownCommand::GETDEL,
KnownCommand::GETEX,
KnownCommand::GETRANGE,
KnownCommand::GETSET,
KnownCommand::HDEL,
KnownCommand::HGET,
KnownCommand::HSET,
KnownCommand::INCR,
KnownCommand::INCRBY,
KnownCommand::MGET,
KnownCommand::MOVE,
KnownCommand::MSET,
KnownCommand::PERSIST,
KnownCommand::RENAME,
KnownCommand::SET,
KnownCommand::STRLEN,
KnownCommand::TTL,
KnownCommand::TYPE:
case RedisCommand::APPEND,
RedisCommand::BITCOUNT,
RedisCommand::BITFIELD,
RedisCommand::BITFIELD_RO,
RedisCommand::BITPOS,
RedisCommand::BLPOP,
RedisCommand::BRPOP,
RedisCommand::COPY,
RedisCommand::DECR,
RedisCommand::DECRBY,
RedisCommand::DEL,
RedisCommand::DUMP,
RedisCommand::EXISTS,
RedisCommand::EXPIRE,
RedisCommand::EXPIREAT,
RedisCommand::EXPIRETIME,
RedisCommand::GET,
RedisCommand::GETBIT,
RedisCommand::GETDEL,
RedisCommand::GETEX,
RedisCommand::GETRANGE,
RedisCommand::GETSET,
RedisCommand::HDEL,
RedisCommand::HGET,
RedisCommand::HSET,
RedisCommand::INCR,
RedisCommand::INCRBY,
RedisCommand::MGET,
RedisCommand::MOVE,
RedisCommand::MSET,
RedisCommand::PERSIST,
RedisCommand::RENAME,
RedisCommand::SET,
RedisCommand::STRLEN,
RedisCommand::TTL,
RedisCommand::TYPE:
parsed.key = raw[1];
default: ();
}
@ -194,22 +203,22 @@ function parse_command(raw: vector<bytes>): Command {
if (|raw| >= 3) {
switch (*cmd) {
case KnownCommand::SET,
KnownCommand::APPEND,
KnownCommand::DECRBY,
KnownCommand::EXPIRE,
KnownCommand::EXPIREAT,
KnownCommand::GETBIT,
KnownCommand::GETSET,
KnownCommand::HDEL,
KnownCommand::HGET,
KnownCommand::INCRBY,
KnownCommand::MOVE,
KnownCommand::MSET,
KnownCommand::RENAME:
case RedisCommand::SET,
RedisCommand::APPEND,
RedisCommand::DECRBY,
RedisCommand::EXPIRE,
RedisCommand::EXPIREAT,
RedisCommand::GETBIT,
RedisCommand::GETSET,
RedisCommand::HDEL,
RedisCommand::HGET,
RedisCommand::INCRBY,
RedisCommand::MOVE,
RedisCommand::MSET,
RedisCommand::RENAME:
parsed.value = raw[2];
# Op first, destination second, then a list of keys. Just log dest
case KnownCommand::BITOP: parsed.key = raw[2];
case RedisCommand::BITOP: parsed.key = raw[2];
default: ();
}
}
@ -217,7 +226,7 @@ function parse_command(raw: vector<bytes>): Command {
if (|raw| >= 4) {
switch (*cmd) {
# timeout, numkeys, then key
case KnownCommand::BLMPOP: parsed.key = raw[3];
case RedisCommand::BLMPOP: parsed.key = raw[3];
default: ();
}
}
@ -225,51 +234,60 @@ function parse_command(raw: vector<bytes>): Command {
return parsed;
}
function command_from(cmd_bytes: bytes): optional<KnownCommand> {
local cmd: optional<KnownCommand> = Null;
function command_from(cmd_bytes: bytes): optional<RedisCommand> {
local cmd: optional<RedisCommand> = Null;
switch (cmd_bytes.lower()) {
case b"set": cmd = KnownCommand::SET;
case b"append": cmd = KnownCommand::APPEND;
case b"auth": cmd = KnownCommand::AUTH;
case b"bitcount": cmd = KnownCommand::BITCOUNT;
case b"bitfield": cmd = KnownCommand::BITFIELD;
case b"bitfield_ro": cmd = KnownCommand::BITFIELD_RO;
case b"bitop": cmd = KnownCommand::BITOP;
case b"bitpos": cmd = KnownCommand::BITPOS;
case b"blmpop": cmd = KnownCommand::BLMPOP;
case b"blpop": cmd = KnownCommand::BLPOP;
case b"brpop": cmd = KnownCommand::BRPOP;
case b"client": cmd = KnownCommand::CLIENT;
case b"copy": cmd = KnownCommand::COPY;
case b"decr": cmd = KnownCommand::DECR;
case b"decrby": cmd = KnownCommand::DECRBY;
case b"del": cmd = KnownCommand::DEL;
case b"dump": cmd = KnownCommand::DUMP;
case b"exists": cmd = KnownCommand::EXISTS;
case b"expire": cmd = KnownCommand::EXPIRE;
case b"expireat": cmd = KnownCommand::EXPIREAT;
case b"expiretime": cmd = KnownCommand::EXPIRETIME;
case b"expiretime": cmd = KnownCommand::EXPIRETIME;
case b"get": cmd = KnownCommand::GET;
case b"getbit": cmd = KnownCommand::GETBIT;
case b"getdel": cmd = KnownCommand::GETDEL;
case b"getex": cmd = KnownCommand::GETEX;
case b"getrange": cmd = KnownCommand::GETRANGE;
case b"getset": cmd = KnownCommand::GETSET;
case b"hdel": cmd = KnownCommand::HDEL;
case b"hget": cmd = KnownCommand::HGET;
case b"hset": cmd = KnownCommand::HSET;
case b"incr": cmd = KnownCommand::INCR;
case b"incrby": cmd = KnownCommand::INCRBY;
case b"keys": cmd = KnownCommand::KEYS;
case b"mget": cmd = KnownCommand::MGET;
case b"move": cmd = KnownCommand::MOVE;
case b"mset": cmd = KnownCommand::MSET;
case b"persist": cmd = KnownCommand::PERSIST;
case b"rename": cmd = KnownCommand::RENAME;
case b"strlen": cmd = KnownCommand::STRLEN;
case b"ttl": cmd = KnownCommand::TTL;
case b"type": cmd = KnownCommand::TYPE;
case b"append": cmd = RedisCommand::APPEND;
case b"auth": cmd = RedisCommand::AUTH;
case b"bitcount": cmd = RedisCommand::BITCOUNT;
case b"bitfield": cmd = RedisCommand::BITFIELD;
case b"bitfield_ro": cmd = RedisCommand::BITFIELD_RO;
case b"bitop": cmd = RedisCommand::BITOP;
case b"bitpos": cmd = RedisCommand::BITPOS;
case b"blmpop": cmd = RedisCommand::BLMPOP;
case b"blpop": cmd = RedisCommand::BLPOP;
case b"brpop": cmd = RedisCommand::BRPOP;
case b"client": cmd = RedisCommand::CLIENT;
case b"copy": cmd = RedisCommand::COPY;
case b"decr": cmd = RedisCommand::DECR;
case b"decrby": cmd = RedisCommand::DECRBY;
case b"del": cmd = RedisCommand::DEL;
case b"dump": cmd = RedisCommand::DUMP;
case b"exists": cmd = RedisCommand::EXISTS;
case b"expire": cmd = RedisCommand::EXPIRE;
case b"expireat": cmd = RedisCommand::EXPIREAT;
case b"expiretime": cmd = RedisCommand::EXPIRETIME;
case b"expiretime": cmd = RedisCommand::EXPIRETIME;
case b"get": cmd = RedisCommand::GET;
case b"getbit": cmd = RedisCommand::GETBIT;
case b"getdel": cmd = RedisCommand::GETDEL;
case b"getex": cmd = RedisCommand::GETEX;
case b"getrange": cmd = RedisCommand::GETRANGE;
case b"getset": cmd = RedisCommand::GETSET;
case b"hdel": cmd = RedisCommand::HDEL;
case b"hello": cmd = RedisCommand::HELLO;
case b"hget": cmd = RedisCommand::HGET;
case b"hset": cmd = RedisCommand::HSET;
case b"incr": cmd = RedisCommand::INCR;
case b"incrby": cmd = RedisCommand::INCRBY;
case b"keys": cmd = RedisCommand::KEYS;
case b"mget": cmd = RedisCommand::MGET;
case b"move": cmd = RedisCommand::MOVE;
case b"mset": cmd = RedisCommand::MSET;
case b"persist": cmd = RedisCommand::PERSIST;
case b"psubscribe": cmd = RedisCommand::PSUBSCRIBE;
case b"punsubscribe": cmd = RedisCommand::PUNSUBSCRIBE;
case b"quit": cmd = RedisCommand::QUIT;
case b"rename": cmd = RedisCommand::RENAME;
case b"reset": cmd = RedisCommand::RESET;
case b"set": cmd = RedisCommand::SET;
case b"strlen": cmd = RedisCommand::STRLEN;
case b"ssubscribe": cmd = RedisCommand::SSUBSCRIBE;
case b"subscribe": cmd = RedisCommand::SUBSCRIBE;
case b"sunsubscribe": cmd = RedisCommand::SUNSUBSCRIBE;
case b"ttl": cmd = RedisCommand::TTL;
case b"type": cmd = RedisCommand::TYPE;
case b"unsubscribe": cmd = RedisCommand::UNSUBSCRIBE;
default: cmd = Null;
}
@ -334,7 +352,7 @@ public function make_set(command: Command): Set {
}
public function is_set(data: RESP::ClientData): bool {
return data.command.known && *(data.command.known) == KnownCommand::SET && data.command.key && data.command.value;
return data.command.known && *(data.command.known) == RedisCommand::SET && data.command.key && data.command.value;
}
type Get = struct {
@ -347,7 +365,7 @@ public function make_get(command: Command): Get {
}
public function is_get(data: RESP::ClientData): bool {
return data.command.known && *(data.command.known) == KnownCommand::GET && |data.command.raw| >= 2;
return data.command.known && *(data.command.known) == RedisCommand::GET && |data.command.raw| >= 2;
}
type Auth = struct {
@ -365,15 +383,45 @@ public function make_auth(command: Command): Auth {
}
public function is_auth(data: RESP::ClientData): bool {
return data.command.known && *(data.command.known) == KnownCommand::AUTH && |data.command.raw| >= 2;
return data.command.known && *(data.command.known) == RedisCommand::AUTH && |data.command.raw| >= 2;
}
type Hello = struct {
requested_resp_version: optional<bytes>;
};
public function make_hello(command: Command): Hello {
local hi: Hello = [$requested_resp_version = Null];
if (|command.raw| > 1)
hi.requested_resp_version = command.raw[1];
return hi;
}
public function is_hello(data: RESP::ClientData): bool {
return data.command.known && *(data.command.known) == RedisCommand::HELLO;
}
type ReplyData = struct {
value: optional<bytes>;
attributes: optional<bytes>;
value: bytes;
min_protocol_version: uint8;
};
public function is_err(server_data: RESP::ServerData): bool {
return server_data.data?.simple_error || server_data.data?.bulk_error;
public type ReplyType = enum {
Reply, # A response to a command
Error, # An error response to a command
Push, # A server message that is not responding to a command
};
public function classify(server_data: RESP::ServerData): ReplyType {
if (server_data.data?.simple_error || server_data.data?.bulk_error)
return ReplyType::Error;
# We can tell with RESP3 this is push here, but RESP2 relies on scripts
if (server_data.data?.push)
return ReplyType::Push;
return ReplyType::Reply;
}
function bulk_string_content(bulk: RESP::BulkString): bytes {
@ -383,21 +431,108 @@ function bulk_string_content(bulk: RESP::BulkString): bytes {
return b"";
}
# Gets the server reply in a simpler form
public function make_server_reply(data: RESP::ServerData): ReplyData {
local res: ReplyData = [$value = Null];
if (data.data?.simple_error)
res.value = data.data.simple_error.content;
else if (data.data?.bulk_error)
res.value = bulk_string_content(data.data.bulk_error);
else if (data.data?.simple_string)
res.value = data.data.simple_string.content;
else if (data.data?.bulk_string)
res.value = bulk_string_content(data.data.bulk_string);
else if (data.data?.verbatim_string)
res.value = bulk_string_content(data.data.verbatim_string);
else if (data.data?.boolean)
res.value = data.data.boolean.val ? b"T" : b"F";
function stringify_map(data: RESP::Map&): bytes {
local res = b"{";
local first = True;
local i = 0;
# num_elements refers to the number of map entries, each with 2 entries
# in the raw data
while (i < data.num_elements) {
if (!first)
res += b", ";
res += stringify(data.raw_data[i * 2]);
res += b": ";
res += stringify(data.raw_data[(i * 2) + 1]);
i += 1;
first = False;
}
res += b"}";
return res;
}
# Returns the bytes string value of this, or Null if it cannot.
function stringify(data: RESP::Data&): bytes {
if (data?.simple_string)
return data.simple_string.content;
else if (data?.simple_error)
return data.simple_error.content;
else if (data?.integer)
return data.integer.val;
else if (data?.bulk_string)
return bulk_string_content(data.bulk_string);
else if (data?.array) {
local res = b"[";
local first = True;
for (ele in data.array.elements) {
if (!first)
res += b", ";
res += stringify(ele);
first = False;
}
res += b"]";
return res;
} else if (data?.null)
return b"null";
else if (data?.boolean)
return data.boolean.val ? b"T" : b"F";
else if (data?.double)
return data.double.val;
else if (data?.big_num)
return data.big_num.val;
else if (data?.bulk_error)
return bulk_string_content(data.bulk_error);
else if (data?.verbatim_string)
return bulk_string_content(data.verbatim_string);
else if (data?.map_) {
return stringify_map(data.map_);
} else if (data?.set_) {
local res = b"(";
local first = True;
for (ele in data.set_.elements) {
if (!first)
res += b", ";
res += stringify(ele);
first = False;
}
res += b")";
return res;
} else if (data?.push) {
local res = b"[";
local first = True;
for (ele in data.push.elements) {
if (!first)
res += b", ";
res += stringify(ele);
first = False;
}
res += b"]";
return res;
}
throw "unknown RESP type";
}
# Gets the server reply in a simpler form
public function make_server_reply(data: RESP::ServerData): ReplyData {
local min_protocol_version: uint8 = 2;
switch (data.data.ty) {
case RESP::DataType::NULL,
RESP::DataType::BOOLEAN,
RESP::DataType::DOUBLE,
RESP::DataType::BIG_NUM,
RESP::DataType::BULK_ERROR,
RESP::DataType::VERBATIM_STRING,
RESP::DataType::MAP,
RESP::DataType::SET,
RESP::DataType::PUSH: min_protocol_version = 3;
default: min_protocol_version = 2;
}
local attributes: optional<bytes> = Null;
if (data.data?.attributes) {
min_protocol_version = 3;
attributes = stringify_map(data.data.attributes);
}
return [$attributes = attributes, $value = stringify(data.data), $min_protocol_version = min_protocol_version];
}

11
src/analyzer/protocol/redis/resp.evt
View file

@ -7,14 +7,19 @@ protocol analyzer Redis over TCP:
import RESP;
import Redis;
export Redis::KnownCommand;
export Redis::RedisCommand;
on RESP::ClientData if ( Redis::is_set(self) ) -> event Redis::set_command($conn, Redis::make_set(self.command));
on RESP::ClientData if ( Redis::is_get(self) ) -> event Redis::get_command($conn, Redis::make_get(self.command).key);
on RESP::ClientData if ( Redis::is_auth(self) ) -> event Redis::auth_command($conn, Redis::make_auth(self.command));
on RESP::ClientData if ( Redis::is_hello(self) ) -> event Redis::hello_command($conn, Redis::make_hello(self.command));
# All client data is a command
on RESP::ClientData -> event Redis::command($conn, self.command);
on RESP::ServerData if ( ! Redis::is_err(self) ) -> event Redis::reply($conn, Redis::make_server_reply(self));
on RESP::ServerData if ( Redis::is_err(self) ) -> event Redis::error($conn, Redis::make_server_reply(self));
on RESP::ServerData if ( Redis::classify(self) == Redis::ReplyType::Reply ) ->
event Redis::reply($conn, Redis::make_server_reply(self));
on RESP::ServerData if ( Redis::classify(self) == Redis::ReplyType::Error ) ->
event Redis::error($conn, Redis::make_server_reply(self));
on RESP::ServerData if ( Redis::classify(self) == Redis::ReplyType::Push ) ->
event Redis::server_push($conn, Redis::make_server_reply(self));

33
src/analyzer/protocol/redis/resp.spicy
View file

@ -79,11 +79,25 @@ public type ServerData = unit {
%synchronize-after = b"\x0d\x0a";
var depth: uint8& = new uint8;
data: Data(self.depth);
var type_: Redis::ReplyType;
on %done {
self.type_ = Redis::classify(self);
}
};
type Data = unit(depth: uint8&) {
%synchronize-after = b"\x0d\x0a";
ty: uint8 &convert=DataType($$);
# Attributes are special, they precede the actual data
if (self.ty == DataType::ATTRIBUTE) {
attributes: Map(depth);
: uint8 &convert=DataType($$) {
self.ty = $$;
}
};
switch (self.ty) {
DataType::SIMPLE_STRING -> simple_string: SimpleString(False);
DataType::SIMPLE_ERROR -> simple_error: SimpleString(True);
@ -102,7 +116,7 @@ type Data = unit(depth: uint8&) {
DataType::MAP -> map_: Map(depth);
DataType::SET -> set_: Set(depth);
# "Push events are encoded similarly to arrays, differing only in their
# first byte" - TODO: can probably make it more obvious, though
# first byte"
DataType::PUSH -> push: Array(depth);
};
@ -130,6 +144,7 @@ type DataType = enum {
BULK_ERROR = '!',
VERBATIM_STRING = '=',
MAP = '%',
ATTRIBUTE = '|',
SET = '~',
PUSH = '>',
};
@ -144,7 +159,7 @@ type SimpleString = unit(is_error: bool) {
};
type Integer = unit {
int: RedisBytes &convert=$$.to_int(10);
val: RedisBytes;
};
type BulkString = unit(is_error: bool) {
@ -173,32 +188,20 @@ type Boolean = unit {
};
type Double = unit {
val: RedisBytes &convert=$$.to_real();
val: RedisBytes;
};
type BigNum = unit {
# Big num can be very big so leave it in bytes.
val: RedisBytes;
};
type Map = unit(depth: uint8&) {
num_elements: RedisBytes &convert=$$.to_uint(10);
# TODO: How can I make this into a map? Alternatively, how can I do this better?
raw_data: Data(depth)[self.num_elements * 2];
# TODO: This is broken. See https://github.com/zeek/spicy/issues/2061
# var key_val_pairs: vector<tuple<Data, Data>>;
# on raw_data {
# while (local i = 0; i < self.num_elements) {
# self.key_val_pairs.push_back(($$[i], $$[i + 1]));
# i += 2;
# }
# }
};
type Set = unit(depth: uint8&) {
num_elements: RedisBytes &convert=$$.to_uint(10) &requires=self.num_elements <= MAX_SIZE;
# TODO: This should be a set but doesn't go in the backed C++ set
elements: Data(depth)[self.num_elements];
};

9
testing/btest/Baseline/coverage.record-fields/out.default
View file

@ -589,7 +589,7 @@ connection {
* cmd: record Redis::Command, log=T, optional=F
Redis::Command {
* key: string, log=T, optional=T
* known: enum Redis::KnownCommand, log=F, optional=T
* known: enum Redis::RedisCommand, log=F, optional=T
* name: string, log=T, optional=F
* raw: vector of string, log=F, optional=F
* value: string, log=T, optional=T
@ -598,7 +598,9 @@ connection {
conn_id { ... }
* reply: record Redis::ReplyData, log=T, optional=T
Redis::ReplyData {
* value: string, log=T, optional=T
* attributes: string, log=F, optional=T
* min_protocol_version: count, log=F, optional=F
* value: string, log=T, optional=F
}
* success: bool, log=T, optional=T
* ts: time, log=T, optional=F
@ -615,6 +617,9 @@ connection {
}
* pending: table[count] of record Redis::Info, log=F, optional=F
Redis::Info { ... }
* resp_version: enum Redis::RESPVersion, log=F, optional=T
* skip_commands: set[count], log=F, optional=F
* subscribed_mode: bool, log=F, optional=T
* violation: bool, log=F, optional=T
}
* removal_hooks: set[func], log=F, optional=T

3
testing/btest/Baseline/scripts.base.protocols.redis.attributes/output Normal file
View file

@ -0,0 +1,3 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Got data:, [1, 2, 3]
Got data:, [2039123, 9543892], with attributes:, {key-popularity: {a: 0.1923, b: 0.0012}}

12
testing/btest/Baseline/scripts.base.protocols.redis.attributes/redis.log Normal file
View file

@ -0,0 +1,12 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path redis
#open XXXX-XX-XX-XX-XX-XX
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.name cmd.key cmd.value success reply.value
#types time string addr port addr port string string string bool string
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 62283 127.0.0.1 6379 FAKE - - T [1, 2, 3]
XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 127.0.0.1 62286 127.0.0.1 6379 FAKE2 - - T [2039123, 9543892]
#close XXXX-XX-XX-XX-XX-XX

6
testing/btest/Baseline/scripts.base.protocols.redis.pubsub-resp3/output Normal file
View file

@ -0,0 +1,6 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Got published data!, [subscribe, Foo, 1]
Got published data!, [psubscribe, F*, 2]
Got published data!, [message, Foo, Hi:)]
Got published data!, [pmessage, F*, Foo, Hi:)]
Got published data!, [pmessage, F*, Foobar, Hello!]

5
testing/btest/Baseline/scripts.base.protocols.redis.pubsub/output
View file

@ -1 +1,6 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Got published data!, [subscribe, Foo, 1]
Got published data!, [psubscribe, F*, 2]
Got published data!, [message, Foo, Hi there :)]
Got published data!, [pmessage, F*, Foo, Hi there :)]
Got published data!, [pmessage, F*, FeeFooFiiFum, Hello! :)]

10
testing/btest/Baseline/scripts.base.protocols.redis.pubsub/redis.log
View file

@ -7,7 +7,11 @@
#open XXXX-XX-XX-XX-XX-XX
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.name cmd.key cmd.value success reply.value
#types time string addr port addr port string string string bool string
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 56162 127.0.0.1 6379 SUBSCRIBE - - T -
XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 127.0.0.1 56163 127.0.0.1 6379 PUBLISH - - T -
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 56162 127.0.0.1 6379 - - - T -
XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 127.0.0.1 60833 127.0.0.1 6379 PUBLISH - - T 2
XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 60837 127.0.0.1 6379 PUBLISH - - T 1
XXXXXXXXXX.XXXXXX CtPZjS20MLrsMUOJi2 127.0.0.1 60838 127.0.0.1 6379 SET sanity_check you_are_sane T OK
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 SUBSCRIBE - - - -
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 PSUBSCRIBE - - - -
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 RESET - - T RESET
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 60831 127.0.0.1 6379 GET sanity_check - T you_are_sane
#close XXXX-XX-XX-XX-XX-XX

2
testing/btest/Baseline/scripts.base.protocols.redis.stream/redis.log
View file

@ -10,5 +10,5 @@
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622832637-0
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622836953-0
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XADD - - T 1729622840530-0
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XRANGE - - T -
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 49992 127.0.0.1 6379 XRANGE - - T [[1729622770972-0, [rider, Castilla, speed, 30.2, position, 1, location_id, 1]], [1729622778221-0, [rider, Norem, speed, 28.8, position, 3, location_id, 1]]]
#close XXXX-XX-XX-XX-XX-XX

BIN
testing/btest/Traces/redis/attr.pcap Normal file
View file

Binary file not shown.

BIN
testing/btest/Traces/redis/pubsub-resp3.pcap Normal file
View file

Binary file not shown.

BIN
testing/btest/Traces/redis/pubsub.pcap
View file

Binary file not shown.

19
testing/btest/scripts/base/protocols/redis/attributes.zeek Normal file
View file

@ -0,0 +1,19 @@
# @TEST-DOC: Test Redis protocol handling with replies with attributes
# @TEST-REQUIRES: have-spicy
#
# @TEST-EXEC: zeek -b -r $TRACES/redis/attr.pcap %INPUT >output
# @TEST-EXEC: btest-diff output
# @TEST-EXEC: btest-diff redis.log
# IMPORTANT: The test data was made synthetically, since real commands that
# return attributes may be version-specific. Real traffic would be better.
@load base/protocols/redis
event Redis::reply(c: connection, data: Redis::ReplyData)
{
if ( ! data?$attributes )
print "Got data:", data$value;
else
print "Got data:", data$value, "with attributes:", data$attributes;
}

16
testing/btest/scripts/base/protocols/redis/pubsub-resp3.zeek Normal file
View file

@ -0,0 +1,16 @@
# @TEST-DOC: Test Zeek parsing pubsub commands in RESP3
# @TEST-REQUIRES: have-spicy
#
# @TEST-EXEC: zeek -b -r $TRACES/redis/pubsub-resp3.pcap %INPUT >output
# @TEST-EXEC: btest-diff output
# Test pub/sub from Redis. This has two subscribers, one using a pattern. Then, the
# messages that were published get printed to output.
@load base/protocols/redis
event Redis::server_push(c: connection, data: Redis::ReplyData)
{
# The first 2 are SUBSCRIBE replies, the other 3 are message and pmessage
print "Got published data!", data$value;
}

10
testing/btest/scripts/base/protocols/redis/pubsub.zeek
View file

@ -5,8 +5,12 @@
# @TEST-EXEC: btest-diff output
# @TEST-EXEC: btest-diff redis.log
# Testing the example of pub sub in REDIS docs:
# https://redis.io/docs/latest/develop/interact/pubsub/
# These are just commands between two different clients, one PUBLISH and one SUBSCRIBE
# Test pub/sub from Redis. This has two subscribers, one using a pattern. Then, the
# messages that were published get printed to output.
@load base/protocols/redis
event Redis::server_push(c: connection, data: Redis::ReplyData)
{
print "Got published data!", data$value;
}