Handle more Redis RESP3 protocol pieces

This passes the "minimum protocol version" along in the reply and adds
support for attributes, which were added relatively recently.

scripts/base/protocols/redis/main.zeek

@@ -296,6 +296,12 @@ event Redis::reply(c: connection, data: ReplyData)
 	if ( ! c?$redis_state )
 		make_new_state(c);
 
+	if ( data$min_protocol_version == 3 )
+		{
+		c$redis_state$resp_version = RESP3;
+		c$redis_state$subscribed_mode = F;
+		}
+
 	if ( c$redis_state$subscribed_mode )
 		{
 		event server_push(c, data);

scripts/base/protocols/redis/spicy-events.zeek

@@ -60,7 +60,12 @@ export {
 
 	## A generic Redis reply from the client.
 	type ReplyData: record {
-		value: string &log &optional;
+		## The RESP3 attributes applied to this, if any
+		attributes: string &optional;
+		## The string version of the reply data
+		value: string &log;
+		## The minimum RESP version that supports this reply type
+		min_protocol_version: count;
 	};
 }
 

src/analyzer/protocol/redis/redis.spicy

@@ -402,7 +402,9 @@ public function is_hello(data: RESP::ClientData): bool {
 }
 
 type ReplyData = struct {
-    value: optional<bytes>;
+    attributes: optional<bytes>;
+    value: bytes;
+    min_protocol_version: uint8;
 };
 
 public type ReplyType = enum {
@@ -429,6 +431,25 @@ function bulk_string_content(bulk: RESP::BulkString): bytes {
         return b"";
 }
 
+function stringify_map(data: RESP::Map): bytes {
+    local res = b"{";
+    local first = True;
+    local i = 0;
+    # num_elements refers to the number of map entries, each with 2 entries
+    # in the raw data
+    while (i < data.num_elements) {
+        if (!first)
+            res += b", ";
+        res += stringify(data.raw_data[i * 2]);
+        res += b": ";
+        res += stringify(data.raw_data[(i * 2) + 1]);
+        i += 1;
+        first = False;
+    }
+    res += b"}";
+    return res;
+}
+
 # Returns the bytes string value of this, or Null if it cannot.
 function stringify(data: RESP::Data): bytes {
     if (data?.simple_string)
@@ -463,20 +484,7 @@ function stringify(data: RESP::Data): bytes {
     else if (data?.verbatim_string)
         return bulk_string_content(data.verbatim_string);
     else if (data?.map_) {
-        local res = b"{";
+        return stringify_map(data.map_);
-        local first = True;
-        local i = 0;
-        while (i < data.map_.num_elements) {
-            if (!first)
-                res += b", ";
-            res += stringify(data.map_.raw_data[i]);
-            res += b": ";
-            res += stringify(data.map_.raw_data[i + 1]);
-            i += 2;
-            first = False;
-        }
-        res += b"}";
-        return res;
     } else if (data?.set_) {
         local res = b"(";
         local first = True;
@@ -506,5 +514,25 @@ function stringify(data: RESP::Data): bytes {
 
 # Gets the server reply in a simpler form
 public function make_server_reply(data: RESP::ServerData): ReplyData {
-    return [$value = stringify(data.data)];
+    local min_protocol_version: uint8 = 2;
+    switch (data.data.ty) {
+        case RESP::DataType::NULL,
+             RESP::DataType::BOOLEAN,
+             RESP::DataType::DOUBLE,
+             RESP::DataType::BIG_NUM,
+             RESP::DataType::BULK_ERROR,
+             RESP::DataType::VERBATIM_STRING,
+             RESP::DataType::MAP,
+             RESP::DataType::SET,
+             RESP::DataType::PUSH: min_protocol_version = 3;
+        default: min_protocol_version = 2;
+    }
+
+    local attributes: optional<bytes> = Null;
+    if (data.data?.attributes) {
+        min_protocol_version = 3;
+        attributes = stringify_map(data.data.attributes);
+    }
+
+    return [$attributes = attributes, $value = stringify(data.data), $min_protocol_version = min_protocol_version];
 }

src/analyzer/protocol/redis/resp.spicy

@@ -89,6 +89,15 @@ public type ServerData = unit {
 type Data = unit(depth: uint8&) {
     %synchronize-after = b"\x0d\x0a";
     ty: uint8 &convert=DataType($$);
+
+    # Attributes are special, they precede the actual data
+    if (self.ty == DataType::ATTRIBUTE) {
+        attributes: Map(depth);
+        : uint8 &convert=DataType($$) {
+            self.ty = $$;
+        }
+    };
+
     switch (self.ty) {
         DataType::SIMPLE_STRING -> simple_string: SimpleString(False);
         DataType::SIMPLE_ERROR -> simple_error: SimpleString(True);
@@ -135,6 +144,7 @@ type DataType = enum {
     BULK_ERROR = '!',
     VERBATIM_STRING = '=',
     MAP = '%',
+    ATTRIBUTE = '|',
     SET = '~',
     PUSH = '>',
 };

testing/btest/Baseline/coverage.record-fields/out.default

@@ -598,7 +598,9 @@ connection {
             conn_id { ... }
         * reply: record Redis::ReplyData, log=T, optional=T
             Redis::ReplyData {
-              * value: string, log=T, optional=T
+              * attributes: string, log=F, optional=T
+              * min_protocol_version: count, log=F, optional=F
+              * value: string, log=T, optional=F
              }
         * success: bool, log=T, optional=T
         * ts: time, log=T, optional=F

testing/btest/Baseline/scripts.base.protocols.redis.attributes/output

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Got data:, [1, 2, 3]
+Got data:, [2039123, 9543892], with attributes:, {key-popularity: {a: 0.1923, b: 0.0012}}

testing/btest/Baseline/scripts.base.protocols.redis.attributes/redis.log

@@ -0,0 +1,12 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	redis
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	cmd.name	cmd.key	cmd.value	success	reply.value
+#types	time	string	addr	port	addr	port	string	string	string	bool	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	62283	127.0.0.1	6379	FAKE	-	-	T	[1, 2, 3]
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	127.0.0.1	62286	127.0.0.1	6379	FAKE2	-	-	T	[2039123, 9543892]
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.redis.pubsub-resp3/output

@@ -1,6 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-Got published data!, [value=[subscribe, Foo, 1]]
+Got published data!, [subscribe, Foo, 1]
-Got published data!, [value=[psubscribe, F*, 2]]
+Got published data!, [psubscribe, F*, 2]
-Got published data!, [value=[message, Foo, Hi:)]]
+Got published data!, [message, Foo, Hi:)]
-Got published data!, [value=[pmessage, F*, Foo, Hi:)]]
+Got published data!, [pmessage, F*, Foo, Hi:)]
-Got published data!, [value=[pmessage, F*, Foobar, Hello!]]
+Got published data!, [pmessage, F*, Foobar, Hello!]

testing/btest/Baseline/scripts.base.protocols.redis.pubsub/output

@@ -1,6 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-Got published data!, [value=[subscribe, Foo, 1]]
+Got published data!, [subscribe, Foo, 1]
-Got published data!, [value=[psubscribe, F*, 2]]
+Got published data!, [psubscribe, F*, 2]
-Got published data!, [value=[message, Foo, Hi there :)]]
+Got published data!, [message, Foo, Hi there :)]
-Got published data!, [value=[pmessage, F*, Foo, Hi there :)]]
+Got published data!, [pmessage, F*, Foo, Hi there :)]
-Got published data!, [value=[pmessage, F*, FeeFooFiiFum, Hello! :)]]
+Got published data!, [pmessage, F*, FeeFooFiiFum, Hello! :)]

testing/btest/scripts/base/protocols/redis/attributes.zeek

@@ -0,0 +1,19 @@
+# @TEST-DOC: Test Redis protocol handling with replies with attributes
+# @TEST-REQUIRES: have-spicy
+#
+# @TEST-EXEC: zeek -b -r $TRACES/redis/attr.pcap %INPUT >output
+# @TEST-EXEC: btest-diff output
+# @TEST-EXEC: btest-diff redis.log
+
+# IMPORTANT: The test data was made synthetically, since real commands that
+# return attributes may be version-specific. Real traffic would be better.
+
+@load base/protocols/redis
+
+event Redis::reply(c: connection, data: Redis::ReplyData)
+	{
+	if ( ! data?$attributes )
+		print "Got data:", data$value;
+	else
+		print "Got data:", data$value, "with attributes:", data$attributes;
+	}

testing/btest/scripts/base/protocols/redis/pubsub-resp3.zeek

@@ -12,5 +12,5 @@
 event Redis::server_push(c: connection, data: Redis::ReplyData)
 	{
 	# The first 2 are SUBSCRIBE replies, the other 3 are message and pmessage
-	print "Got published data!", data;
+	print "Got published data!", data$value;
 	}

testing/btest/scripts/base/protocols/redis/pubsub.zeek

@@ -12,5 +12,5 @@
 
 event Redis::server_push(c: connection, data: Redis::ReplyData)
 	{
-	print "Got published data!", data;
+	print "Got published data!", data$value;
 	}