diff --git a/scripts/base/protocols/redis/__load__.zeek b/scripts/base/protocols/redis/__load__.zeek new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/protocols/redis/__load__.zeek @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/protocols/redis/main.zeek b/scripts/base/protocols/redis/main.zeek new file mode 100644 index 0000000000..f71044b373 --- /dev/null +++ b/scripts/base/protocols/redis/main.zeek @@ -0,0 +1,97 @@ +@load base/protocols/conn/removal-hooks + +module RESP; + +export { + ## Log stream identifier. + redef enum Log::ID += { LOG }; + + ## The ports to register RESP for. + const ports = { + 6379/tcp, + } &redef; + + type RESPData: record { + simple_string: string &optional &log; + simple_error: string &optional &log; + i: int &optional &log; + bulk_string: string &optional &log; + #array: + is_null: bool &log; + boolean: bool &optional &log; + double_: double &optional &log; + big_num: string &optional &log; + bulk_error: string &optional &log; + verbatim_string: string &optional &log; + }; + + ## Record type containing the column fields of the RESP log. + type Info: record { + ## Timestamp for when the activity happened. + ts: time &log; + ## Unique ID for the connection. + uid: string &log; + ## The connection's 4-tuple of endpoint addresses/ports. + id: conn_id &log; + resp_data: RESPData &log; + }; + + ## A default logging policy hook for the stream. + global log_policy: Log::PolicyHook; + + ## Default hook into RESP logging. + global log_resp: event(rec: Info); +} + +redef record connection += { + redis_resp: Info &optional; +}; + +redef likely_server_ports += { ports }; + +# TODO: If you're going to send file data into the file analysis framework, you +# need to provide a file handle function. This is a simple example that's +# sufficient if the protocol only transfers a single, complete file at a time. +# +# function get_file_handle(c: connection, is_orig: bool): string +# { +# return cat(Analyzer::ANALYZER_SPICY_RESP, c$start_time, c$id, is_orig); +# } + +event zeek_init() &priority=5 + { + Log::create_stream(RESP::LOG, [$columns=Info, $ev=log_resp, $path="resp", $policy=log_policy]); + + Analyzer::register_for_ports(Analyzer::ANALYZER_SPICY_RESP, ports); + + # TODO: To activate the file handle function above, uncomment this. + # Files::register_protocol(Analyzer::ANALYZER_SPICY_RESP, [$get_file_handle=RESP::get_file_handle ]); + } + +# Initialize logging state. +hook set_session(c: connection) + { + if ( c?$redis_resp ) + return; + + c$redis_resp = Info($ts=network_time(), $uid=c$uid, $id=c$id); + } + +function emit_log(c: connection) + { + if ( ! c?$redis_resp ) + return; + + Log::write(RESP::LOG, c$redis_resp); + delete c$redis_resp; + } + +# Example event defined in resp.evt. +event RESP::data(c: connection, payload: RESPData) + { + hook set_session(c); + + local info = c$redis_resp; + info$resp_data = payload; + emit_log(c); + } diff --git a/src/analyzer/protocol/redis/CMakeLists.txt b/src/analyzer/protocol/redis/CMakeLists.txt new file mode 100644 index 0000000000..e2fb1ab53c --- /dev/null +++ b/src/analyzer/protocol/redis/CMakeLists.txt @@ -0,0 +1,6 @@ +spicy_add_analyzer( + NAME RESP + PACKAGE_NAME spicy-resp + SOURCES resp.spicy resp.evt zeek_resp.spicy + SCRIPTS __load__.zeek main.zeek +) diff --git a/src/analyzer/protocol/redis/resp.evt b/src/analyzer/protocol/redis/resp.evt new file mode 100644 index 0000000000..25cd1ebdd1 --- /dev/null +++ b/src/analyzer/protocol/redis/resp.evt @@ -0,0 +1,8 @@ +protocol analyzer spicy::RESP over TCP: + parse with RESP::Messages, + port 6379/tcp; + +import RESP; +import Zeek_RESP; + +on RESP::Data -> event RESP::data($conn, Zeek_RESP::create_data(self)); diff --git a/src/analyzer/protocol/redis/resp.spicy b/src/analyzer/protocol/redis/resp.spicy new file mode 100644 index 0000000000..f661a8e0bb --- /dev/null +++ b/src/analyzer/protocol/redis/resp.spicy @@ -0,0 +1,107 @@ +module RESP; + +public type Messages = unit { + : Data[]; +}; + +public type Data = unit { + ty: uint8 &convert=DataType($$); + switch ( self.ty ) { + DataType::SIMPLE_STRING -> simple_string: SimpleString(False); + DataType::SIMPLE_ERROR -> simple_error: SimpleString(True); + DataType::INTEGER -> integer: Integer; + DataType::BULK_STRING -> bulk_string: BulkString(False); + DataType::ARRAY -> array: Array; + DataType::NULL -> null: Null_; + DataType::BOOLEAN -> boolean: Boolean; + DataType::DOUBLE -> double: Double; + DataType::BIG_NUM -> big_num: BigNum; + DataType::BULK_ERROR -> bulk_error: BulkString(True); + # This can be a different type, but the docs also say: + # "Some client libraries may ignore the difference between this type and the string type" + # It just includes the encoding first in the content + DataType::VERBATIM_STRING -> verbatim_string: BulkString(False); + DataType::MAP -> map_: Map; + DataType::SET -> set_: Set; + # "Push events are encoded similarly to arrays, differing only in their + # first byte" - TODO: can probably make it more obvious, though + DataType::PUSH -> push: Array; + }; +}; + +type DataType = enum { + SIMPLE_STRING = '+', + SIMPLE_ERROR = '-', + INTEGER = ':', + BULK_STRING = '$', + ARRAY = '*', + NULL = '_', + BOOLEAN = '#', + DOUBLE = ',', + BIG_NUM = '(', + BULK_ERROR = '!', + VERBATIM_STRING = '=', + MAP = '%', + SET = '~', + PUSH = '>' +}; + +type SimpleString = unit(is_error: bool) { + content: bytes &until=b"\x0d\x0a"; +}; + +type Integer = unit { + int: bytes &convert=$$.to_int(10) &until=b"\x0d\x0a"; +}; + +type BulkString = unit(is_error: bool) { + length: bytes &convert=$$.to_int(10) &until=b"\x0d\x0a"; + # NullBulkString is a BulkString with content unset + content: bytes &size=uint64( self.length ) if ( self.length >= 0 ); + + # Consume last CLRF + : bytes &until=b"\x0d\x0a"; +}; + +type Array = unit { + num_elements: bytes &convert=$$.to_int(10) &until=b"\x0d\x0a"; + # Null array is an array with elements unset. This is different from an empty array + elements: Data[uint64(self.num_elements)] if ( self.num_elements >= 0 ); +}; + +type Null_ = unit { + # Still must consume CLRF + : bytes &until=b"\x0d\x0a"; +}; + +type Boolean = unit { + val: uint8 &convert=$$ == 't'; + : bytes &until=b"\x0d\x0a"; +}; + +type Double = unit { + val: bytes &convert=$$.to_real() &until=b"\x0d\x0a"; +}; + +type BigNum = unit { + # Big num can be very big so put it in a UTF-8 decoded string + val: bytes &convert=$$.decode() &until=b"\x0d\x0a"; +}; + +type Map = unit { + var key_val_pairs: vector>; + num_elements: bytes &convert=$$.to_uint(10) &until=b"\x0d\x0a"; + # TODO: How can I make this into a map? Alternatively, how can I do this better? + raw_data: Data[self.num_elements*2] { + while ( local i = 0; i < self.num_elements ) { + self.key_val_pairs.push_back(($$[i], $$[i+1])); + i += 2; + } + } +}; + +type Set = unit { + num_elements: bytes &convert=$$.to_uint(10) &until=b"\x0d\x0a"; + # TODO: This should be a set but doesn't go in the backed C++ set + elements: Data[self.num_elements]; +}; diff --git a/src/analyzer/protocol/redis/zeek_resp.spicy b/src/analyzer/protocol/redis/zeek_resp.spicy new file mode 100644 index 0000000000..89a163daa9 --- /dev/null +++ b/src/analyzer/protocol/redis/zeek_resp.spicy @@ -0,0 +1,72 @@ +# Set up protocol confirmation/rejection for analyzers, as well as any further +# Zeek-specific analysis. + +module Zeek_RESP; + +import RESP; +import zeek; + +# Any error bubbling up to the top unit will trigger a protocol rejection. +on RESP::Data::%done { + zeek::confirm_protocol(); +} + +on RESP::Data::%error { + zeek::reject_protocol("error while parsing RESP data"); +} + +type ZeekData = tuple< + optional, + optional, + optional, + optional, + #optional>, # TODO: This segfaults because recursive type :( + bool, + optional, + optional, + optional, + optional, + optional, +>; + +public function create_data(data: RESP::Data): ZeekData { + local simple_string: optional; + local simple_error: optional; + local i: optional; + local bulk_string: optional; + #local array: optional>; + local null: bool; + local boolean: optional; + local double: optional; + local big_num: optional; + local bulk_error: optional; + local verbatim_string: optional; + if (data?.simple_string) + simple_string = data.simple_string.content; + if (data?.simple_error) + simple_error = data.simple_error.content; + if (data?.integer) + i = data.integer.int; + if (data?.bulk_string) + bulk_string = data.bulk_string.content; + #if (data?.array) { + # for ( data in data.array.elements ) { + # array.push_back(data); + # } + #} + if (data?.null) + null = True; + else + null = False; + if (data?.boolean) + boolean = data.boolean.val; + if (data?.double) + double = data.double.val; + if (data?.big_num) + big_num = data.big_num.val; + if (data?.bulk_error) + bulk_error = data.bulk_error.content; + if (data?.verbatim_string) + verbatim_string = data.verbatim_string.content; + return (simple_string, simple_error, i, bulk_string, null, boolean, double, big_num, bulk_error, verbatim_string); +} diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.run-pcap/conn.log.filtered b/testing/btest/Baseline/scripts.base.protocols.redis.run-pcap/conn.log.filtered new file mode 100644 index 0000000000..1e6917b8c7 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.run-pcap/conn.log.filtered @@ -0,0 +1,2 @@ +ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 10.1.9.63 63526 54.175.222.246 80 tcp http 0.755677 207 489 SF 0 ShADTadFf 7 790 4 705 - diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.run-pcap/output b/testing/btest/Baseline/scripts.base.protocols.redis.run-pcap/output new file mode 100644 index 0000000000..781c9fcbd9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.run-pcap/output @@ -0,0 +1,2 @@ +Hello world! +Goodbye world! diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.standalone/output b/testing/btest/Baseline/scripts.base.protocols.redis.standalone/output new file mode 100644 index 0000000000..6a6e880e2d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.standalone/output @@ -0,0 +1,25 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +RESP::Data { + ty: SIMPLE_STRING + simple_string: RESP::SimpleString { + content: OK + } +} +RESP::Data { + ty: INTEGER + integer: RESP::Integer { + int_: 1000 + } +} +RESP::Data { + ty: INTEGER + integer: RESP::Integer { + int_: -1000 + } +} +RESP::Data { + ty: INTEGER + integer: RESP::Integer { + int_: 1000 + } +} diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.trace/output b/testing/btest/Baseline/scripts.base.protocols.redis.trace/output new file mode 100644 index 0000000000..a3c0b852bb --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.trace/output @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=SET, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=hi:2, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=2, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=OK, simple_error=, i=, bulk_string=, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=SET, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=hi:3, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=sup, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=OK, simple_error=, i=, bulk_string=, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=GET, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=hi:3, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] +Testing RESP: [simple_string=, simple_error=, i=, bulk_string=sup, is_null=F, boolean=, double_=, big_num=, bulk_error=, verbatim_string=] diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.trace/resp.log b/testing/btest/Baseline/scripts.base.protocols.redis.trace/resp.log new file mode 100644 index 0000000000..85c62c2387 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.trace/resp.log @@ -0,0 +1,24 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path resp +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p resp_data.simple_string resp_data.simple_error resp_data.i resp_data.bulk_string resp_data.is_null resp_data.boolean resp_data.double_ resp_data.big_num resp_data.bulk_error resp_data.verbatim_string +#types time string addr port addr port string string int string bool bool double string string string +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - SET F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - hi:2 F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - 2 F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - - F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 OK - - - F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - SET F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - hi:3 F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - sup F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - - F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 OK - - - F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - GET F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - hi:3 F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - - F - - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 127.0.0.1 58972 127.0.0.1 6379 - - - sup F - - - - - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Traces/redis/loop-redis.trace b/testing/btest/Traces/redis/loop-redis.trace new file mode 100644 index 0000000000..4ce10fdf55 Binary files /dev/null and b/testing/btest/Traces/redis/loop-redis.trace differ diff --git a/testing/btest/scripts/base/protocols/redis/availability.zeek b/testing/btest/scripts/base/protocols/redis/availability.zeek new file mode 100644 index 0000000000..3f0f3e20b9 --- /dev/null +++ b/testing/btest/scripts/base/protocols/redis/availability.zeek @@ -0,0 +1,3 @@ +# @TEST-DOC: Check that the RESP analyzer is available. +# +# @TEST-EXEC: zeek -NN | grep -Eqi 'ANALYZER_SPICY_RESP' diff --git a/testing/btest/scripts/base/protocols/redis/standalone.spicy b/testing/btest/scripts/base/protocols/redis/standalone.spicy new file mode 100644 index 0000000000..6d99320b39 --- /dev/null +++ b/testing/btest/scripts/base/protocols/redis/standalone.spicy @@ -0,0 +1,11 @@ +# @TEST-DOC: Test parsing behavior of RESP. +# +# @TEST-EXEC: spicyc ${DIST}/analyzer/resp.spicy -j -d -o resp.hlto +# +# TODO: A lot of tests are possible from the docs and having them would be nice. +# But, a lot of characters ($, -, etc.) cause problems with TEST_EXEC. ugh. +# @TEST-EXEC: printf "+OK\x0d\x0a" | spicy-dump -p RESP::Data resp.hlto >>output 2>&1 +# @TEST-EXEC: printf ":1000\x0d\x0a" | spicy-dump -p RESP::Data resp.hlto >>output 2>&1 +# @TEST-EXEC: printf ":-1000\x0d\x0a" | spicy-dump -p RESP::Data resp.hlto >>output 2>&1 +# @TEST-EXEC: printf ":+1000\x0d\x0a" | spicy-dump -p RESP::Data resp.hlto >>output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output diff --git a/testing/btest/scripts/base/protocols/redis/trace.zeek b/testing/btest/scripts/base/protocols/redis/trace.zeek new file mode 100644 index 0000000000..bc20102b6c --- /dev/null +++ b/testing/btest/scripts/base/protocols/redis/trace.zeek @@ -0,0 +1,10 @@ +# @TEST-DOC: Test Zeek parsing a trace file through the RESP analyzer. +# +# @TEST-EXEC: zeek -Cr $TRACES/redis/loop-redis.trace base/protocols/redis %INPUT >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff resp.log + +event RESP::data(c: connection, payload: RESP::RESPData) + { + print fmt("Testing RESP: %s", payload); + }