diff --git a/scripts/base/protocols/redis/main.zeek b/scripts/base/protocols/redis/main.zeek index e43b4ce322..4d11aede9f 100644 --- a/scripts/base/protocols/redis/main.zeek +++ b/scripts/base/protocols/redis/main.zeek @@ -76,36 +76,30 @@ export { current_request: count &default=0; ## Current response in the pending queue. current_response: count &default=0; + ## Ranges where we do not expect a response + ## Each range is one or two elements, one meaning it's unbounded, two meaning + ## it begins at one and ends at the second. + no_response_ranges: vector of vector of count; }; + # Redis specifically mentions 10k commands as a good pipelining threshold, so + # we'll piggyback on that. + option max_pending_requests = 10000; } redef record connection += { - # TODO: Rename - redis_resp: Info &optional; + redis: Info &optional; redis_state: State &optional; }; redef likely_server_ports += { ports }; -# TODO: If you're going to send file data into the file analysis framework, you -# need to provide a file handle function. This is a simple example that's -# sufficient if the protocol only transfers a single, complete file at a time. -# -# function get_file_handle(c: connection, is_orig: bool): string -# { -# return cat(Analyzer::ANALYZER_SPICY_REDIS, c$start_time, c$id, is_orig); -# } - event zeek_init() &priority=5 { - Log::create_stream(Redis::LOG, [ $columns=Info, $ev=log_resp, $path="resp", + Log::create_stream(Redis::LOG, [ $columns=Info, $ev=log_resp, $path="redis", $policy=log_policy ]); Analyzer::register_for_ports(Analyzer::ANALYZER_SPICY_REDIS, ports); - - # TODO: To activate the file handle function above, uncomment this. - # Files::register_protocol(Analyzer::ANALYZER_SPICY_REDIS, [$get_file_handle=Redis::get_file_handle ]); } function new_redis_session(c: connection): Info @@ -113,66 +107,140 @@ function new_redis_session(c: connection): Info return Info($ts=network_time(), $uid=c$uid, $id=c$id); } +function make_new_state(c: connection) + { + local s: State; + c$redis_state = s; + Conn::register_removal_hook(c, finalize_redis); + } + function set_state(c: connection, is_orig: bool) { - if ( ! c?$redis_state ) - { - local s: State; - c$redis_state = s; - Conn::register_removal_hook(c, finalize_redis); - } + if ( ! c?$redis_state ) make_new_state(c); - if ( is_orig ) - { - if ( c$redis_state$current_request !in c$redis_state$pending ) - c$redis_state$pending[c$redis_state$current_request] = new_redis_session(c); + local current: count; + if ( is_orig ) current = c$redis_state$current_request; + else current = c$redis_state$current_response; - c$redis_resp = c$redis_state$pending[c$redis_state$current_request]; - } - else - { - if ( c$redis_state$current_response !in c$redis_state$pending ) - c$redis_state$pending[c$redis_state$current_response] = new_redis_session(c); + if ( current !in c$redis_state$pending ) + c$redis_state$pending[current] = new_redis_session(c); - c$redis_resp = c$redis_state$pending[c$redis_state$current_response]; - } + c$redis = c$redis_state$pending[current]; + } + +# Returns true if the last interval exists and is closed +function is_last_interval_closed(c: connection): bool + { + return |c$redis_state$no_response_ranges| == 0 || |c$redis_state$no_response_ranges[|c$redis_state$no_response_ranges| - 1]| != 1; } event Redis::command(c: connection, is_orig: bool, command: Command) { - #hook set_session(c, command); + if ( ! c?$redis_state ) make_new_state(c); - # TODO: We need to care about whether the reply was suppressed with - # CLIENT REPLY [OFF|SKIP] - #local info = c$redis_resp; - #emit_log(c); - # TODO refactor this since it's used a couple times - if ( ! c?$redis_state ) + if ( max_pending_requests > 0 && |c$redis_state$pending| > max_pending_requests ) { - local s: State; - c$redis_state = s; - Conn::register_removal_hook(c, finalize_redis); + Reporter::conn_weird("Redis_excessive_pipelining", c); + + # Just spit out what we have + while ( c$redis_state$current_response < c$redis_state$current_request ) + { + local cr = c$redis_state$current_response; + if ( cr in c$redis_state$pending ) + { + Log::write(Redis::LOG, c$redis_state$pending[cr]); + delete c$redis_state$pending[cr]; + } + ++c$redis_state$current_response; + } } + ++c$redis_state$current_request; + if ( command?$known && command$known == KnownCommand_CLIENT ) + { + # All 3 CLIENT commands we care about have 3 elements + if ( |command$raw| == 3 ) + { + if ( to_lower(command$raw[2]) == "on" ) + { + # If the last range is open, close it here. Otherwise, noop + if ( |c$redis_state$no_response_ranges| > 0 ) + { + local range = c$redis_state$no_response_ranges[|c$redis_state$no_response_ranges| - 1]; + if ( |range| == 1 ) + { + range += c$redis_state$current_request; + } + } + } + if ( to_lower(command$raw[2]) == "off" ) + { + # Only add a new interval if the last one is closed + if ( is_last_interval_closed(c) ) + { + c$redis_state$no_response_ranges += vector(c$redis_state$current_request); + } + } + if ( to_lower(command$raw[2]) == "skip" ) + { + if ( is_last_interval_closed(c) ) + # It skips this one and the next one + c$redis_state$no_response_ranges += vector(c$redis_state$current_request, c$redis_state$current_request + 2); + } + } + } set_state(c, T); - c$redis_resp$cmd = command; + c$redis$cmd = command; + } + +## Gets the next response number based on a connection. This is necessary since +## some responses may have been skipped. +function response_num(c: connection): count + { + local resp_num = c$redis_state$current_response + 1; + for ( i in c$redis_state$no_response_ranges ) + { + local range = c$redis_state$no_response_ranges[i]; + assert |range| >= 1; + if ( |range| == 1 && resp_num > range[0] ) + {} # TODO: This is necessary if not using pipelining + if ( |range| == 2 && resp_num >= range[0] && resp_num < range[1] ) + return range[1]; + } + + # Default: no disable/enable shenanigans + return resp_num; } event Redis::server_data(c: connection, is_orig: bool, data: ServerData) { - if ( ! c?$redis_state ) - { - local s: State; - c$redis_state = s; - Conn::register_removal_hook(c, finalize_redis); - } - ++c$redis_state$current_response; + if ( ! c?$redis_state ) make_new_state(c); + + local previous_response_num = c$redis_state$current_response; + c$redis_state$current_response = response_num(c); set_state(c, F); - c$redis_resp$response = data; - # TODO: Do stuff with pending so that finalize_redis and pipelining work - Log::write(Redis::LOG, c$redis_resp); + c$redis$response = data; + # Log each of the pending responses to this point - we will not go + # back. + while ( previous_response_num < c$redis_state$current_response ) + { + if ( previous_response_num == 0 ) + { + ++previous_response_num; + next; + } + + if ( previous_response_num in c$redis_state$pending ) + { + Log::write(Redis::LOG, c$redis_state$pending[previous_response_num]); + delete c$redis_state$pending[previous_response_num]; + } + previous_response_num += 1; + } + # Log this one + Log::write(Redis::LOG, c$redis); delete c$redis_state$pending[c$redis_state$current_response]; } @@ -185,10 +253,7 @@ hook finalize_redis(c: connection) { # We don't use pending elements at index 0. if ( r == 0 ) next; - #Log::write(HTTP::LOG, info); Log::write(Redis::LOG, info); - #delete c$redis_resp; } } } - diff --git a/src/analyzer/protocol/redis/CMakeLists.txt b/src/analyzer/protocol/redis/CMakeLists.txt index 0cff4d5709..6f28f11baf 100644 --- a/src/analyzer/protocol/redis/CMakeLists.txt +++ b/src/analyzer/protocol/redis/CMakeLists.txt @@ -1,6 +1,6 @@ spicy_add_analyzer( - NAME RESP - PACKAGE_NAME spicy-resp + NAME Redis + PACKAGE_NAME spicy-redis SOURCES resp.spicy resp.evt redis.spicy zeek_redis.spicy SCRIPTS __load__.zeek main.zeek ) diff --git a/src/analyzer/protocol/redis/redis.spicy b/src/analyzer/protocol/redis/redis.spicy index 66b99b773c..b782380cd1 100644 --- a/src/analyzer/protocol/redis/redis.spicy +++ b/src/analyzer/protocol/redis/redis.spicy @@ -14,6 +14,7 @@ public type KnownCommand = enum { BLMPOP, BLPOP, BRPOP, + CLIENT, COPY, DECR, DECRBY, @@ -241,6 +242,7 @@ function command_from(cmd_bytes: bytes): optional { case b"blmpop": cmd = KnownCommand::BLMPOP; case b"blpop": cmd = KnownCommand::BLPOP; case b"brpop": cmd = KnownCommand::BRPOP; + case b"client": cmd = KnownCommand::CLIENT; case b"copy": cmd = KnownCommand::COPY; case b"decr": cmd = KnownCommand::DECR; case b"decrby": cmd = KnownCommand::DECRBY; diff --git a/src/analyzer/protocol/redis/resp.spicy b/src/analyzer/protocol/redis/resp.spicy index 4588393a3e..7302f4103f 100644 --- a/src/analyzer/protocol/redis/resp.spicy +++ b/src/analyzer/protocol/redis/resp.spicy @@ -8,10 +8,6 @@ import spicy; # exhausting main memory. const MAX_SIZE = 1024 * 1024; -public type Messages = unit { - : (Data &synchronize)[]; -}; - public type ClientMessages = unit { : (ClientData &synchronize)[]; }; diff --git a/src/analyzer/protocol/redis/zeek_redis.spicy b/src/analyzer/protocol/redis/zeek_redis.spicy index a2288c5f1c..5b30717525 100644 --- a/src/analyzer/protocol/redis/zeek_redis.spicy +++ b/src/analyzer/protocol/redis/zeek_redis.spicy @@ -11,7 +11,6 @@ public type ZeekServerData = struct { public function make_server_data(data: RESP::ServerData): ZeekServerData { local res: ZeekServerData = [$err = False, $data = Null]; - # TODO: Redo this so it's not ugly and supports more. maybe if (data.data?.simple_error) { res.err = True; res.data = data.data.simple_error.content; diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.client-reply-off-2conn/redis.log b/testing/btest/Baseline/scripts.base.protocols.redis.client-reply-off-2conn/redis.log new file mode 100644 index 0000000000..859d98f6ad --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.client-reply-off-2conn/redis.log @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path redis +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data +#types time string addr port addr port string string string bool string +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 61211 ::1 6379 PING - - F PONG +XXXXXXXXXX.XXXXXX CtPZjS20MLrsMUOJi2 ::1 61212 ::1 6379 PING - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 61211 ::1 6379 CLIENT - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 61211 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX CtPZjS20MLrsMUOJi2 ::1 61212 ::1 6379 CLIENT - - - - +XXXXXXXXXX.XXXXXX CtPZjS20MLrsMUOJi2 ::1 61212 ::1 6379 PING - - - - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.client-reply-off/redis.log b/testing/btest/Baseline/scripts.base.protocols.redis.client-reply-off/redis.log new file mode 100644 index 0000000000..23edcacbc1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.client-reply-off/redis.log @@ -0,0 +1,18 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path redis +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data +#types time string addr port addr port string string string bool string +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 60761 ::1 6379 PING - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 60761 ::1 6379 CLIENT - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 60761 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 60761 ::1 6379 CLIENT - - F OK +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 60761 ::1 6379 PING - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 60761 ::1 6379 CLIENT - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 60761 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 60761 ::1 6379 PING - - F PONG +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.client-skip-while-off/redis.log b/testing/btest/Baseline/scripts.base.protocols.redis.client-skip-while-off/redis.log new file mode 100644 index 0000000000..248e739b85 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.client-skip-while-off/redis.log @@ -0,0 +1,17 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path redis +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data +#types time string addr port addr port string string string bool string +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc ::1 56348 ::1 6379 PING - - F PONG +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc ::1 56348 ::1 6379 CLIENT - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc ::1 56348 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc ::1 56348 ::1 6379 CLIENT - - - - +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc ::1 56348 ::1 6379 CLIENT - - F OK +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc ::1 56348 ::1 6379 PING - - F PONG +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc ::1 56348 ::1 6379 PING - - F PONG +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.django-cloud/resp.log b/testing/btest/Baseline/scripts.base.protocols.redis.django-cloud/redis.log similarity index 99% rename from testing/btest/Baseline/scripts.base.protocols.redis.django-cloud/resp.log rename to testing/btest/Baseline/scripts.base.protocols.redis.django-cloud/redis.log index 9a79080e56..d6bfebaf43 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.django-cloud/resp.log +++ b/testing/btest/Baseline/scripts.base.protocols.redis.django-cloud/redis.log @@ -3,7 +3,7 @@ #set_separator , #empty_field (empty) #unset_field - -#path resp +#path redis #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data #types time string addr port addr port string string string bool string diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.django/resp.log b/testing/btest/Baseline/scripts.base.protocols.redis.django/redis.log similarity index 99% rename from testing/btest/Baseline/scripts.base.protocols.redis.django/resp.log rename to testing/btest/Baseline/scripts.base.protocols.redis.django/redis.log index 7d82bff464..a05f819818 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.django/resp.log +++ b/testing/btest/Baseline/scripts.base.protocols.redis.django/redis.log @@ -3,7 +3,7 @@ #set_separator , #empty_field (empty) #unset_field - -#path resp +#path redis #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data #types time string addr port addr port string string string bool string diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.excessive-pipelining/redis.log b/testing/btest/Baseline/scripts.base.protocols.redis.excessive-pipelining/redis.log new file mode 100644 index 0000000000..e5c9e70dbb --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.excessive-pipelining/redis.log @@ -0,0 +1,33 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path redis +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data +#types time string addr port addr port string string string bool string +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - - - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 PING - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 - - - F PONG +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.excessive-pipelining/weird.log b/testing/btest/Baseline/scripts.base.protocols.redis.excessive-pipelining/weird.log new file mode 100644 index 0000000000..df63cb7b93 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.redis.excessive-pipelining/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h ::1 57156 ::1 6379 Redis_excessive_pipelining - F zeek - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.pipeline-with-quotes/resp.log b/testing/btest/Baseline/scripts.base.protocols.redis.pipeline-with-quotes/redis.log similarity index 99% rename from testing/btest/Baseline/scripts.base.protocols.redis.pipeline-with-quotes/resp.log rename to testing/btest/Baseline/scripts.base.protocols.redis.pipeline-with-quotes/redis.log index a600907625..46356a134d 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.pipeline-with-quotes/resp.log +++ b/testing/btest/Baseline/scripts.base.protocols.redis.pipeline-with-quotes/redis.log @@ -3,7 +3,7 @@ #set_separator , #empty_field (empty) #unset_field - -#path resp +#path redis #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data #types time string addr port addr port string string string bool string diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.pipelined-with-commands/resp.log b/testing/btest/Baseline/scripts.base.protocols.redis.pipelined-with-commands/redis.log similarity index 98% rename from testing/btest/Baseline/scripts.base.protocols.redis.pipelined-with-commands/resp.log rename to testing/btest/Baseline/scripts.base.protocols.redis.pipelined-with-commands/redis.log index 31f207bf96..4228f180da 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.pipelined-with-commands/resp.log +++ b/testing/btest/Baseline/scripts.base.protocols.redis.pipelined-with-commands/redis.log @@ -3,7 +3,7 @@ #set_separator , #empty_field (empty) #unset_field - -#path resp +#path redis #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data #types time string addr port addr port string string string bool string diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.pipelined/resp.log b/testing/btest/Baseline/scripts.base.protocols.redis.pipelined/redis.log similarity index 98% rename from testing/btest/Baseline/scripts.base.protocols.redis.pipelined/resp.log rename to testing/btest/Baseline/scripts.base.protocols.redis.pipelined/redis.log index 59744f461b..311117484f 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.pipelined/resp.log +++ b/testing/btest/Baseline/scripts.base.protocols.redis.pipelined/redis.log @@ -3,7 +3,7 @@ #set_separator , #empty_field (empty) #unset_field - -#path resp +#path redis #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data #types time string addr port addr port string string string bool string diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/resp.log b/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/redis.log similarity index 98% rename from testing/btest/Baseline/scripts.base.protocols.redis.pubsub/resp.log rename to testing/btest/Baseline/scripts.base.protocols.redis.pubsub/redis.log index b242952a83..80ca87d962 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/resp.log +++ b/testing/btest/Baseline/scripts.base.protocols.redis.pubsub/redis.log @@ -3,7 +3,7 @@ #set_separator , #empty_field (empty) #unset_field - -#path resp +#path redis #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data #types time string addr port addr port string string string bool string diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.stream/resp.log b/testing/btest/Baseline/scripts.base.protocols.redis.stream/redis.log similarity index 98% rename from testing/btest/Baseline/scripts.base.protocols.redis.stream/resp.log rename to testing/btest/Baseline/scripts.base.protocols.redis.stream/redis.log index 2afa293a3b..3b47ddf394 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.stream/resp.log +++ b/testing/btest/Baseline/scripts.base.protocols.redis.stream/redis.log @@ -3,7 +3,7 @@ #set_separator , #empty_field (empty) #unset_field - -#path resp +#path redis #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data #types time string addr port addr port string string string bool string diff --git a/testing/btest/Baseline/scripts.base.protocols.redis.trace/resp.log b/testing/btest/Baseline/scripts.base.protocols.redis.trace/redis.log similarity index 98% rename from testing/btest/Baseline/scripts.base.protocols.redis.trace/resp.log rename to testing/btest/Baseline/scripts.base.protocols.redis.trace/redis.log index 14300550eb..4c929cacd2 100644 --- a/testing/btest/Baseline/scripts.base.protocols.redis.trace/resp.log +++ b/testing/btest/Baseline/scripts.base.protocols.redis.trace/redis.log @@ -3,7 +3,7 @@ #set_separator , #empty_field (empty) #unset_field - -#path resp +#path redis #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p cmd.command cmd.key cmd.value response.err response.data #types time string addr port addr port string string string bool string diff --git a/testing/btest/Traces/redis/client-skip-while-off.trace b/testing/btest/Traces/redis/client-skip-while-off.trace new file mode 100644 index 0000000000..ef40431fef Binary files /dev/null and b/testing/btest/Traces/redis/client-skip-while-off.trace differ diff --git a/testing/btest/Traces/redis/excessive-pipelining.trace b/testing/btest/Traces/redis/excessive-pipelining.trace new file mode 100644 index 0000000000..04fcfd76af Binary files /dev/null and b/testing/btest/Traces/redis/excessive-pipelining.trace differ diff --git a/testing/btest/Traces/redis/reply-off-on-2conn.trace b/testing/btest/Traces/redis/reply-off-on-2conn.trace new file mode 100644 index 0000000000..0703e5f1c5 Binary files /dev/null and b/testing/btest/Traces/redis/reply-off-on-2conn.trace differ diff --git a/testing/btest/Traces/redis/reply-off-on.trace b/testing/btest/Traces/redis/reply-off-on.trace new file mode 100644 index 0000000000..befddf9dbc Binary files /dev/null and b/testing/btest/Traces/redis/reply-off-on.trace differ diff --git a/testing/btest/scripts/base/protocols/redis/client-reply-off-2conn.zeek b/testing/btest/scripts/base/protocols/redis/client-reply-off-2conn.zeek new file mode 100644 index 0000000000..6a707fc7d7 --- /dev/null +++ b/testing/btest/scripts/base/protocols/redis/client-reply-off-2conn.zeek @@ -0,0 +1,5 @@ +# @TEST-DOC: Test CLIENT REPLY OFF, but turns on with new connection +# +# @TEST-EXEC: zeek -Cr $TRACES/redis/reply-off-on-2conn.trace base/protocols/redis %INPUT >output +# @TEST-EXEC: btest-diff redis.log + diff --git a/testing/btest/scripts/base/protocols/redis/client-reply-off.zeek b/testing/btest/scripts/base/protocols/redis/client-reply-off.zeek new file mode 100644 index 0000000000..d977272279 --- /dev/null +++ b/testing/btest/scripts/base/protocols/redis/client-reply-off.zeek @@ -0,0 +1,5 @@ +# @TEST-DOC: Test CLIENT REPLY OFF then ON again and a SKIP +# +# @TEST-EXEC: zeek -Cr $TRACES/redis/reply-off-on.trace base/protocols/redis %INPUT >output +# @TEST-EXEC: btest-diff redis.log + diff --git a/testing/btest/scripts/base/protocols/redis/client-skip-while-off.zeek b/testing/btest/scripts/base/protocols/redis/client-skip-while-off.zeek new file mode 100644 index 0000000000..eeed61422b --- /dev/null +++ b/testing/btest/scripts/base/protocols/redis/client-skip-while-off.zeek @@ -0,0 +1,5 @@ +# @TEST-DOC: Test CLIENT REPLY OFF then ON again and a SKIP +# +# @TEST-EXEC: zeek -Cr $TRACES/redis/client-skip-while-off.trace base/protocols/redis %INPUT >output +# @TEST-EXEC: btest-diff redis.log + diff --git a/testing/btest/scripts/base/protocols/redis/django-cloud.zeek b/testing/btest/scripts/base/protocols/redis/django-cloud.zeek index f4c1983aca..2aef19e4e8 100644 --- a/testing/btest/scripts/base/protocols/redis/django-cloud.zeek +++ b/testing/btest/scripts/base/protocols/redis/django-cloud.zeek @@ -2,7 +2,7 @@ # # @TEST-EXEC: zeek -Cr $TRACES/redis/django-cloud.trace base/protocols/redis %INPUT >output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: btest-diff resp.log +# @TEST-EXEC: btest-diff redis.log redef Redis::ports += { 10625/tcp, diff --git a/testing/btest/scripts/base/protocols/redis/django.zeek b/testing/btest/scripts/base/protocols/redis/django.zeek index fecdde13cc..0831e8e3a3 100644 --- a/testing/btest/scripts/base/protocols/redis/django.zeek +++ b/testing/btest/scripts/base/protocols/redis/django.zeek @@ -2,7 +2,7 @@ # # @TEST-EXEC: zeek -Cr $TRACES/redis/django-cache.trace base/protocols/redis %INPUT >output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: btest-diff resp.log +# @TEST-EXEC: btest-diff redis.log event Redis::set_command(c: connection, is_orig: bool, command: Redis::SetCommand) { diff --git a/testing/btest/scripts/base/protocols/redis/excessive-pipelining.zeek b/testing/btest/scripts/base/protocols/redis/excessive-pipelining.zeek new file mode 100644 index 0000000000..7f3ff9ce2b --- /dev/null +++ b/testing/btest/scripts/base/protocols/redis/excessive-pipelining.zeek @@ -0,0 +1,8 @@ +# @TEST-DOC: Test Zeek parsing "pipelined" data responses +# +# @TEST-EXEC: zeek -Cr $TRACES/redis/excessive-pipelining.trace base/protocols/redis %INPUT >output +# @TEST-EXEC: btest-diff redis.log +# @TEST-EXEC: btest-diff weird.log + +# Make sure we get a weird if we go over the pipelining threshold (intentionally limited) +redef Redis::max_pending_requests = 5; diff --git a/testing/btest/scripts/base/protocols/redis/pipeline-with-quotes.zeek b/testing/btest/scripts/base/protocols/redis/pipeline-with-quotes.zeek index 972e1385c4..e57069c02d 100644 --- a/testing/btest/scripts/base/protocols/redis/pipeline-with-quotes.zeek +++ b/testing/btest/scripts/base/protocols/redis/pipeline-with-quotes.zeek @@ -2,7 +2,7 @@ # # @TEST-EXEC: zeek -Cr $TRACES/redis/pipeline-quotes.trace base/protocols/redis %INPUT >output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: btest-diff resp.log +# @TEST-EXEC: btest-diff redis.log # TODO: Make it so weird.log exists again with `zeek::weird` for inline commands # btest-diff weird.log diff --git a/testing/btest/scripts/base/protocols/redis/pipelined-with-commands.zeek b/testing/btest/scripts/base/protocols/redis/pipelined-with-commands.zeek index a2a1862bf2..23559e93b1 100644 --- a/testing/btest/scripts/base/protocols/redis/pipelined-with-commands.zeek +++ b/testing/btest/scripts/base/protocols/redis/pipelined-with-commands.zeek @@ -2,7 +2,7 @@ # # @TEST-EXEC: zeek -Cr $TRACES/redis/pipeline-with-commands.trace base/protocols/redis %INPUT >output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: btest-diff resp.log +# @TEST-EXEC: btest-diff redis.log # Sometimes commands aren't serialized, like when pipelining. This still works! So we # should handle this. This particular example has a few commands, amongst them a SET and diff --git a/testing/btest/scripts/base/protocols/redis/pipelined.zeek b/testing/btest/scripts/base/protocols/redis/pipelined.zeek index c6b113c95d..f91e4bec4e 100644 --- a/testing/btest/scripts/base/protocols/redis/pipelined.zeek +++ b/testing/btest/scripts/base/protocols/redis/pipelined.zeek @@ -2,7 +2,7 @@ # # @TEST-EXEC: zeek -Cr $TRACES/redis/pipelining-example.trace base/protocols/redis %INPUT >output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: btest-diff resp.log +# @TEST-EXEC: btest-diff redis.log # Testing the example of "pipelining" in REDIS docs: # https://redis.io/docs/latest/develop/use/pipelining/ diff --git a/testing/btest/scripts/base/protocols/redis/pubsub.zeek b/testing/btest/scripts/base/protocols/redis/pubsub.zeek index febd515135..711cb33059 100644 --- a/testing/btest/scripts/base/protocols/redis/pubsub.zeek +++ b/testing/btest/scripts/base/protocols/redis/pubsub.zeek @@ -2,7 +2,7 @@ # # @TEST-EXEC: zeek -Cr $TRACES/redis/pubsub.trace base/protocols/redis %INPUT >output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: btest-diff resp.log +# @TEST-EXEC: btest-diff redis.log # Testing the example of pub sub in REDIS docs: # https://redis.io/docs/latest/develop/interact/pubsub/ diff --git a/testing/btest/scripts/base/protocols/redis/standalone.spicy b/testing/btest/scripts/base/protocols/redis/standalone.spicy index 8512b325d7..17bd61ed15 100644 --- a/testing/btest/scripts/base/protocols/redis/standalone.spicy +++ b/testing/btest/scripts/base/protocols/redis/standalone.spicy @@ -1,11 +1,11 @@ # @TEST-DOC: Test parsing behavior of RESP. # -# @TEST-EXEC: spicyc ${DIST}/analyzer/resp.spicy ${DIST}/analyzer/redis.spicy -j -d -o resp.hlto +# @TEST-EXEC: spicyc ${DIST}/analyzer/resp.spicy ${DIST}/analyzer/redis.spicy -j -d -o redis.hlto # # TODO: A lot of tests are possible from the docs and having them would be nice. # But, a lot of characters ($, -, etc.) cause problems with TEST_EXEC. ugh. -# @TEST-EXEC: printf "+OK\x0d\x0a" | spicy-dump -p RESP::Data resp.hlto >>output 2>&1 -# @TEST-EXEC: printf ":1000\x0d\x0a" | spicy-dump -p RESP::Data resp.hlto >>output 2>&1 -# @TEST-EXEC: printf ":-1000\x0d\x0a" | spicy-dump -p RESP::Data resp.hlto >>output 2>&1 -# @TEST-EXEC: printf ":+1000\x0d\x0a" | spicy-dump -p RESP::Data resp.hlto >>output 2>&1 +# @TEST-EXEC: printf "+OK\x0d\x0a" | spicy-dump -p RESP::Data redis.hlto >>output 2>&1 +# @TEST-EXEC: printf ":1000\x0d\x0a" | spicy-dump -p RESP::Data redis.hlto >>output 2>&1 +# @TEST-EXEC: printf ":-1000\x0d\x0a" | spicy-dump -p RESP::Data redis.hlto >>output 2>&1 +# @TEST-EXEC: printf ":+1000\x0d\x0a" | spicy-dump -p RESP::Data redis.hlto >>output 2>&1 # @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output diff --git a/testing/btest/scripts/base/protocols/redis/stream.zeek b/testing/btest/scripts/base/protocols/redis/stream.zeek index 773602804a..98c9e3bf02 100644 --- a/testing/btest/scripts/base/protocols/redis/stream.zeek +++ b/testing/btest/scripts/base/protocols/redis/stream.zeek @@ -1,7 +1,7 @@ # @TEST-DOC: Test Zeek parsing pubsub commands # # @TEST-EXEC: zeek -Cr $TRACES/redis/stream.trace base/protocols/redis %INPUT >output -# @TEST-EXEC: btest-diff resp.log +# @TEST-EXEC: btest-diff redis.log # Streams like with XRANGE return arrays of bulk strings. We shouldn't count the # response as commands. diff --git a/testing/btest/scripts/base/protocols/redis/tls.zeek b/testing/btest/scripts/base/protocols/redis/tls.zeek index d9369aeae7..b81ab435f8 100644 --- a/testing/btest/scripts/base/protocols/redis/tls.zeek +++ b/testing/btest/scripts/base/protocols/redis/tls.zeek @@ -1,6 +1,6 @@ # @TEST-DOC: Test Zeek with RESP over TLS so it doesn't get gibberish # # @TEST-EXEC: zeek -Cr $TRACES/redis/tls.trace base/protocols/redis %INPUT >output -# @TEST-EXEC-FAIL: test -f resp.log +# @TEST-EXEC-FAIL: test -f redis.log # The logs should probably be empty since it's all encrypted diff --git a/testing/btest/scripts/base/protocols/redis/trace.zeek b/testing/btest/scripts/base/protocols/redis/trace.zeek index d22707b71c..1daf90ce16 100644 --- a/testing/btest/scripts/base/protocols/redis/trace.zeek +++ b/testing/btest/scripts/base/protocols/redis/trace.zeek @@ -2,7 +2,7 @@ # # @TEST-EXEC: zeek -Cr $TRACES/redis/loop-redis.trace base/protocols/redis %INPUT >output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: btest-diff resp.log +# @TEST-EXEC: btest-diff redis.log event Redis::set_command(c: connection, is_orig: bool, command: Redis::SetCommand) {