Redis: Add btests for the redis backend

ci/ubuntu-24.04/Dockerfile

@@ -24,6 +24,7 @@ RUN apt-get update && apt-get -y install \
     jq \
     lcov \
     libkrb5-dev \
+    libhiredis-dev \
     libmaxminddb-dev \
     libpcap-dev \
     libssl-dev \
@@ -31,6 +32,7 @@ RUN apt-get update && apt-get -y install \
     python3 \
     python3-dev \
     python3-pip \
+    redis-server \
     ruby \
     sqlite3 \
     swig \

src/storage/backend/redis/Redis.cc

@@ -2,6 +2,7 @@
 
 #include "zeek/storage/backend/redis/Redis.h"
 
+#include "zeek/DebugLogger.h"
 #include "zeek/Func.h"
 #include "zeek/RunState.h"
 #include "zeek/Val.h"
@@ -93,6 +94,8 @@ ErrorResult Redis::DoOpen(RecordValPtr options) {
     async_mode = options->GetField<BoolVal>("async_mode")->Get() && ! zeek::run_state::reading_traces;
     key_prefix = options->GetField<StringVal>("key_prefix")->ToStdString();
 
+    DBG_LOG(DBG_STORAGE, "Redis backend: running in async mode? %d", async_mode);
+
     redisOptions opt = {0};
 
     StringValPtr host = options->GetField<StringVal>("server_host");
@@ -341,6 +344,7 @@ void Redis::HandleEraseResult(redisReply* reply, ErrorResultCallback* callback)
 }
 
 void Redis::OnConnect(int status) {
+    DBG_LOG(DBG_STORAGE, "Redis backend: connection event");
     if ( status == REDIS_OK ) {
         connected = true;
         return;
@@ -350,6 +354,7 @@ void Redis::OnConnect(int status) {
 }
 
 void Redis::OnDisconnect(int status) {
+    DBG_LOG(DBG_STORAGE, "Redis backend: disconnection event");
     if ( status == REDIS_OK ) {
         // TODO: this was an intentional disconnect, nothing to do?
     }

testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+put result, T
+get result, value5678
+get result same as inserted, T

testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+put result, T
+get result, value5678
+get result same as inserted, T

testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out

@@ -0,0 +1,7 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+put result, T
+get result, value1234
+get result same as inserted, T
+overwrite put result, T
+get result, value5678
+get result same as inserted after overwrite, F

testing/btest/Files/redis.conf

@@ -0,0 +1,73 @@
+bind 127.0.0.1
+port %REDIS_PORT%
+pidfile %RUN_PATH%/redis.pid
+loglevel verbose
+logfile %RUN_PATH%/redis.log
+dir %RUN_PATH%
+daemonize no
+databases 1
+
+# All of the values from here down are the default values in the sample config file
+# that comes with redis-server v7.2.7.
+protected-mode yes
+tcp-backlog 511
+timeout 0
+always-show-logo no
+set-proc-title yes
+proc-title-template "{title} {listen-addr} {server-mode}"
+stop-writes-on-bgsave-error yes
+rdbcompression yes
+rdbchecksum yes
+dbfilename dump.rdb
+rdb-del-sync-files no
+replica-serve-stale-data yes
+replica-read-only yes
+repl-diskless-sync yes
+repl-diskless-sync-delay 5
+repl-diskless-sync-max-replicas 0
+repl-diskless-load disabled
+repl-disable-tcp-nodelay no
+replica-priority 100
+acllog-max-len 128
+lazyfree-lazy-eviction no
+lazyfree-lazy-expire no
+lazyfree-lazy-server-del no
+replica-lazy-flush no
+lazyfree-lazy-user-del no
+lazyfree-lazy-user-flush no
+oom-score-adj no
+oom-score-adj-values 0 200 800
+disable-thp yes
+appendonly no
+appendfilename "appendonly.aof"
+appenddirname "appendonlydir"
+appendfsync everysec
+no-appendfsync-on-rewrite no
+auto-aof-rewrite-percentage 100
+auto-aof-rewrite-min-size 64mb
+aof-load-truncated yes
+aof-use-rdb-preamble yes
+aof-timestamp-enabled no
+slowlog-log-slower-than 10000
+slowlog-max-len 128
+latency-monitor-threshold 0
+notify-keyspace-events ""
+hash-max-listpack-entries 512
+hash-max-listpack-value 64
+list-max-listpack-size -2
+list-compress-depth 0
+set-max-intset-entries 512
+zset-max-listpack-entries 128
+zset-max-listpack-value 64
+hll-sparse-max-bytes 3000
+stream-node-max-bytes 4096
+stream-node-max-entries 100
+activerehashing yes
+client-output-buffer-limit normal 0 0 0
+client-output-buffer-limit replica 256mb 64mb 60
+client-output-buffer-limit pubsub 32mb 8mb 60
+hz 10
+dynamic-hz yes
+aof-rewrite-incremental-fsync yes
+rdb-save-incremental-fsync yes
+jemalloc-bg-thread yes

testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek

@@ -0,0 +1,49 @@
+# @TEST-DOC: Tests that Redis storage backend defaults back to sync mode reading pcaps
+
+# @TEST-REQUIRES: have-redis
+# @TEST-PORT: REDIS_PORT
+
+# Generate a redis.conf file with the port defined above, but without the /tcp at the end of
+# it. This also sets some paths in the conf to the testing directory.
+# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf
+# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf
+# @TEST-EXEC: zeek -r $TRACES/http/get.trace -b %INPUT > out
+# @TEST-EXEC: btest-bg-wait -k 0
+
+# @TEST-EXEC: btest-diff out
+
+@load base/frameworks/storage
+@load policy/frameworks/storage/backend/redis
+
+# Create a typename here that can be passed down into open_backend()
+type str: string;
+
+event zeek_init() {
+	local opts : Storage::Backend::Redis::Options;
+	opts$server_host = "127.0.0.1";
+	opts$server_port = to_port(getenv("REDIS_PORT"));
+	opts$key_prefix = "testing";
+	opts$async_mode = T;
+
+	local key = "key1234";
+	local value = "value5678";
+
+	local b = Storage::open_backend(Storage::REDIS, opts, str, str);
+
+	when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) {
+		print "put result", res;
+
+		when [b, key, value] ( local res2 = Storage::get(b, key) ) {
+			print "get result", res2;
+			print "get result same as inserted", value == (res2 as string);
+
+			Storage::close_backend(b);
+		}
+		timeout 5 sec {
+			print "get requeest timed out";
+		}
+	}
+	timeout 5 sec {
+		print "put request timed out";
+	}
+}

testing/btest/scripts/base/frameworks/storage/redis-async.zeek

@@ -0,0 +1,55 @@
+# @TEST-DOC: Tests basic Redis storage backend functions in async mode
+
+# @TEST-REQUIRES: have-redis
+# @TEST-PORT: REDIS_PORT
+
+# Generate a redis.conf file with the port defined above, but without the /tcp at the end of
+# it. This also sets some paths in the conf to the testing directory.
+# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf
+# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf
+# @TEST-EXEC: zeek -b %INPUT > out
+# @TEST-EXEC: btest-bg-wait -k 0
+
+# @TEST-EXEC: btest-diff out
+
+@load base/frameworks/storage
+@load policy/frameworks/storage/backend/redis
+
+redef exit_only_after_terminate = T;
+
+# Create a typename here that can be passed down into open_backend()
+type str: string;
+
+event zeek_init() {
+	local opts : Storage::Backend::Redis::Options;
+	opts$server_host = "127.0.0.1";
+	opts$server_port = to_port(getenv("REDIS_PORT"));
+	opts$key_prefix = "testing";
+	opts$async_mode = T;
+
+	local key = "key1234";
+	local value = "value5678";
+
+	local b = Storage::open_backend(Storage::REDIS, opts, str, str);
+
+	when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) {
+		print "put result", res;
+
+		when [b, key, value] ( local res2 = Storage::get(b, key) ) {
+			print "get result", res2;
+			print "get result same as inserted", value == (res2 as string);
+
+			Storage::close_backend(b);
+
+			terminate();
+		}
+		timeout 5 sec {
+			print "get requeest timed out";
+			terminate();
+		}
+	}
+	timeout 5 sec {
+		print "put request timed out";
+		terminate();
+	}
+}

testing/btest/scripts/base/frameworks/storage/redis-sync.zeek

@@ -0,0 +1,49 @@
+# @TEST-DOC: Tests basic Redis storage backend functions in sync mode, including overwriting
+
+# @TEST-REQUIRES: have-redis
+# @TEST-PORT: REDIS_PORT
+
+# Generate a redis.conf file with the port defined above, but without the /tcp at the end of
+# it. This also sets some paths in the conf to the testing directory.
+# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf
+# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf
+# @TEST-EXEC: zeek -b %INPUT > out
+# @TEST-EXEC: btest-bg-wait -k 0
+
+# @TEST-EXEC: btest-diff out
+
+@load base/frameworks/storage
+@load policy/frameworks/storage/backend/redis
+
+# Create a typename here that can be passed down into open_backend()
+type str: string;
+
+event zeek_init() {
+	local opts : Storage::Backend::Redis::Options;
+	opts$server_host = "127.0.0.1";
+	opts$server_port = to_port(getenv("REDIS_PORT"));
+	opts$key_prefix = "testing";
+	opts$async_mode = F;
+
+	local key = "key1234";
+	local value = "value1234";
+
+	local b = Storage::open_backend(Storage::REDIS, opts, str, str);
+
+	local res = Storage::put(b, [$key=key, $value=value, $async_mode=F]);
+	print "put result", res;
+
+	local res2 = Storage::get(b, key, F);
+	print "get result", res2;
+	print "get result same as inserted", value == (res2 as string);
+
+	local value2 = "value5678";
+	res = Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]);
+	print "overwrite put result", res;
+
+	res2 = Storage::get(b, key, F);
+	print "get result", res2;
+	print "get result same as inserted after overwrite", value == (res2 as string);
+
+	Storage::close_backend(b);
+}

testing/scripts/have-redis

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+zeek -N Zeek::Storage_Backend_Redis && which redis-server >/dev/null
+exit $?