From 08bebaa426e454d6cfe2d7012ef00173b75751a4 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 18 Dec 2024 10:28:31 -0700 Subject: [PATCH] Redis: Add btests for the redis backend --- ci/ubuntu-24.04/Dockerfile | 2 + src/storage/backend/redis/Redis.cc | 5 ++ .../out | 4 + .../out | 4 + .../out | 7 ++ testing/btest/Files/redis.conf | 73 +++++++++++++++++++ .../storage/redis-async-reading-pcap.zeek | 49 +++++++++++++ .../base/frameworks/storage/redis-async.zeek | 55 ++++++++++++++ .../base/frameworks/storage/redis-sync.zeek | 49 +++++++++++++ testing/scripts/have-redis | 4 + 10 files changed, 252 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out create mode 100644 testing/btest/Files/redis.conf create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-async.zeek create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-sync.zeek create mode 100755 testing/scripts/have-redis diff --git a/ci/ubuntu-24.04/Dockerfile b/ci/ubuntu-24.04/Dockerfile index 85f0e36d7f..1a7fec855e 100644 --- a/ci/ubuntu-24.04/Dockerfile +++ b/ci/ubuntu-24.04/Dockerfile @@ -24,6 +24,7 @@ RUN apt-get update && apt-get -y install \ jq \ lcov \ libkrb5-dev \ + libhiredis-dev \ libmaxminddb-dev \ libpcap-dev \ libssl-dev \ @@ -31,6 +32,7 @@ RUN apt-get update && apt-get -y install \ python3 \ python3-dev \ python3-pip \ + redis-server \ ruby \ sqlite3 \ swig \ diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 29a893d704..3fded1452a 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -2,6 +2,7 @@ #include "zeek/storage/backend/redis/Redis.h" +#include "zeek/DebugLogger.h" #include "zeek/Func.h" #include "zeek/RunState.h" #include "zeek/Val.h" @@ -93,6 +94,8 @@ ErrorResult Redis::DoOpen(RecordValPtr options) { async_mode = options->GetField("async_mode")->Get() && ! zeek::run_state::reading_traces; key_prefix = options->GetField("key_prefix")->ToStdString(); + DBG_LOG(DBG_STORAGE, "Redis backend: running in async mode? %d", async_mode); + redisOptions opt = {0}; StringValPtr host = options->GetField("server_host"); @@ -341,6 +344,7 @@ void Redis::HandleEraseResult(redisReply* reply, ErrorResultCallback* callback) } void Redis::OnConnect(int status) { + DBG_LOG(DBG_STORAGE, "Redis backend: connection event"); if ( status == REDIS_OK ) { connected = true; return; @@ -350,6 +354,7 @@ void Redis::OnConnect(int status) { } void Redis::OnDisconnect(int status) { + DBG_LOG(DBG_STORAGE, "Redis backend: disconnection event"); if ( status == REDIS_OK ) { // TODO: this was an intentional disconnect, nothing to do? } diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out new file mode 100644 index 0000000000..5125c8494e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, value5678 +get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out new file mode 100644 index 0000000000..5125c8494e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, value5678 +get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out new file mode 100644 index 0000000000..e7d5445bc7 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, value1234 +get result same as inserted, T +overwrite put result, T +get result, value5678 +get result same as inserted after overwrite, F diff --git a/testing/btest/Files/redis.conf b/testing/btest/Files/redis.conf new file mode 100644 index 0000000000..875c46f9f7 --- /dev/null +++ b/testing/btest/Files/redis.conf @@ -0,0 +1,73 @@ +bind 127.0.0.1 +port %REDIS_PORT% +pidfile %RUN_PATH%/redis.pid +loglevel verbose +logfile %RUN_PATH%/redis.log +dir %RUN_PATH% +daemonize no +databases 1 + +# All of the values from here down are the default values in the sample config file +# that comes with redis-server v7.2.7. +protected-mode yes +tcp-backlog 511 +timeout 0 +always-show-logo no +set-proc-title yes +proc-title-template "{title} {listen-addr} {server-mode}" +stop-writes-on-bgsave-error yes +rdbcompression yes +rdbchecksum yes +dbfilename dump.rdb +rdb-del-sync-files no +replica-serve-stale-data yes +replica-read-only yes +repl-diskless-sync yes +repl-diskless-sync-delay 5 +repl-diskless-sync-max-replicas 0 +repl-diskless-load disabled +repl-disable-tcp-nodelay no +replica-priority 100 +acllog-max-len 128 +lazyfree-lazy-eviction no +lazyfree-lazy-expire no +lazyfree-lazy-server-del no +replica-lazy-flush no +lazyfree-lazy-user-del no +lazyfree-lazy-user-flush no +oom-score-adj no +oom-score-adj-values 0 200 800 +disable-thp yes +appendonly no +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no +slowlog-log-slower-than 10000 +slowlog-max-len 128 +latency-monitor-threshold 0 +notify-keyspace-events "" +hash-max-listpack-entries 512 +hash-max-listpack-value 64 +list-max-listpack-size -2 +list-compress-depth 0 +set-max-intset-entries 512 +zset-max-listpack-entries 128 +zset-max-listpack-value 64 +hll-sparse-max-bytes 3000 +stream-node-max-bytes 4096 +stream-node-max-entries 100 +activerehashing yes +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit replica 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 +hz 10 +dynamic-hz yes +aof-rewrite-incremental-fsync yes +rdb-save-incremental-fsync yes +jemalloc-bg-thread yes \ No newline at end of file diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek new file mode 100644 index 0000000000..0cd3dcc2b9 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -0,0 +1,49 @@ +# @TEST-DOC: Tests that Redis storage backend defaults back to sync mode reading pcaps + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT + +# Generate a redis.conf file with the port defined above, but without the /tcp at the end of +# it. This also sets some paths in the conf to the testing directory. +# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf +# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: zeek -r $TRACES/http/get.trace -b %INPUT > out +# @TEST-EXEC: btest-bg-wait -k 0 + +# @TEST-EXEC: btest-diff out + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/redis + +# Create a typename here that can be passed down into open_backend() +type str: string; + +event zeek_init() { + local opts : Storage::Backend::Redis::Options; + opts$server_host = "127.0.0.1"; + opts$server_port = to_port(getenv("REDIS_PORT")); + opts$key_prefix = "testing"; + opts$async_mode = T; + + local key = "key1234"; + local value = "value5678"; + + local b = Storage::open_backend(Storage::REDIS, opts, str, str); + + when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) { + print "put result", res; + + when [b, key, value] ( local res2 = Storage::get(b, key) ) { + print "get result", res2; + print "get result same as inserted", value == (res2 as string); + + Storage::close_backend(b); + } + timeout 5 sec { + print "get requeest timed out"; + } + } + timeout 5 sec { + print "put request timed out"; + } +} diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek new file mode 100644 index 0000000000..47f913e5ae --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek @@ -0,0 +1,55 @@ +# @TEST-DOC: Tests basic Redis storage backend functions in async mode + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT + +# Generate a redis.conf file with the port defined above, but without the /tcp at the end of +# it. This also sets some paths in the conf to the testing directory. +# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf +# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: btest-bg-wait -k 0 + +# @TEST-EXEC: btest-diff out + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/redis + +redef exit_only_after_terminate = T; + +# Create a typename here that can be passed down into open_backend() +type str: string; + +event zeek_init() { + local opts : Storage::Backend::Redis::Options; + opts$server_host = "127.0.0.1"; + opts$server_port = to_port(getenv("REDIS_PORT")); + opts$key_prefix = "testing"; + opts$async_mode = T; + + local key = "key1234"; + local value = "value5678"; + + local b = Storage::open_backend(Storage::REDIS, opts, str, str); + + when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) { + print "put result", res; + + when [b, key, value] ( local res2 = Storage::get(b, key) ) { + print "get result", res2; + print "get result same as inserted", value == (res2 as string); + + Storage::close_backend(b); + + terminate(); + } + timeout 5 sec { + print "get requeest timed out"; + terminate(); + } + } + timeout 5 sec { + print "put request timed out"; + terminate(); + } +} diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek new file mode 100644 index 0000000000..d21d5979a8 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -0,0 +1,49 @@ +# @TEST-DOC: Tests basic Redis storage backend functions in sync mode, including overwriting + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT + +# Generate a redis.conf file with the port defined above, but without the /tcp at the end of +# it. This also sets some paths in the conf to the testing directory. +# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf +# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: btest-bg-wait -k 0 + +# @TEST-EXEC: btest-diff out + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/redis + +# Create a typename here that can be passed down into open_backend() +type str: string; + +event zeek_init() { + local opts : Storage::Backend::Redis::Options; + opts$server_host = "127.0.0.1"; + opts$server_port = to_port(getenv("REDIS_PORT")); + opts$key_prefix = "testing"; + opts$async_mode = F; + + local key = "key1234"; + local value = "value1234"; + + local b = Storage::open_backend(Storage::REDIS, opts, str, str); + + local res = Storage::put(b, [$key=key, $value=value, $async_mode=F]); + print "put result", res; + + local res2 = Storage::get(b, key, F); + print "get result", res2; + print "get result same as inserted", value == (res2 as string); + + local value2 = "value5678"; + res = Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]); + print "overwrite put result", res; + + res2 = Storage::get(b, key, F); + print "get result", res2; + print "get result same as inserted after overwrite", value == (res2 as string); + + Storage::close_backend(b); +} diff --git a/testing/scripts/have-redis b/testing/scripts/have-redis new file mode 100755 index 0000000000..b00289a280 --- /dev/null +++ b/testing/scripts/have-redis @@ -0,0 +1,4 @@ +#!/bin/sh + +zeek -N Zeek::Storage_Backend_Redis && which redis-server >/dev/null +exit $?