Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-17 14:08:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Redis: Rework everything to only use async mode

Browse source
This commit is contained in:
Tim Wojtulewicz 2025-02-14 14:24:55 -08:00
parent 40f60f26b3
commit c247de8ec3
11 changed files with 410 additions and 317 deletions
Download patch file Download diff file Expand all files Collapse all files

32
testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek
View file

@ -1,5 +1,6 @@
# @TEST-DOC: Tests that Redis storage backend defaults back to sync mode reading pcaps
# @TEST-KNOWN-FAILURE: Currently broken due to the redis async rework
# @TEST-REQUIRES: have-redis
# @TEST-PORT: REDIS_PORT
@ -19,30 +20,37 @@
# Create a typename here that can be passed down into open_backend()
type str: string;
event zeek_init() {
local opts : Storage::BackendOptions;
opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = T];
event zeek_init()
{
local opts: Storage::BackendOptions;
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
"REDIS_PORT")), $key_prefix="testing" ];
local key = "key1234";
local value = "value5678";
local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str);
when [b, key, value] ( local res = Storage::Async::put(b, [$key=key, $value=value]) ) {
when [b, key, value] ( local res = Storage::Async::put(b, [ $key=key,
$value=value ]) )
{
print "put result", res;
when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) {
when [b, key, value] ( local res2 = Storage::Async::get(b, key) )
{
print "get result", res2;
if ( res2?$val )
print "get result same as inserted", value == (res2$val as string);
print "get result same as inserted", value == ( res2$val as string );
Storage::Sync::close_backend(b);
}
timeout 5sec
{
print "get request timed out";
}
}
timeout 5 sec {
print "get requeest timed out";
}
}
timeout 5 sec {
timeout 5sec
{
print "put request timed out";
}
}
}

31
testing/btest/scripts/base/frameworks/storage/redis-async.zeek
View file

@ -21,34 +21,41 @@ redef exit_only_after_terminate = T;
# Create a typename here that can be passed down into open_backend()
type str: string;
event zeek_init() {
local opts : Storage::BackendOptions;
opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = T];
event zeek_init()
{
local opts: Storage::BackendOptions;
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
"REDIS_PORT")), $key_prefix="testing" ];
local key = "key1234";
local value = "value5678";
local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str);
when [b, key, value] ( local res = Storage::Async::put(b, [$key=key, $value=value]) ) {
when [b, key, value] ( local res = Storage::Async::put(b, [ $key=key,
$value=value ]) )
{
print "put result", res;
when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) {
when [b, key, value] ( local res2 = Storage::Async::get(b, key) )
{
print "get result", res2;
if ( res2?$val )
print "get result same as inserted", value == (res2$val as string);
print "get result same as inserted", value == ( res2$val as string );
Storage::Sync::close_backend(b);
terminate();
}
timeout 5 sec {
print "get requeest timed out";
}
timeout 5sec
{
print "get request timed out";
terminate();
}
}
}
timeout 5 sec {
timeout 5sec
{
print "put request timed out";
terminate();
}
}
}

32
testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek
View file

@ -39,46 +39,52 @@ global redis_data_written: event() &is_used;
global backend: opaque of Storage::BackendHandle;
type str: string;
event zeek_init() {
local opts : Storage::BackendOptions;
opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F];
event zeek_init()
{
local opts: Storage::BackendOptions;
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
"REDIS_PORT")), $key_prefix="testing" ];
backend = Storage::Sync::open_backend(Storage::REDIS, opts, str, str);
}
}
event redis_data_written() {
event redis_data_written()
{
print "redis_data_written";
local res = Storage::Sync::get(backend, "1234");
print Cluster::node, res;
Storage::Sync::close_backend(backend);
terminate();
}
}
@else
global node_count: count = 0;
event Cluster::node_down(name: string, id: string) {
event Cluster::node_down(name: string, id: string)
{
++node_count;
if ( node_count == 2 )
terminate();
}
}
event redis_data_written() {
event redis_data_written()
{
local e = Cluster::make_event(redis_data_written);
Cluster::publish(Cluster::worker_topic, e);
}
}
@endif
@if ( Cluster::node == "worker-1" )
event Cluster::Experimental::cluster_started() {
local res = Storage::Sync::put(backend, [$key="1234", $value="5678"]);
event Cluster::Experimental::cluster_started()
{
local res = Storage::Sync::put(backend, [ $key="1234", $value="5678" ]);
print Cluster::node, "put result", res;
local e = Cluster::make_event(redis_data_written);
Cluster::publish(Cluster::manager_topic, e);
}
}
@endif

30
testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek
View file

@ -15,7 +15,7 @@
@load base/frameworks/storage/sync
@load policy/frameworks/storage/backend/redis
redef Storage::expire_interval = 2 secs;
redef Storage::expire_interval = 2secs;
redef exit_only_after_terminate = T;
# Create a typename here that can be passed down into open_backend()
@ -25,34 +25,38 @@ global b: opaque of Storage::BackendHandle;
global key: string = "key1234";
global value: string = "value7890";
event check_removed() {
event check_removed()
{
local res2 = Storage::Sync::get(b, key);
print "get result after expiration", res2;
Storage::Sync::close_backend(b);
terminate();
}
}
event setup_test() {
local opts : Storage::BackendOptions;
opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F];
event setup_test()
{
local opts: Storage::BackendOptions;
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
"REDIS_PORT")), $key_prefix="testing" ];
b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str);
local res = Storage::Sync::put(b, [$key=key, $value=value, $expire_time=2 secs]);
local res = Storage::Sync::put(b, [ $key=key, $value=value, $expire_time=2secs ]);
print "put result", res;
local res2 = Storage::Sync::get(b, key);
print "get result", res2;
if ( res2?$val )
print "get result same as inserted", value == (res2$val as string);
print "get result same as inserted", value == ( res2$val as string );
schedule 5 secs { check_removed() };
}
schedule 5secs { check_removed() };
}
event zeek_init() {
event zeek_init()
{
# We need network time to be set to something other than zero for the
# expiration time to be set correctly. Schedule an event on a short
# timer so packets start getting read and do the setup there.
schedule 100 msecs { setup_test() };
}
schedule 100msecs { setup_test() };
}

18
testing/btest/scripts/base/frameworks/storage/redis-sync.zeek
View file

@ -18,31 +18,33 @@
# Create a typename here that can be passed down into open_backend()
type str: string;
event zeek_init() {
local opts : Storage::BackendOptions;
opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F];
event zeek_init()
{
local opts: Storage::BackendOptions;
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
"REDIS_PORT")), $key_prefix="testing" ];
local key = "key1234";
local value = "value1234";
local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str);
local res = Storage::Sync::put(b, [$key=key, $value=value]);
local res = Storage::Sync::put(b, [ $key=key, $value=value ]);
print "put result", res;
local res2 = Storage::Sync::get(b, key);
print "get result", res2;
if ( res2?$val )
print "get result same as inserted", value == (res2$val as string);
print "get result same as inserted", value == ( res2$val as string );
local value2 = "value5678";
res = Storage::Sync::put(b, [$key=key, $value=value2, $overwrite=T]);
res = Storage::Sync::put(b, [ $key=key, $value=value2, $overwrite=T ]);
print "overwrite put result", res;
res2 = Storage::Sync::get(b, key);
print "get result", res2;
if ( res2?$val )
print "get result same as inserted", value2 == (res2$val as string);
print "get result same as inserted", value2 == ( res2$val as string );
Storage::Sync::close_backend(b);
}
}

30
testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek
View file

@ -12,10 +12,11 @@ redef exit_only_after_terminate = T;
# Create a typename here that can be passed down into get().
type str: string;
event zeek_init() {
event zeek_init()
{
# Create a database file in the .tmp directory with a 'testing' table
local opts : Storage::BackendOptions;
opts$sqlite = [$database_path = "test.sqlite", $table_name = "testing"];
local opts: Storage::BackendOptions;
opts$sqlite = [ $database_path="test.sqlite", $table_name="testing" ];
local key = "key1234";
local value = "value5678";
@ -24,25 +25,30 @@ event zeek_init() {
# the backend yet.
local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str);
when [b, key, value] ( local res = Storage::Async::put(b, [$key=key, $value=value]) ) {
when [b, key, value] ( local res = Storage::Async::put(b, [ $key=key,
$value=value ]) )
{
print "put result", res;
when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) {
when [b, key, value] ( local res2 = Storage::Async::get(b, key) )
{
print "get result", res2;
if ( res2?$val )
print "get result same as inserted", value == (res2$val as string);
print "get result same as inserted", value == ( res2$val as string );
Storage::Sync::close_backend(b);
terminate();
}
timeout 5 sec {
print "get requeest timed out";
}
timeout 5sec
{
print "get request timed out";
terminate();
}
}
}
timeout 5 sec {
timeout 5sec
{
print "put request timed out";
terminate();
}
}
}

48
testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek
View file

@ -11,47 +11,59 @@ redef exit_only_after_terminate = T;
# Create a typename here that can be passed down into get().
type str: string;
event zeek_init() {
event zeek_init()
{
# Create a database file in the .tmp directory with a 'testing' table
local opts : Storage::BackendOptions;
opts$sqlite = [$database_path = "test.sqlite", $table_name="testing"];
local opts: Storage::BackendOptions;
opts$sqlite = [ $database_path="test.sqlite", $table_name="testing" ];
local key = "key1234";
local value = "value5678";
# Test inserting/retrieving a key/value pair that we know won't be in
# the backend yet.
when [opts, key, value] ( local b = Storage::Async::open_backend(Storage::SQLITE, opts, str, str) ) {
when [opts, key, value] ( local b = Storage::Async::open_backend(
Storage::SQLITE, opts, str, str) )
{
print "open successful";
when [b, key, value] ( local put_res = Storage::Async::put(b, [$key=key, $value=value]) ) {
when [b, key, value] ( local put_res = Storage::Async::put(b, [ $key=key,
$value=value ]) )
{
print "put result", put_res;
when [b, key, value] ( local get_res = Storage::Async::get(b, key) ) {
when [b, key, value] ( local get_res = Storage::Async::get(b, key) )
{
print "get result", get_res;
if ( get_res?$val )
print "get result same as inserted", value == (get_res$val as string);
print "get result same as inserted", value == ( get_res$val as string );
when [b] ( local close_res = Storage::Async::close_backend(b) ) {
when [b] ( local close_res = Storage::Async::close_backend(b) )
{
print "closed succesfully";
terminate();
} timeout 5 sec {
}
timeout 5sec
{
print "close request timed out";
terminate();
}
}
timeout 5sec
{
print "get request timed out";
terminate();
}
}
timeout 5 sec {
print "get requeest timed out";
terminate();
}
}
timeout 5 sec {
timeout 5sec
{
print "put request timed out";
terminate();
}
}
}
timeout 5 sec {
timeout 5sec
{
print "open request timed out";
terminate();
}
}
}