Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Make backend options a record, move actual options to be sub-records

Browse source
This commit is contained in:
Tim Wojtulewicz 2025-02-06 14:30:10 -07:00
parent 64f3969434
commit a485b1d237
23 changed files with 71 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

5
scripts/base/frameworks/storage/async.zeek
View file

@ -24,7 +24,7 @@ export {
## ##
## Returns: A handle to the new backend connection, or ``F`` if the connection ## Returns: A handle to the new backend connection, or ``F`` if the connection
## failed. ## failed.
global open_backend: function(btype: Storage::Backend, options: any, key_type: any, global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any,
val_type: any): opaque of Storage::BackendHandle; val_type: any): opaque of Storage::BackendHandle;
## Closes an existing backend connection asynchronously. ## Closes an existing backend connection asynchronously.
@ -74,7 +74,8 @@ export {
global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool;
} }
function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any,
val_type: any): opaque of Storage::BackendHandle
{ {
return Storage::Async::__open_backend(btype, options, key_type, val_type); return Storage::Async::__open_backend(btype, options, key_type, val_type);
} }

7
scripts/base/frameworks/storage/main.zeek
View file

@ -5,7 +5,12 @@
module Storage; module Storage;
export { export {
## Record for passing arguments to :zeek:see:`Storage::put`. ## Base record for backend options. Backend plugins can redef this record to add
## relevant fields to it.
type BackendOptions: record {};
## Record for passing arguments to :zeek:see:`Storage::Async::put` and
## :zeek:see:`Storage::Sync::put`.
type PutArgs: record { type PutArgs: record {
# The key to store the value under. # The key to store the value under.
key: any; key: any;

5
scripts/base/frameworks/storage/sync.zeek
View file

@ -22,7 +22,7 @@ export {
## ##
## Returns: A handle to the new backend connection, or ``F`` if the connection ## Returns: A handle to the new backend connection, or ``F`` if the connection
## failed. ## failed.
global open_backend: function(btype: Storage::Backend, options: any, key_type: any, global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any,
val_type: any): opaque of Storage::BackendHandle; val_type: any): opaque of Storage::BackendHandle;
## Closes an existing backend connection. ## Closes an existing backend connection.
@ -72,7 +72,8 @@ export {
global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool;
} }
function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any,
val_type: any): opaque of Storage::BackendHandle
{ {
return Storage::Sync::__open_backend(btype, options, key_type, val_type); return Storage::Sync::__open_backend(btype, options, key_type, val_type);
} }

4
scripts/policy/frameworks/storage/backend/redis/main.zeek
View file

@ -34,4 +34,8 @@ export {
# traffic. # traffic.
async_mode: bool &default=T; async_mode: bool &default=T;
}; };
redef record Storage::BackendOptions += {
redis: Options &optional;
};
} }

4
scripts/policy/frameworks/storage/backend/sqlite/main.zeek
View file

@ -28,4 +28,8 @@ export {
["temp_store"] = "memory" ["temp_store"] = "memory"
); );
}; };
redef record Storage::BackendOptions += {
sqlite: Options &optional;
};
} }

16
src/storage/backend/redis/Redis.cc
View file

@ -83,29 +83,27 @@ storage::BackendPtr Redis::Instantiate(std::string_view tag) { return make_intru
/** /**
* Called by the manager system to open the backend. * Called by the manager system to open the backend.
*
* Derived classes must implement this method. If successful, the
* implementation must call \a Opened(); if not, it must call Error()
* with a corresponding message.
*/ */
ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) {
RecordValPtr backend_options = options->GetField<RecordVal>("redis");
// When reading traces we disable storage async mode globally (see src/storage/Backend.cc) since // When reading traces we disable storage async mode globally (see src/storage/Backend.cc) since
// time moves forward based on the pcap and not based on real time. // time moves forward based on the pcap and not based on real time.
async_mode = options->GetField<BoolVal>("async_mode")->Get() && ! zeek::run_state::reading_traces; async_mode = backend_options->GetField<BoolVal>("async_mode")->Get() && ! zeek::run_state::reading_traces;
key_prefix = options->GetField<StringVal>("key_prefix")->ToStdString(); key_prefix = backend_options->GetField<StringVal>("key_prefix")->ToStdString();
DBG_LOG(DBG_STORAGE, "Redis backend: running in async mode? %d", async_mode); DBG_LOG(DBG_STORAGE, "Redis backend: running in async mode? %d", async_mode);
redisOptions opt = {0}; redisOptions opt = {0};
StringValPtr host = options->GetField<StringVal>("server_host"); StringValPtr host = backend_options->GetField<StringVal>("server_host");
if ( host ) { if ( host ) {
PortValPtr port = options->GetField<PortVal>("server_port"); PortValPtr port = backend_options->GetField<PortVal>("server_port");
server_addr = util::fmt("%s:%d", host->ToStdStringView().data(), port->Port()); server_addr = util::fmt("%s:%d", host->ToStdStringView().data(), port->Port());
REDIS_OPTIONS_SET_TCP(&opt, host->ToStdStringView().data(), port->Port()); REDIS_OPTIONS_SET_TCP(&opt, host->ToStdStringView().data(), port->Port());
} }
else { else {
StringValPtr unix_sock = options->GetField<StringVal>("server_unix_socket"); StringValPtr unix_sock = backend_options->GetField<StringVal>("server_unix_socket");
if ( ! unix_sock ) if ( ! unix_sock )
return util::fmt( return util::fmt(
"Either server_host/server_port or server_unix_socket must be set in Redis options record"); "Either server_host/server_port or server_unix_socket must be set in Redis options record");

7
src/storage/backend/sqlite/SQLite.cc
View file

@ -28,9 +28,10 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) {
sqlite3_enable_shared_cache(1); sqlite3_enable_shared_cache(1);
#endif #endif
StringValPtr path = options->GetField<StringVal>("database_path"); RecordValPtr backend_options = options->GetField<RecordVal>("sqlite");
StringValPtr path = backend_options->GetField<StringVal>("database_path");
full_path = zeek::filesystem::path(path->ToStdString()).string(); full_path = zeek::filesystem::path(path->ToStdString()).string();
table_name = options->GetField<StringVal>("table_name")->ToStdString(); table_name = backend_options->GetField<StringVal>("table_name")->ToStdString();
auto open_res = auto open_res =
checkError(sqlite3_open_v2(full_path.c_str(), &db, checkError(sqlite3_open_v2(full_path.c_str(), &db,
@ -61,7 +62,7 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) {
return err; return err;
} }
auto tuning_params = options->GetField<TableVal>("tuning_params")->ToMap(); auto tuning_params = backend_options->GetField<TableVal>("tuning_params")->ToMap();
for ( const auto& [k, v] : tuning_params ) { for ( const auto& [k, v] : tuning_params ) {
auto ks = k->AsListVal()->Idx(0)->AsStringVal(); auto ks = k->AsListVal()->Idx(0)->AsStringVal();
auto vs = v->AsStringVal(); auto vs = v->AsStringVal();

6
testing/btest/Baseline/plugins.storage/zeek-stderr
View file

@ -1,4 +1,4 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
error in <...>/storage.zeek, line 41: Failed to retrieve data: Failed to find key error in <...>/storage.zeek, line 45: Failed to retrieve data: Failed to find key
error in <...>/sync.zeek, line 74: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::Sync::__open_backend(Storage::Sync::btype, Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) error in <...>/sync.zeek, line 75: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::Sync::__open_backend(Storage::Sync::btype, to_any_coerce Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type))
error in <...>/sync.zeek, line 79: Invalid storage handle (Storage::Sync::__close_backend(Storage::Sync::backend) and F) error in <...>/sync.zeek, line 80: Invalid storage handle (Storage::Sync::__close_backend(Storage::Sync::backend) and F)

4
testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr
View file

@ -1,3 +1,3 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
error in <...>/sync.zeek, line 74: Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file (Storage::Sync::__open_backend(Storage::Sync::btype, Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) error in <...>/sync.zeek, line 75: Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file (Storage::Sync::__open_backend(Storage::Sync::btype, to_any_coerce Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type))
error in <...>/sync.zeek, line 84: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::Sync::__put(Storage::Sync::backend, Storage::Sync::args$key, Storage::Sync::args$value, Storage::Sync::args$overwrite, Storage::Sync::args$expire_time)) error in <...>/sync.zeek, line 85: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::Sync::__put(Storage::Sync::backend, Storage::Sync::args$key, Storage::Sync::args$value, Storage::Sync::args$overwrite, Storage::Sync::args$expire_time))

3
testing/btest/plugins/storage-plugin/src/StorageDummy.cc
View file

@ -19,7 +19,8 @@ zeek::storage::BackendPtr StorageDummy::Instantiate(std::string_view tag) {
* with a corresponding message. * with a corresponding message.
*/ */
zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options, zeek::storage::OpenResultCallback* cb) { zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options, zeek::storage::OpenResultCallback* cb) {
bool open_fail = options->GetField<zeek::BoolVal>("open_fail")->Get(); zeek::RecordValPtr backend_options = options->GetField<zeek::RecordVal>("dummy");
bool open_fail = backend_options->GetField<zeek::BoolVal>("open_fail")->Get();
if ( open_fail ) if ( open_fail )
return "open_fail was set to true, returning error"; return "open_fail was set to true, returning error";

10
testing/btest/plugins/storage.zeek
View file

@ -17,9 +17,13 @@ type StorageDummyOpts : record {
open_fail: bool; open_fail: bool;
}; };
redef record Storage::BackendOptions += {
dummy: StorageDummyOpts &optional;
};
event zeek_init() { event zeek_init() {
local opts : StorageDummyOpts; local opts : Storage::BackendOptions;
opts$open_fail = F; opts$dummy = [$open_fail = F];
local key = "key1234"; local key = "key1234";
local value = "value5678"; local value = "value5678";
@ -49,7 +53,7 @@ event zeek_init() {
get_res?$val, put_res, erase_res)); get_res?$val, put_res, erase_res));
# Test failing to open the handle and test closing an invalid handle. # Test failing to open the handle and test closing an invalid handle.
opts$open_fail = T; opts$dummy$open_fail = T;
local b2 = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, str, str); local b2 = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, str, str);
Storage::Sync::close_backend(b2); Storage::Sync::close_backend(b2);
} }

5
testing/btest/scripts/base/frameworks/storage/compound-types.zeek
View file

@ -37,9 +37,8 @@ type tbl: table[count] of string;
event zeek_init() { event zeek_init() {
# Create a database file in the .tmp directory with a 'testing' table # Create a database file in the .tmp directory with a 'testing' table
local opts : Storage::Backend::SQLite::Options; local opts : Storage::BackendOptions;
opts$database_path = "types_test.sqlite"; opts$sqlite = [$database_path = "types_test.sqlite", $table_name = "types_testing"];
opts$table_name = "types_testing";
local key : Rec; local key : Rec;
key$hello = "hello"; key$hello = "hello";

5
testing/btest/scripts/base/frameworks/storage/erase.zeek
View file

@ -12,9 +12,8 @@ type str: string;
event zeek_init() { event zeek_init() {
# Create a database file in the .tmp directory with a 'testing' table # Create a database file in the .tmp directory with a 'testing' table
local opts : Storage::Backend::SQLite::Options; local opts : Storage::BackendOptions;
opts$database_path = "storage-test.sqlite"; opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"];
opts$table_name = "testing";
local key = "key1234"; local key = "key1234";

5
testing/btest/scripts/base/frameworks/storage/expiration.zeek
View file

@ -28,9 +28,8 @@ event check_removed() {
} }
event setup_test() { event setup_test() {
local opts : Storage::Backend::SQLite::Options; local opts : Storage::BackendOptions;
opts$database_path = "storage-test.sqlite"; opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"];
opts$table_name = "testing";
backend = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); backend = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str);

5
testing/btest/scripts/base/frameworks/storage/overwriting.zeek
View file

@ -11,9 +11,8 @@
type str: string; type str: string;
event zeek_init() { event zeek_init() {
local opts : Storage::Backend::SQLite::Options; local opts : Storage::BackendOptions;
opts$database_path = "storage-test.sqlite"; opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"];
opts$table_name = "testing";
local key = "key1234"; local key = "key1234";
local value = "value7890"; local value = "value7890";

7
testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek
View file

@ -20,11 +20,8 @@
type str: string; type str: string;
event zeek_init() { event zeek_init() {
local opts : Storage::Backend::Redis::Options; local opts : Storage::BackendOptions;
opts$server_host = "127.0.0.1"; opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = T];
opts$server_port = to_port(getenv("REDIS_PORT"));
opts$key_prefix = "testing";
opts$async_mode = T;
local key = "key1234"; local key = "key1234";
local value = "value5678"; local value = "value5678";

7
testing/btest/scripts/base/frameworks/storage/redis-async.zeek
View file

@ -22,11 +22,8 @@ redef exit_only_after_terminate = T;
type str: string; type str: string;
event zeek_init() { event zeek_init() {
local opts : Storage::Backend::Redis::Options; local opts : Storage::BackendOptions;
opts$server_host = "127.0.0.1"; opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = T];
opts$server_port = to_port(getenv("REDIS_PORT"));
opts$key_prefix = "testing";
opts$async_mode = T;
local key = "key1234"; local key = "key1234";
local value = "value5678"; local value = "value5678";

7
testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek
View file

@ -40,11 +40,8 @@ global backend: opaque of Storage::BackendHandle;
type str: string; type str: string;
event zeek_init() { event zeek_init() {
local opts : Storage::Backend::Redis::Options; local opts : Storage::BackendOptions;
opts$server_host = "127.0.0.1"; opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F];
opts$server_port = to_port(getenv("REDIS_PORT"));
opts$key_prefix = "testing";
opts$async_mode = F;
backend = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); backend = Storage::Sync::open_backend(Storage::REDIS, opts, str, str);
} }

7
testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek
View file

@ -34,11 +34,8 @@ event check_removed() {
} }
event setup_test() { event setup_test() {
local opts : Storage::Backend::Redis::Options; local opts : Storage::BackendOptions;
opts$server_host = "127.0.0.1"; opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F];
opts$server_port = to_port(getenv("REDIS_PORT"));
opts$key_prefix = "testing";
opts$async_mode = F;
b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str);

7
testing/btest/scripts/base/frameworks/storage/redis-sync.zeek
View file

@ -19,11 +19,8 @@
type str: string; type str: string;
event zeek_init() { event zeek_init() {
local opts : Storage::Backend::Redis::Options; local opts : Storage::BackendOptions;
opts$server_host = "127.0.0.1"; opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F];
opts$server_port = to_port(getenv("REDIS_PORT"));
opts$key_prefix = "testing";
opts$async_mode = F;
local key = "key1234"; local key = "key1234";
local value = "value1234"; local value = "value1234";

5
testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek
View file

@ -14,9 +14,8 @@ type str: string;
event zeek_init() { event zeek_init() {
# Create a database file in the .tmp directory with a 'testing' table # Create a database file in the .tmp directory with a 'testing' table
local opts : Storage::Backend::SQLite::Options; local opts : Storage::BackendOptions;
opts$database_path = "test.sqlite"; opts$sqlite = [$database_path = "test.sqlite", $table_name = "testing"];
opts$table_name = "testing";
local key = "key1234"; local key = "key1234";
local value = "value5678"; local value = "value5678";

5
testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek
View file

@ -13,9 +13,8 @@ type str: string;
event zeek_init() { event zeek_init() {
# Create a database file in the .tmp directory with a 'testing' table # Create a database file in the .tmp directory with a 'testing' table
local opts : Storage::Backend::SQLite::Options; local opts : Storage::BackendOptions;
opts$database_path = "test.sqlite"; opts$sqlite = [$database_path = "test.sqlite", $table_name="testing"];
opts$table_name = "testing";
local key = "key1234"; local key = "key1234";
local value = "value5678"; local value = "value5678";

8
testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek
View file

@ -12,15 +12,15 @@ type str: string;
event zeek_init() { event zeek_init() {
# Test opening a database with an invalid path # Test opening a database with an invalid path
local opts : Storage::Backend::SQLite::Options; local opts : Storage::BackendOptions;
opts$database_path = "/this/path/should/not/exist/test.sqlite"; opts$sqlite = [$database_path = "/this/path/should/not/exist/test.sqlite",
opts$table_name = "testing"; $table_name = "testing"];
# This should report an error in .stderr and reporter.log # This should report an error in .stderr and reporter.log
local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str);
# Open a valid database file # Open a valid database file
opts$database_path = "test.sqlite"; opts$sqlite$database_path = "test.sqlite";
b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str);
local bad_key: count = 12345; local bad_key: count = 12345;