diff --git a/scripts/base/frameworks/storage/async.zeek b/scripts/base/frameworks/storage/async.zeek index 290b5fe424..3b2ba08dcd 100644 --- a/scripts/base/frameworks/storage/async.zeek +++ b/scripts/base/frameworks/storage/async.zeek @@ -24,7 +24,7 @@ export { ## ## Returns: A handle to the new backend connection, or ``F`` if the connection ## failed. - global open_backend: function(btype: Storage::Backend, options: any, key_type: any, + global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, val_type: any): opaque of Storage::BackendHandle; ## Closes an existing backend connection asynchronously. @@ -74,7 +74,8 @@ export { global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; } -function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle +function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, + val_type: any): opaque of Storage::BackendHandle { return Storage::Async::__open_backend(btype, options, key_type, val_type); } diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index cd46583aa9..66ab1c2744 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -5,7 +5,12 @@ module Storage; export { - ## Record for passing arguments to :zeek:see:`Storage::put`. + ## Base record for backend options. Backend plugins can redef this record to add + ## relevant fields to it. + type BackendOptions: record {}; + + ## Record for passing arguments to :zeek:see:`Storage::Async::put` and + ## :zeek:see:`Storage::Sync::put`. type PutArgs: record { # The key to store the value under. key: any; diff --git a/scripts/base/frameworks/storage/sync.zeek b/scripts/base/frameworks/storage/sync.zeek index b50859cef0..90aaa6a302 100644 --- a/scripts/base/frameworks/storage/sync.zeek +++ b/scripts/base/frameworks/storage/sync.zeek @@ -22,7 +22,7 @@ export { ## ## Returns: A handle to the new backend connection, or ``F`` if the connection ## failed. - global open_backend: function(btype: Storage::Backend, options: any, key_type: any, + global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, val_type: any): opaque of Storage::BackendHandle; ## Closes an existing backend connection. @@ -72,7 +72,8 @@ export { global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; } -function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle +function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, + val_type: any): opaque of Storage::BackendHandle { return Storage::Sync::__open_backend(btype, options, key_type, val_type); } diff --git a/scripts/policy/frameworks/storage/backend/redis/main.zeek b/scripts/policy/frameworks/storage/backend/redis/main.zeek index 413c201083..35e6a6f27f 100644 --- a/scripts/policy/frameworks/storage/backend/redis/main.zeek +++ b/scripts/policy/frameworks/storage/backend/redis/main.zeek @@ -34,4 +34,8 @@ export { # traffic. async_mode: bool &default=T; }; + + redef record Storage::BackendOptions += { + redis: Options &optional; + }; } diff --git a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek index 6cab5911f2..9b4fd386f8 100644 --- a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek +++ b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek @@ -28,4 +28,8 @@ export { ["temp_store"] = "memory" ); }; + + redef record Storage::BackendOptions += { + sqlite: Options &optional; + }; } diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index fb215aa5ae..576edd801e 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -83,29 +83,27 @@ storage::BackendPtr Redis::Instantiate(std::string_view tag) { return make_intru /** * Called by the manager system to open the backend. - * - * Derived classes must implement this method. If successful, the - * implementation must call \a Opened(); if not, it must call Error() - * with a corresponding message. */ ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { + RecordValPtr backend_options = options->GetField("redis"); + // When reading traces we disable storage async mode globally (see src/storage/Backend.cc) since // time moves forward based on the pcap and not based on real time. - async_mode = options->GetField("async_mode")->Get() && ! zeek::run_state::reading_traces; - key_prefix = options->GetField("key_prefix")->ToStdString(); + async_mode = backend_options->GetField("async_mode")->Get() && ! zeek::run_state::reading_traces; + key_prefix = backend_options->GetField("key_prefix")->ToStdString(); DBG_LOG(DBG_STORAGE, "Redis backend: running in async mode? %d", async_mode); redisOptions opt = {0}; - StringValPtr host = options->GetField("server_host"); + StringValPtr host = backend_options->GetField("server_host"); if ( host ) { - PortValPtr port = options->GetField("server_port"); + PortValPtr port = backend_options->GetField("server_port"); server_addr = util::fmt("%s:%d", host->ToStdStringView().data(), port->Port()); REDIS_OPTIONS_SET_TCP(&opt, host->ToStdStringView().data(), port->Port()); } else { - StringValPtr unix_sock = options->GetField("server_unix_socket"); + StringValPtr unix_sock = backend_options->GetField("server_unix_socket"); if ( ! unix_sock ) return util::fmt( "Either server_host/server_port or server_unix_socket must be set in Redis options record"); diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index 8af6246a4b..25b47cca0f 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -28,9 +28,10 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { sqlite3_enable_shared_cache(1); #endif - StringValPtr path = options->GetField("database_path"); + RecordValPtr backend_options = options->GetField("sqlite"); + StringValPtr path = backend_options->GetField("database_path"); full_path = zeek::filesystem::path(path->ToStdString()).string(); - table_name = options->GetField("table_name")->ToStdString(); + table_name = backend_options->GetField("table_name")->ToStdString(); auto open_res = checkError(sqlite3_open_v2(full_path.c_str(), &db, @@ -61,7 +62,7 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { return err; } - auto tuning_params = options->GetField("tuning_params")->ToMap(); + auto tuning_params = backend_options->GetField("tuning_params")->ToMap(); for ( const auto& [k, v] : tuning_params ) { auto ks = k->AsListVal()->Idx(0)->AsStringVal(); auto vs = v->AsStringVal(); diff --git a/testing/btest/Baseline/plugins.storage/zeek-stderr b/testing/btest/Baseline/plugins.storage/zeek-stderr index e114ceba55..8dcb29da7c 100644 --- a/testing/btest/Baseline/plugins.storage/zeek-stderr +++ b/testing/btest/Baseline/plugins.storage/zeek-stderr @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/storage.zeek, line 41: Failed to retrieve data: Failed to find key -error in <...>/sync.zeek, line 74: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::Sync::__open_backend(Storage::Sync::btype, Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) -error in <...>/sync.zeek, line 79: Invalid storage handle (Storage::Sync::__close_backend(Storage::Sync::backend) and F) +error in <...>/storage.zeek, line 45: Failed to retrieve data: Failed to find key +error in <...>/sync.zeek, line 75: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::Sync::__open_backend(Storage::Sync::btype, to_any_coerce Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) +error in <...>/sync.zeek, line 80: Invalid storage handle (Storage::Sync::__close_backend(Storage::Sync::backend) and F) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr index e54152627d..1013a7f2f4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/sync.zeek, line 74: Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file (Storage::Sync::__open_backend(Storage::Sync::btype, Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) -error in <...>/sync.zeek, line 84: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::Sync::__put(Storage::Sync::backend, Storage::Sync::args$key, Storage::Sync::args$value, Storage::Sync::args$overwrite, Storage::Sync::args$expire_time)) +error in <...>/sync.zeek, line 75: Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file (Storage::Sync::__open_backend(Storage::Sync::btype, to_any_coerce Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) +error in <...>/sync.zeek, line 85: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::Sync::__put(Storage::Sync::backend, Storage::Sync::args$key, Storage::Sync::args$value, Storage::Sync::args$overwrite, Storage::Sync::args$expire_time)) diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc index c1460058a7..8007131f3d 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -19,7 +19,8 @@ zeek::storage::BackendPtr StorageDummy::Instantiate(std::string_view tag) { * with a corresponding message. */ zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options, zeek::storage::OpenResultCallback* cb) { - bool open_fail = options->GetField("open_fail")->Get(); + zeek::RecordValPtr backend_options = options->GetField("dummy"); + bool open_fail = backend_options->GetField("open_fail")->Get(); if ( open_fail ) return "open_fail was set to true, returning error"; diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek index 92a598586f..fe876ff31a 100644 --- a/testing/btest/plugins/storage.zeek +++ b/testing/btest/plugins/storage.zeek @@ -17,9 +17,13 @@ type StorageDummyOpts : record { open_fail: bool; }; +redef record Storage::BackendOptions += { + dummy: StorageDummyOpts &optional; +}; + event zeek_init() { - local opts : StorageDummyOpts; - opts$open_fail = F; + local opts : Storage::BackendOptions; + opts$dummy = [$open_fail = F]; local key = "key1234"; local value = "value5678"; @@ -49,7 +53,7 @@ event zeek_init() { get_res?$val, put_res, erase_res)); # Test failing to open the handle and test closing an invalid handle. - opts$open_fail = T; + opts$dummy$open_fail = T; local b2 = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, str, str); Storage::Sync::close_backend(b2); } diff --git a/testing/btest/scripts/base/frameworks/storage/compound-types.zeek b/testing/btest/scripts/base/frameworks/storage/compound-types.zeek index 7b37289179..ba68c4e006 100644 --- a/testing/btest/scripts/base/frameworks/storage/compound-types.zeek +++ b/testing/btest/scripts/base/frameworks/storage/compound-types.zeek @@ -37,9 +37,8 @@ type tbl: table[count] of string; event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "types_test.sqlite"; - opts$table_name = "types_testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "types_test.sqlite", $table_name = "types_testing"]; local key : Rec; key$hello = "hello"; diff --git a/testing/btest/scripts/base/frameworks/storage/erase.zeek b/testing/btest/scripts/base/frameworks/storage/erase.zeek index 25594d9123..585da798b2 100644 --- a/testing/btest/scripts/base/frameworks/storage/erase.zeek +++ b/testing/btest/scripts/base/frameworks/storage/erase.zeek @@ -12,9 +12,8 @@ type str: string; event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "storage-test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; local key = "key1234"; diff --git a/testing/btest/scripts/base/frameworks/storage/expiration.zeek b/testing/btest/scripts/base/frameworks/storage/expiration.zeek index 8332987a38..d53e23fa36 100644 --- a/testing/btest/scripts/base/frameworks/storage/expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/expiration.zeek @@ -28,9 +28,8 @@ event check_removed() { } event setup_test() { - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "storage-test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; backend = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); diff --git a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek index 8aeabecaf6..a31360802f 100644 --- a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek +++ b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek @@ -11,9 +11,8 @@ type str: string; event zeek_init() { - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "storage-test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; local key = "key1234"; local value = "value7890"; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek index 758b0e409f..dec60fce83 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -20,11 +20,8 @@ type str: string; event zeek_init() { - local opts : Storage::Backend::Redis::Options; - opts$server_host = "127.0.0.1"; - opts$server_port = to_port(getenv("REDIS_PORT")); - opts$key_prefix = "testing"; - opts$async_mode = T; + local opts : Storage::BackendOptions; + opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = T]; local key = "key1234"; local value = "value5678"; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek index 6d4b06383f..37947d349d 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek @@ -22,11 +22,8 @@ redef exit_only_after_terminate = T; type str: string; event zeek_init() { - local opts : Storage::Backend::Redis::Options; - opts$server_host = "127.0.0.1"; - opts$server_port = to_port(getenv("REDIS_PORT")); - opts$key_prefix = "testing"; - opts$async_mode = T; + local opts : Storage::BackendOptions; + opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = T]; local key = "key1234"; local value = "value5678"; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek index 1fc9140cbb..3336c31101 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek @@ -40,11 +40,8 @@ global backend: opaque of Storage::BackendHandle; type str: string; event zeek_init() { - local opts : Storage::Backend::Redis::Options; - opts$server_host = "127.0.0.1"; - opts$server_port = to_port(getenv("REDIS_PORT")); - opts$key_prefix = "testing"; - opts$async_mode = F; + local opts : Storage::BackendOptions; + opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F]; backend = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek index 8b813d6a36..6c9b1398e8 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek @@ -34,11 +34,8 @@ event check_removed() { } event setup_test() { - local opts : Storage::Backend::Redis::Options; - opts$server_host = "127.0.0.1"; - opts$server_port = to_port(getenv("REDIS_PORT")); - opts$key_prefix = "testing"; - opts$async_mode = F; + local opts : Storage::BackendOptions; + opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F]; b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek index 8579aa5a0c..cd65443290 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -19,11 +19,8 @@ type str: string; event zeek_init() { - local opts : Storage::Backend::Redis::Options; - opts$server_host = "127.0.0.1"; - opts$server_port = to_port(getenv("REDIS_PORT")); - opts$key_prefix = "testing"; - opts$async_mode = F; + local opts : Storage::BackendOptions; + opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F]; local key = "key1234"; local value = "value1234"; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek index 30aabfaf63..76989fa1c5 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek @@ -14,9 +14,8 @@ type str: string; event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "test.sqlite", $table_name = "testing"]; local key = "key1234"; local value = "value5678"; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index 7e9e5383b9..63d262e55b 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -13,9 +13,8 @@ type str: string; event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "test.sqlite", $table_name="testing"]; local key = "key1234"; local value = "value5678"; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek index 84bffa4dc3..899ba497aa 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek @@ -12,15 +12,15 @@ type str: string; event zeek_init() { # Test opening a database with an invalid path - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "/this/path/should/not/exist/test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "/this/path/should/not/exist/test.sqlite", + $table_name = "testing"]; # This should report an error in .stderr and reporter.log local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); # Open a valid database file - opts$database_path = "test.sqlite"; + opts$sqlite$database_path = "test.sqlite"; b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); local bad_key: count = 12345;