From aeedd25cff52652592c1711c7fa16ca4b4cec7d4 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 12 Aug 2024 10:43:32 -0700 Subject: [PATCH 01/52] Add martinmoene/expected-lite as a submodule --- .gitmodules | 3 +++ CMakeLists.txt | 15 +++++++++++++++ auxil/expected-lite | 1 + 3 files changed, 19 insertions(+) create mode 160000 auxil/expected-lite diff --git a/.gitmodules b/.gitmodules index 7736a3c40c..9eb32420d1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -82,3 +82,6 @@ [submodule "src/cluster/websocket/auxil/IXWebSocket"] path = src/cluster/websocket/auxil/IXWebSocket url = https://github.com/zeek/IXWebSocket.git +[submodule "auxil/expected-lite"] + path = auxil/expected-lite + url = https://github.com/martinmoene/expected-lite.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 3635b54ed9..da8ad5a5f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,6 +343,7 @@ add_zeek_dynamic_plugin_build_interface_include_directories( ${PROJECT_SOURCE_DIR}/auxil/broker/libbroker ${PROJECT_SOURCE_DIR}/auxil/paraglob/include ${PROJECT_SOURCE_DIR}/auxil/prometheus-cpp/core/include + ${PROJECT_SOURCE_DIR}/auxil/expected-lite/include ${CMAKE_BINARY_DIR}/src ${CMAKE_BINARY_DIR}/src/include ${CMAKE_BINARY_DIR}/auxil/binpac/lib @@ -353,6 +354,10 @@ target_include_directories( zeek_dynamic_plugin_base SYSTEM INTERFACE $) +target_include_directories( + zeek_dynamic_plugin_base SYSTEM + INTERFACE $) + # Convenience function for adding an OBJECT library that feeds directly into the # main target(s). # @@ -1015,6 +1020,9 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/auxil/prometheus-cpp/core/include/ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/auxil/prometheus-cpp/core/include/prometheus DESTINATION include/zeek/3rdparty/prometheus-cpp/include) +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/auxil/expected-lite/include/nonstd + DESTINATION include/zeek/3rdparty/) + # Create 3rdparty/ghc within the build directory so that the include for # "zeek/3rdparty/ghc/filesystem.hpp" works within the build tree. execute_process(COMMAND "${CMAKE_COMMAND}" -E make_directory @@ -1025,6 +1033,13 @@ execute_process( "${CMAKE_CURRENT_SOURCE_DIR}/auxil/filesystem/include/ghc" "${CMAKE_CURRENT_BINARY_DIR}/3rdparty/ghc") +# Do the same for nonstd. +execute_process( + COMMAND + "${CMAKE_COMMAND}" -E create_symlink + "${CMAKE_CURRENT_SOURCE_DIR}/auxil/expected-lite/include/nonstd" + "${CMAKE_CURRENT_BINARY_DIR}/3rdparty/nonstd") + # Optional Dependencies set(USE_GEOIP false) diff --git a/auxil/expected-lite b/auxil/expected-lite new file mode 160000 index 0000000000..f339d2f737 --- /dev/null +++ b/auxil/expected-lite @@ -0,0 +1 @@ +Subproject commit f339d2f73730f8fee4412f5e4938717866ecef48 From e2b9e81c535b46edadbab6c73cec42315913bb4b Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 20 Nov 2024 13:40:41 -0700 Subject: [PATCH 02/52] plugin: Add component enum for storage backends --- src/plugin/Component.cc | 2 ++ src/plugin/Component.h | 1 + 2 files changed, 3 insertions(+) diff --git a/src/plugin/Component.cc b/src/plugin/Component.cc index 96476f2aea..7374622910 100644 --- a/src/plugin/Component.cc +++ b/src/plugin/Component.cc @@ -45,6 +45,8 @@ void Component::Describe(ODesc* d) const { case component::LOG_SERIALIZER: d->Add("Log Serializer"); break; + case component::STORAGE_BACKEND: d->Add("Storage Backend"); break; + default: reporter->InternalWarning("unknown component type in plugin::Component::Describe"); d->Add(""); diff --git a/src/plugin/Component.h b/src/plugin/Component.h index f61fe0555b..259fc4ebc8 100644 --- a/src/plugin/Component.h +++ b/src/plugin/Component.h @@ -33,6 +33,7 @@ enum Type { CLUSTER_BACKEND, /// A cluster backend. EVENT_SERIALIZER, /// A serializer for events, used by cluster backends. LOG_SERIALIZER, /// A serializer for log batches, used by cluster backends. + STORAGE_BACKEND, /// A backend for the storage framework. }; } // namespace component From 3d6e7c85b01d61799fe273cf2d9ebcc13b7a4468 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 20 Nov 2024 13:40:55 -0700 Subject: [PATCH 03/52] DebugLogger: add stream for storage --- src/DebugLogger.cc | 2 +- src/DebugLogger.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 6b586d5ab6..bcae3d0803 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -22,7 +22,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {"tm", 0, false}, {"logging", 0, false}, {"input", 0, false}, {"threading", 0, false}, {"plugins", 0, false}, {"zeekygen", 0, false}, {"pktio", 0, false}, {"broker", 0, false}, {"scripts", 0, false}, {"supervisor", 0, false}, {"hashkey", 0, false}, {"spicy", 0, false}, - {"cluster", 0, false}}; + {"cluster", 0, false}, {"storage", 0, false}}; DebugLogger::~DebugLogger() { if ( file && file != stderr ) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 63269443c9..bd61531de1 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -57,6 +57,7 @@ enum DebugStream { DBG_HASHKEY, // HashKey buffers DBG_SPICY, // Spicy functionality DBG_CLUSTER, // Cluster functionality + DBG_STORAGE, // Storage framework NUM_DBGS // Has to be last }; From 2ea0f3e70a0e61dc167022c70037a6b917e67010 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 11 Sep 2023 12:21:58 -0700 Subject: [PATCH 04/52] Lay out initial parts for the Storage framework This includes a manager, component manager, BIF and script code, and parts to support new storage backend plugins. --- CMakeLists.txt | 1 + scripts/base/frameworks/storage/__load__.zeek | 1 + scripts/base/frameworks/storage/main.zeek | 85 +++++++++++ scripts/base/init-default.zeek | 1 + src/CMakeLists.txt | 1 + src/script_opt/FuncInfo.cc | 5 + src/storage/Backend.cc | 42 ++++++ src/storage/Backend.h | 137 ++++++++++++++++++ src/storage/CMakeLists.txt | 10 ++ src/storage/Component.cc | 25 ++++ src/storage/Component.h | 59 ++++++++ src/storage/Manager.cc | 58 ++++++++ src/storage/Manager.h | 52 +++++++ src/storage/backend/CMakeLists.txt | 1 + src/storage/storage.bif | 116 +++++++++++++++ src/util.h | 9 ++ src/zeek-setup.cc | 5 + src/zeekygen/ScriptInfo.cc | 4 + .../canonified_loaded_scripts.log | 1 + .../canonified_loaded_scripts.log | 3 + .../Baseline/opt.ZAM-bif-tracking/output | 2 +- testing/btest/Baseline/plugins.hooks/output | 6 + testing/btest/Baseline/plugins.storage/output | 2 + .../Baseline/plugins.storage/zeek-stderr | 4 + testing/btest/opt/ZAM-bif-tracking.zeek | 5 + .../plugins/storage-plugin/.btest-ignore | 0 .../plugins/storage-plugin/CMakeLists.txt | 16 ++ .../plugins/storage-plugin/src/Plugin.cc | 23 +++ .../btest/plugins/storage-plugin/src/Plugin.h | 16 ++ .../storage-plugin/src/StorageDummy.cc | 77 ++++++++++ .../plugins/storage-plugin/src/StorageDummy.h | 56 +++++++ testing/btest/plugins/storage.zeek | 52 +++++++ 32 files changed, 874 insertions(+), 1 deletion(-) create mode 100644 scripts/base/frameworks/storage/__load__.zeek create mode 100644 scripts/base/frameworks/storage/main.zeek create mode 100644 src/storage/Backend.cc create mode 100644 src/storage/Backend.h create mode 100644 src/storage/CMakeLists.txt create mode 100644 src/storage/Component.cc create mode 100644 src/storage/Component.h create mode 100644 src/storage/Manager.cc create mode 100644 src/storage/Manager.h create mode 100644 src/storage/backend/CMakeLists.txt create mode 100644 src/storage/storage.bif create mode 100644 testing/btest/Baseline/plugins.storage/output create mode 100644 testing/btest/Baseline/plugins.storage/zeek-stderr create mode 100644 testing/btest/plugins/storage-plugin/.btest-ignore create mode 100644 testing/btest/plugins/storage-plugin/CMakeLists.txt create mode 100644 testing/btest/plugins/storage-plugin/src/Plugin.cc create mode 100644 testing/btest/plugins/storage-plugin/src/Plugin.h create mode 100644 testing/btest/plugins/storage-plugin/src/StorageDummy.cc create mode 100644 testing/btest/plugins/storage-plugin/src/StorageDummy.h create mode 100644 testing/btest/plugins/storage.zeek diff --git a/CMakeLists.txt b/CMakeLists.txt index da8ad5a5f8..88d90578a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1143,6 +1143,7 @@ include(FindKqueue) include(FindPrometheusCpp) include_directories(BEFORE "auxil/out_ptr/include") +include_directories(BEFORE "auxil/expected-lite/include") if ((OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0")) set(ZEEK_HAVE_OPENSSL_1_1 true CACHE INTERNAL "" FORCE) diff --git a/scripts/base/frameworks/storage/__load__.zeek b/scripts/base/frameworks/storage/__load__.zeek new file mode 100644 index 0000000000..d551be57d3 --- /dev/null +++ b/scripts/base/frameworks/storage/__load__.zeek @@ -0,0 +1 @@ +@load ./main \ No newline at end of file diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek new file mode 100644 index 0000000000..e879c64504 --- /dev/null +++ b/scripts/base/frameworks/storage/main.zeek @@ -0,0 +1,85 @@ +##! The storage framework provides a way to store long-term data to disk. + +@load base/bif/storage.bif + +module Storage; + +export { + ## Opens a new backend connection based on a configuration object. + ## + ## btype: A tag indicating what type of backend should be opened. + ## + ## options: A record containing the configuration for the connection. + ## + ## Returns: A handle to the new backend connection, or null if the + ## connection failed. + global open_backend: function(btype: Storage::Backend, options: any): opaque of Storage::BackendHandle; + + ## Closes an existing backend connection. + ## + ## backend: A handle to a backend connection. + ## + ## Returns: A boolean indicating success or failure of the operation. + global close_backend: function(backend: opaque of Storage::BackendHandle): bool; + + ## Inserts a new entry into a backend. + ## + ## backend: A handle to a backend connection. + ## + ## key: A key value. + ## + ## value: A corresponding value. + ## + ## overwrite: A flag indicating whether this value should overwrite an + ## existing entry for the key. + ## + ## Returns: A boolean indicating success or failure of the operation. + global put: function(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool): bool; + + ## Gets an entry from the backend. + ## + ## backend: A handle to a backend connection. + ## + ## key: The key to look up. + ## + ## val_type: The type of the value to return. + ## + ## Returns: A boolean indicating success or failure of the + ## operation. Type conversion failures for the value will + ## return false. + global get: function(backend: opaque of Storage::BackendHandle, key: any, val_type: any): any; + + ## Erases an entry from the backend. + ## + ## backend: A handle to a backend connection. + ## + ## key: The key to erase. + ## + ## Returns: A boolean indicating success or failure of the operation. + global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; +} + +function open_backend(btype: Storage::Backend, options: any): opaque of Storage::BackendHandle +{ + return Storage::__open_backend(btype, options); +} + +function close_backend(backend: opaque of Storage::BackendHandle): bool +{ + return Storage::__close_backend(backend); +} + +function put(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool): bool +{ + return Storage::__put(backend, key, value, overwrite); +} + +function get(backend: opaque of Storage::BackendHandle, key: any, val_type: any): any +{ + return Storage::__get(backend, key, val_type); +} + +function erase(backend: opaque of Storage::BackendHandle, key: any): bool +{ + return Storage::__erase(backend, key); +} diff --git a/scripts/base/init-default.zeek b/scripts/base/init-default.zeek index 35c75875d2..90ccaf3445 100644 --- a/scripts/base/init-default.zeek +++ b/scripts/base/init-default.zeek @@ -44,6 +44,7 @@ @load base/frameworks/openflow @load base/frameworks/netcontrol @load base/frameworks/telemetry +@load base/frameworks/storage @if ( have_spicy() ) @load base/frameworks/spicy diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 356a6eff4c..67347bafca 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -205,6 +205,7 @@ add_subdirectory(iosource) add_subdirectory(logging) add_subdirectory(probabilistic) add_subdirectory(session) +add_subdirectory(storage) if (HAVE_SPICY) add_subdirectory(spicy) diff --git a/src/script_opt/FuncInfo.cc b/src/script_opt/FuncInfo.cc index 8d836facb3..dea2ba7524 100644 --- a/src/script_opt/FuncInfo.cc +++ b/src/script_opt/FuncInfo.cc @@ -148,6 +148,11 @@ static std::unordered_map func_attrs = { {"Reporter::warning", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Spicy::__resource_usage", ATTR_NO_ZEEK_SIDE_EFFECTS}, {"Spicy::__toggle_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::__close_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::__erase", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::__get", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::__open_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::__put", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Supervisor::__create", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Supervisor::__destroy", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Supervisor::__is_supervised", ATTR_IDEMPOTENT}, diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc new file mode 100644 index 0000000000..fa9e3e2487 --- /dev/null +++ b/src/storage/Backend.cc @@ -0,0 +1,42 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/Backend.h" + +#include "zeek/broker/Data.h" + +namespace zeek::storage { + +ErrorResult Backend::Open(RecordValPtr options) { return DoOpen(std::move(options)); } + +ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite) { + // The intention for this method is to do some other heavy lifting in regard + // to backends that need to pass data through the manager instead of directly + // through the workers. For the first versions of the storage framework it + // just calls the backend itself directly. + return DoPut(std::move(key), std::move(value), overwrite); +} + +ValResult Backend::Get(ValPtr key, TypePtr value_type) { + // See the note in Put(). + return DoGet(std::move(key), std::move(value_type)); +} + +ErrorResult Backend::Erase(ValPtr key) { + // See the note in Put(). + return DoErase(std::move(key)); +} + +zeek::OpaqueTypePtr detail::backend_opaque; +IMPLEMENT_OPAQUE_VALUE(detail::BackendHandleVal) + +std::optional detail::BackendHandleVal::DoSerializeData() const { + // Cannot serialize. + return std::nullopt; +} + +bool detail::BackendHandleVal::DoUnserializeData(BrokerDataView) { + // Cannot unserialize. + return false; +} + +} // namespace zeek::storage diff --git a/src/storage/Backend.h b/src/storage/Backend.h new file mode 100644 index 0000000000..fedabe08c5 --- /dev/null +++ b/src/storage/Backend.h @@ -0,0 +1,137 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/OpaqueVal.h" +#include "zeek/Val.h" +#include "zeek/util.h" + +namespace zeek::storage { + +class Manager; + +// Result from storage operations that may return an error message. If the +// optional value is unset, the operation succeeded. +using ErrorResult = std::optional; + +// Result from storage operations that return Vals. The ValPtr is an +// IntrusivePtr to some result, and can be null if the operation failed. The +// string value will store an error message if the result is null. +using ValResult = zeek::expected; + +class Backend : public zeek::Obj { +public: + /** + * Returns a descriptive tag representing the source for debugging. + */ + const char* Tag() { return tag.c_str(); } + + /** + * Store a new key/value pair in the backend. + * + * @param key the key for the pair + * @param value the value for the pair + * @param overwrite whether an existing value for a key should be overwritten. + * @return A result pair containing a bool with the success state, and a + * possible error string if the operation failed. + */ + ErrorResult Put(ValPtr key, ValPtr value, bool overwrite = true); + + /** + * Retrieve a value from the backend for a provided key. + * + * @param key the key to lookup in the backend. + * @param value_type The script-land type to be used when retrieving values + * from the backend. + * @return A result pair containing a ValPtr with the resulting value or + * nullptr retrieval failed, and a string with the error message if the + * operation failed. + */ + ValResult Get(ValPtr key, TypePtr value_type); + + /** + * Erases the value for a key from the backend. + * + * @return An optional value potentially containing an error string if + * needed. Will be unset if the operation succeeded. + * possible error string if the operation failed. + */ + ErrorResult Erase(ValPtr key); + + /** + * Returns whether the backend is opened. + */ + virtual bool IsOpen() = 0; + +protected: + // Allow the manager to call Open/Close. + friend class storage::Manager; + + /** + * Constructor + * + * @param tag A string representation of the tag for this backend. This + * is passed from the Manager through the component factory. + */ + Backend(std::string_view tag) : tag(tag) {} + + /** + * Called by the manager system to open the backend. + * + * @param options A record storing configuration options for the backend. + * @return A result pair containing a bool with the success state, and a + * possible error string if the operation failed. + */ + ErrorResult Open(RecordValPtr options); + + /** + * Finalizes the backend when it's being closed. Can be overridden by + * derived classes. + */ + virtual void Close() {} + + /** + * The workhorse method for Open(). + */ + virtual ErrorResult DoOpen(RecordValPtr options) = 0; + + /** + * The workhorse method for Put(). + */ + virtual ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true) = 0; + + /** + * The workhorse method for Get(). + */ + virtual ValResult DoGet(ValPtr key, TypePtr vt) = 0; + + /** + * The workhorse method for Erase(). + */ + virtual ErrorResult DoErase(ValPtr key) = 0; + + std::string tag; +}; + +using BackendPtr = zeek::IntrusivePtr; + +namespace detail { + +extern OpaqueTypePtr backend_opaque; + +class BackendHandleVal : public OpaqueVal { +public: + BackendHandleVal() : OpaqueVal(detail::backend_opaque) {} + BackendHandleVal(BackendPtr backend) : OpaqueVal(detail::backend_opaque), backend(std::move(backend)) {} + ~BackendHandleVal() override = default; + + BackendPtr backend; + +protected: + IntrusivePtr DoClone(CloneState* state) override { return {NewRef{}, this}; } + + DECLARE_OPAQUE_VALUE_DATA(BackendHandleVal) +}; + +} // namespace detail +} // namespace zeek::storage diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt new file mode 100644 index 0000000000..86071e81bd --- /dev/null +++ b/src/storage/CMakeLists.txt @@ -0,0 +1,10 @@ +zeek_add_subdir_library( + storage + SOURCES + Manager.cc + Backend.cc + Component.cc + BIFS + storage.bif) + +add_subdirectory(backend) diff --git a/src/storage/Component.cc b/src/storage/Component.cc new file mode 100644 index 0000000000..1f5bb1da6e --- /dev/null +++ b/src/storage/Component.cc @@ -0,0 +1,25 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/Component.h" + +#include "zeek/Desc.h" +#include "zeek/storage/Manager.h" + +namespace zeek::storage { + +Component::Component(const std::string& name, factory_callback arg_factory) + : plugin::Component(plugin::component::STORAGE_BACKEND, name, 0, storage_mgr->GetTagType()) { + factory = arg_factory; +} + +void Component::Initialize() { + InitializeTag(); + storage_mgr->RegisterComponent(this); +} + +void Component::DoDescribe(ODesc* d) const { + d->Add("Storage::STORAGE_BACKEND_"); + d->Add(CanonicalName()); +} + +} // namespace zeek::storage diff --git a/src/storage/Component.h b/src/storage/Component.h new file mode 100644 index 0000000000..de8c9899ce --- /dev/null +++ b/src/storage/Component.h @@ -0,0 +1,59 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/plugin/Component.h" + +namespace zeek::storage { + +class Backend; + +/** + * Component description for plugins providing storage backends. + */ +class Component : public plugin::Component { +public: + using factory_callback = IntrusivePtr (*)(std::string_view); + + /** + * Constructor. + * + * @param name The name of the provided backend. This name is used + * across the system to identify the backend. + * + * @param factory A factory function to instantiate instances of the + * backend's class, which must be derived directly or indirectly from + * storage::Backend. This is typically a static \c Instantiate() + * method inside the class that just allocates and returns a new + * instance. + */ + Component(const std::string& name, factory_callback factory); + + /** + * Destructor. + */ + ~Component() override = default; + + /** + * Initialization function. This function has to be called before any + * plugin component functionality is used; it is used to add the + * plugin component to the list of components and to initialize tags + */ + void Initialize() override; + + /** + * Returns the backend's factory function. + */ + factory_callback Factory() const { return factory; } + +protected: + /** + * Overridden from plugin::Component. + */ + void DoDescribe(ODesc* d) const override; + +private: + factory_callback factory; +}; + +} // namespace zeek::storage diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc new file mode 100644 index 0000000000..8a52c04271 --- /dev/null +++ b/src/storage/Manager.cc @@ -0,0 +1,58 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/Manager.h" + +#include "zeek/Desc.h" + +namespace zeek::storage { + +Manager::Manager() : plugin::ComponentManager("Storage", "Backend") {} + +void Manager::InitPostScript() { detail::backend_opaque = make_intrusive("Storage::Backend"); } + +zeek::expected Manager::OpenBackend(const Tag& type, RecordValPtr options) { + Component* c = Lookup(type); + if ( ! c ) { + return zeek::unexpected( + util::fmt("Request to open unknown backend (%d:%d)", type.Type(), type.Subtype())); + } + + if ( ! c->Factory() ) { + return zeek::unexpected( + util::fmt("Factory invalid for backend %s", GetComponentName(type).c_str())); + } + + ODesc d; + type.AsVal()->Describe(&d); + + BackendPtr bp = c->Factory()(d.Description()); + + if ( ! bp ) { + return zeek::unexpected( + util::fmt("Failed to instantiate backend %s", GetComponentName(type).c_str())); + } + + if ( auto res = bp->Open(std::move(options)); res.has_value() ) { + return zeek::unexpected( + util::fmt("Failed to open backend %s: %s", GetComponentName(type).c_str(), res.value().c_str())); + } + + // TODO: post Storage::backend_opened event + + backends.push_back(bp); + + return bp; +} + +void Manager::CloseBackend(BackendPtr backend) { + auto it = std::find(backends.begin(), backends.end(), backend); + if ( it == backends.end() ) + return; + + backends.erase(it); + backend->Close(); + + // TODO: post Storage::backend_lost event +} + +} // namespace zeek::storage diff --git a/src/storage/Manager.h b/src/storage/Manager.h new file mode 100644 index 0000000000..74cbbc8bee --- /dev/null +++ b/src/storage/Manager.h @@ -0,0 +1,52 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/plugin/ComponentManager.h" +#include "zeek/storage/Backend.h" +#include "zeek/storage/Component.h" + +namespace zeek::storage { + +class Manager final : public plugin::ComponentManager { +public: + Manager(); + ~Manager() = default; + + /** + * Initialization of the manager. This is called late during Zeek's + * initialization after any scripts are processed. + */ + void InitPostScript(); + + /** + * Opens a new storage backend. + * + * @param type The tag for the type of backend being opened. + * @param options A record val representing the configuration for this + * type of backend. + * @return A pair containing a pointer to a backend and a string for + * returning error messages if needed. + */ + zeek::expected OpenBackend(const Tag& type, RecordValPtr options); + + /** + * Closes a storage backend. + */ + void CloseBackend(BackendPtr backend); + + // TODO: + // - Hooks for storage-backed tables? + // - Handling aggregation from workers on a single manager? + +private: + std::vector backends; +}; + +} // namespace zeek::storage + +namespace zeek { + +extern storage::Manager* storage_mgr; + +} // namespace zeek diff --git a/src/storage/backend/CMakeLists.txt b/src/storage/backend/CMakeLists.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/src/storage/backend/CMakeLists.txt @@ -0,0 +1 @@ + diff --git a/src/storage/storage.bif b/src/storage/storage.bif new file mode 100644 index 0000000000..326ae9e114 --- /dev/null +++ b/src/storage/storage.bif @@ -0,0 +1,116 @@ +%%{ +#include "zeek/storage/Backend.h" +#include "zeek/storage/Manager.h" + +using namespace zeek; +using namespace zeek::storage; +%%} + +module Storage; + +# Generated when a new backend connection is opened +event Storage::backend_opened%(%); + +# Generated when a backend connection is lost +event Storage::backend_lost%(%); + +function Storage::__open_backend%(btype: Storage::Backend, options: any%): opaque of Storage::BackendHandle + %{ + auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; + Tag tag{btype_val}; + + auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; + + auto b = storage_mgr->OpenBackend(tag, options_val); + + if ( ! b.has_value() ) { + emit_builtin_error(b.error().c_str()); + return val_mgr->Bool(false); + } + + return make_intrusive(b.value()); + %} + +function Storage::__close_backend%(backend: opaque of Storage::BackendHandle%) : bool + %{ + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + // Return true here since the backend is already closed + return val_mgr->Bool(true); + + storage_mgr->CloseBackend(b->backend); + + return val_mgr->Bool(true); + %} + +function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool%): bool + %{ + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + return val_mgr->Bool(false); + + // TODO: add support for when statements (see broker/store.bif) + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto val_v = IntrusivePtr{NewRef{}, value}; + auto result = b->backend->Put(key_v, val_v, overwrite); + if ( result.has_value() ) { + emit_builtin_error(util::fmt("Failed to store data: %s", result.value().c_str())); + return val_mgr->Bool(false); + } + + return val_mgr->Bool(true); + %} + +function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, val_type: any%): any + %{ + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + return val_mgr->Bool(false); + + // TODO: add support for when statements (see broker/store.bif) + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto result = b->backend->Get(key_v, vt); + if ( ! result.has_value() ) { + emit_builtin_error(util::fmt("Failed to retrieve data: %s", result.error().c_str())); + return val_mgr->Bool(false); + } + + return result.value(); + %} + +function Storage::__erase%(backend: opaque of Storage::BackendHandle, key: any%): bool + %{ + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + return val_mgr->Bool(false); + + // TODO: add support for when statements (see broker/store.bif) + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto result = b->backend->Erase(key_v); + if ( result.has_value() ) { + emit_builtin_error(util::fmt("Failed to erase data for key: %s", result.value().c_str())); + return val_mgr->Bool(false); + } + + return val_mgr->Bool(true); + %} diff --git a/src/util.h b/src/util.h index f261506310..1c493949f1 100644 --- a/src/util.h +++ b/src/util.h @@ -97,6 +97,15 @@ namespace filesystem = ghc::filesystem; inline constexpr std::string_view path_list_separator = ":"; #endif +#include "zeek/3rdparty/nonstd/expected.hpp" +namespace zeek { +template +using expected = nonstd::expected; + +template +using unexpected = nonstd::unexpected; +} // namespace zeek + using zeek_int_t = int64_t; using zeek_uint_t = uint64_t; diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 05e4e0fb52..ba5dc89e1f 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -65,6 +65,7 @@ #ifdef HAVE_SPICY #include "zeek/spicy/manager.h" #endif +#include "zeek/storage/Manager.h" #include "zeek/supervisor/Supervisor.h" #include "zeek/telemetry/Manager.h" #include "zeek/threading/Manager.h" @@ -178,6 +179,7 @@ zeek::detail::trigger::Manager* zeek::detail::trigger_mgr = nullptr; #ifdef HAVE_SPICY zeek::spicy::Manager* zeek::spicy_mgr = nullptr; #endif +zeek::storage::Manager* zeek::storage_mgr = nullptr; zeek::cluster::Manager* zeek::cluster::manager = nullptr; zeek::cluster::Backend* zeek::cluster::backend = nullptr; @@ -414,6 +416,7 @@ static void terminate_zeek() { #ifdef HAVE_SPICY delete spicy_mgr; #endif + delete storage_mgr; // free the global scope pop_scope(); @@ -686,6 +689,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) { #ifdef HAVE_SPICY spicy_mgr = new spicy::Manager(); // registers as plugin with the plugin manager #endif + storage_mgr = new storage::Manager(); plugin_mgr->InitPreScript(); file_mgr->InitPreScript(); @@ -873,6 +877,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) { timer_mgr->InitPostScript(); event_mgr.InitPostScript(); + storage_mgr->InitPostScript(); if ( supervisor_mgr ) supervisor_mgr->InitPostScript(); diff --git a/src/zeekygen/ScriptInfo.cc b/src/zeekygen/ScriptInfo.cc index 70347baa88..71464fa87d 100644 --- a/src/zeekygen/ScriptInfo.cc +++ b/src/zeekygen/ScriptInfo.cc @@ -321,6 +321,10 @@ void ScriptInfo::DoInitPostScript() { const auto& log_serializer_id = zeek::detail::global_scope()->Find("Cluster::LogSerializerTag"); types.push_back(new IdentifierInfo(log_serializer_id, this)); } + else if ( name == "base/frameworks/storage/main.zeek" ) { + const auto& backend_id = zeek::detail::global_scope()->Find("Storage::Backend"); + types.push_back(new IdentifierInfo(backend_id, this)); + } } vector ScriptInfo::GetComments() const { return comments; } diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index d99e27e5ec..dedda53b20 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -160,6 +160,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek + build/scripts/base/bif/storage.bif.zeek build/scripts/base/bif/spicy.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 4e61a3a752..ebf298afa4 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -160,6 +160,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek + build/scripts/base/bif/storage.bif.zeek build/scripts/base/bif/spicy.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek @@ -367,6 +368,8 @@ scripts/base/init-default.zeek scripts/base/frameworks/telemetry/__load__.zeek scripts/base/frameworks/telemetry/main.zeek scripts/base/misc/version.zeek + scripts/base/frameworks/storage/__load__.zeek + scripts/base/frameworks/storage/main.zeek scripts/base/frameworks/spicy/__load__.zeek scripts/base/frameworks/spicy/main.zeek scripts/base/protocols/conn/__load__.zeek diff --git a/testing/btest/Baseline/opt.ZAM-bif-tracking/output b/testing/btest/Baseline/opt.ZAM-bif-tracking/output index 0731586167..4fab29b90a 100644 --- a/testing/btest/Baseline/opt.ZAM-bif-tracking/output +++ b/testing/btest/Baseline/opt.ZAM-bif-tracking/output @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -546 seen BiFs, 0 unseen BiFs (), 0 new BiFs () +551 seen BiFs, 0 unseen BiFs (), 0 new BiFs () diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 9174cce539..b66bf362d9 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -505,6 +505,7 @@ 0.000000 MetaHookPost LoadFile(0, ./sftp, <...>/sftp.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./storage.bif.zeek, <...>/storage.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./store, <...>/store.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./store.bif.zeek, <...>/store.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> -1 @@ -815,6 +816,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./sftp, <...>/sftp.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./storage.bif.zeek, <...>/storage.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./store, <...>/store.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> (-1, ) @@ -1458,6 +1460,7 @@ 0.000000 MetaHookPre LoadFile(0, ./sftp, <...>/sftp.zeek) 0.000000 MetaHookPre LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./storage.bif.zeek, <...>/storage.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./store, <...>/store.zeek) 0.000000 MetaHookPre LoadFile(0, ./store.bif.zeek, <...>/store.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek) @@ -1768,6 +1771,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./sftp, <...>/sftp.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./storage.bif.zeek, <...>/storage.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./store, <...>/store.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek) @@ -2422,6 +2426,7 @@ 0.000000 | HookLoadFile ./sftp <...>/sftp.zeek 0.000000 | HookLoadFile ./spicy.bif.zeek <...>/spicy.bif.zeek 0.000000 | HookLoadFile ./stats.bif.zeek <...>/stats.bif.zeek +0.000000 | HookLoadFile ./storage.bif.zeek <...>/storage.bif.zeek 0.000000 | HookLoadFile ./store <...>/store.zeek 0.000000 | HookLoadFile ./store.bif.zeek <...>/store.bif.zeek 0.000000 | HookLoadFile ./strings.bif.zeek <...>/strings.bif.zeek @@ -2732,6 +2737,7 @@ 0.000000 | HookLoadFileExtended ./sftp <...>/sftp.zeek 0.000000 | HookLoadFileExtended ./spicy.bif.zeek <...>/spicy.bif.zeek 0.000000 | HookLoadFileExtended ./stats.bif.zeek <...>/stats.bif.zeek +0.000000 | HookLoadFileExtended ./storage.bif.zeek <...>/storage.bif.zeek 0.000000 | HookLoadFileExtended ./store <...>/store.zeek 0.000000 | HookLoadFileExtended ./store.bif.zeek <...>/store.bif.zeek 0.000000 | HookLoadFileExtended ./strings.bif.zeek <...>/strings.bif.zeek diff --git a/testing/btest/Baseline/plugins.storage/output b/testing/btest/Baseline/plugins.storage/output new file mode 100644 index 0000000000..0812d557db --- /dev/null +++ b/testing/btest/Baseline/plugins.storage/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +results of trying to use closed handle: get: 0, put: 0, erase: 0 diff --git a/testing/btest/Baseline/plugins.storage/zeek-stderr b/testing/btest/Baseline/plugins.storage/zeek-stderr new file mode 100644 index 0000000000..4963c5ee83 --- /dev/null +++ b/testing/btest/Baseline/plugins.storage/zeek-stderr @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/storage.zeek, line 37: Failed to retrieve data: Failed to find key (Storage::get(b, to_any_coerce key, to_any_coerce str)) +error in <...>/storage.zeek, line 50: Failed to open backend STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts)) +error in <...>/storage.zeek, line 51: Invalid storage handle (Storage::close_backend(b2) and F) diff --git a/testing/btest/opt/ZAM-bif-tracking.zeek b/testing/btest/opt/ZAM-bif-tracking.zeek index 1668c4af35..2a906c365f 100644 --- a/testing/btest/opt/ZAM-bif-tracking.zeek +++ b/testing/btest/opt/ZAM-bif-tracking.zeek @@ -177,6 +177,11 @@ global known_BiFs = set( "Reporter::warning", "Spicy::__resource_usage", "Spicy::__toggle_analyzer", + "Storage::__close_backend", + "Storage::__erase", + "Storage::__get", + "Storage::__open_backend", + "Storage::__put", "Supervisor::__create", "Supervisor::__destroy", "Supervisor::__is_supervised", diff --git a/testing/btest/plugins/storage-plugin/.btest-ignore b/testing/btest/plugins/storage-plugin/.btest-ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/plugins/storage-plugin/CMakeLists.txt b/testing/btest/plugins/storage-plugin/CMakeLists.txt new file mode 100644 index 0000000000..da332c5eb9 --- /dev/null +++ b/testing/btest/plugins/storage-plugin/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.15) + +project(Zeek-Plugin-Storage-Demo) + +if (NOT ZEEK_DIST) + message(FATAL_ERROR "ZEEK_DIST not set") +endif () + +set(CMAKE_MODULE_PATH ${ZEEK_DIST}/cmake) + +include(ZeekPlugin) + +zeek_plugin_begin(Testing StorageDummy) +zeek_plugin_cc(src/Plugin.cc) +zeek_plugin_cc(src/StorageDummy.cc) +zeek_plugin_end() diff --git a/testing/btest/plugins/storage-plugin/src/Plugin.cc b/testing/btest/plugins/storage-plugin/src/Plugin.cc new file mode 100644 index 0000000000..ca7e42f4e6 --- /dev/null +++ b/testing/btest/plugins/storage-plugin/src/Plugin.cc @@ -0,0 +1,23 @@ +#include "Plugin.h" + +#include "zeek/storage/Component.h" + +#include "StorageDummy.h" + +namespace btest::plugin::Testing_StorageDummy { +Plugin plugin; +} + +using namespace btest::plugin::Testing_StorageDummy; + +zeek::plugin::Configuration Plugin::Configure() { + AddComponent(new zeek::storage::Component("StorageDummy", btest::storage::backend::StorageDummy::Instantiate)); + + zeek::plugin::Configuration config; + config.name = "Testing::StorageDummy"; + config.description = "A dummy storage plugin"; + config.version.major = 1; + config.version.minor = 0; + config.version.patch = 0; + return config; +} diff --git a/testing/btest/plugins/storage-plugin/src/Plugin.h b/testing/btest/plugins/storage-plugin/src/Plugin.h new file mode 100644 index 0000000000..1a386be0a6 --- /dev/null +++ b/testing/btest/plugins/storage-plugin/src/Plugin.h @@ -0,0 +1,16 @@ + +#pragma once + +#include + +namespace btest::plugin::Testing_StorageDummy { + +class Plugin : public zeek::plugin::Plugin { +protected: + // Overridden from plugin::Plugin. + virtual zeek::plugin::Configuration Configure(); +}; + +extern Plugin plugin; + +} // namespace btest::plugin::Testing_StorageDummy diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc new file mode 100644 index 0000000000..0f1c319fa4 --- /dev/null +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -0,0 +1,77 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "StorageDummy.h" + +#include "zeek/Func.h" +#include "zeek/Val.h" + +namespace btest::storage::backend { + +zeek::storage::BackendPtr StorageDummy::Instantiate(std::string_view tag) { + return zeek::make_intrusive(tag); +} + +/** + * Called by the manager system to open the backend. + * + * Derived classes must implement this method. If successful, the + * implementation must call \a Opened(); if not, it must call Error() + * with a corresponding message. + */ +zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options) { + bool open_fail = options->GetField("open_fail")->Get(); + if ( open_fail ) + return "open_fail was set to true, returning error"; + + open = true; + + return std::nullopt; +} + +/** + * Finalizes the backend when it's being closed. + */ +void StorageDummy::Close() { open = false; } + +/** + * The workhorse method for Put(). This must be implemented by plugins. + */ +zeek::storage::ErrorResult StorageDummy::DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite) { + auto json_key = key->ToJSON()->ToStdString(); + auto json_value = value->ToJSON()->ToStdString(); + data[json_key] = json_value; + return std::nullopt; +} + +/** + * The workhorse method for Get(). This must be implemented for plugins. + */ +zeek::storage::ValResult StorageDummy::DoGet(zeek::ValPtr key, zeek::TypePtr vt) { + auto json_key = key->ToJSON(); + auto it = data.find(json_key->ToStdString()); + if ( it == data.end() ) + return zeek::unexpected("Failed to find key"); + + auto val = zeek::detail::ValFromJSON(it->second.c_str(), vt, zeek::Func::nil); + if ( std::holds_alternative(val) ) { + zeek::ValPtr val_v = std::get(val); + return val_v; + } + + return zeek::unexpected(std::get(val)); +} + +/** + * The workhorse method for Erase(). This must be implemented for plugins. + */ +zeek::storage::ErrorResult StorageDummy::DoErase(zeek::ValPtr key) { + auto json_key = key->ToJSON(); + auto it = data.find(json_key->ToStdString()); + if ( it == data.end() ) + return "Failed to find key"; + + data.erase(it); + return std::nullopt; +} + +} // namespace btest::storage::backend diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h new file mode 100644 index 0000000000..cce7fb0293 --- /dev/null +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -0,0 +1,56 @@ + +#pragma once + +#include +#include + +#include "zeek/storage/Backend.h" + +namespace btest::storage::backend { + +/** + * A Foo reader to measure performance of the input framework. + */ +class StorageDummy : public zeek::storage::Backend { +public: + StorageDummy(std::string_view tag) : Backend(tag) {} + ~StorageDummy() override = default; + + static zeek::storage::BackendPtr Instantiate(std::string_view tag); + + /** + * Called by the manager system to open the backend. + */ + zeek::storage::ErrorResult DoOpen(zeek::RecordValPtr options) override; + + /** + * Finalizes the backend when it's being closed. + */ + void Close() override; + + /** + * Returns whether the backend is opened. + */ + bool IsOpen() override { return open; } + + /** + * The workhorse method for Put(). + */ + zeek::storage::ErrorResult DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite = true) override; + + /** + * The workhorse method for Get(). + */ + zeek::storage::ValResult DoGet(zeek::ValPtr key, zeek::TypePtr vt) override; + + /** + * The workhorse method for Erase(). + */ + zeek::storage::ErrorResult DoErase(zeek::ValPtr key) override; + +private: + std::map data; + bool open = false; +}; + +} // namespace btest::storage::backend diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek new file mode 100644 index 0000000000..8034d27cf4 --- /dev/null +++ b/testing/btest/plugins/storage.zeek @@ -0,0 +1,52 @@ +# @TEST-DOC: Basic test of a plugin implmenting a backend for the storage framework +# @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1" + +# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Testing StorageDummy +# @TEST-EXEC: cp -r %DIR/storage-plugin/* . +# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make +# @TEST-EXEC: ZEEK_PLUGIN_PATH=$(pwd) zeek -b Testing::StorageDummy %INPUT >> output 2>zeek-stderr +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff zeek-stderr + +@load base/frameworks/storage + +# Create a typename here that can be passed down into get(). +type str: string; + +type StorageDummyOpts : record { + open_fail: bool; +}; + +event zeek_init() { + local opts : StorageDummyOpts; + opts$open_fail = F; + + local key = "key1234"; + local value = "value5678"; + + # Test basic operation. The second get() should return an error + # as the key should have been erased. + local b = Storage::open_backend(Storage::STORAGEDUMMY, opts); + local put_res = Storage::put(b, key, value, F); + local get_res = Storage::get(b, key, str); + if ( get_res is bool ) { + print("Got an invalid value in response!"); + } + + local erase_res = Storage::erase(b, key); + get_res = Storage::get(b, key, str); + Storage::close_backend(b); + + # Test attempting to use the closed handle. + put_res = Storage::put(b, "a", "b", F); + get_res = Storage::get(b, "a", str); + erase_res = Storage::erase(b, "a"); + + print(fmt("results of trying to use closed handle: get: %d, put: %d, erase: %d", + get_res, put_res, erase_res)); + + # Test failing to open the handle and test closing an invalid handle. + opts$open_fail = T; + local b2 = Storage::open_backend(Storage::STORAGEDUMMY, opts); + Storage::close_backend(b2); +} From 69d940533d9e309bed106e540af8dc9fa12206b8 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 6 Dec 2024 14:24:27 -0700 Subject: [PATCH 05/52] Pass key/value types for validation when opening backends --- scripts/base/frameworks/storage/main.zeek | 50 ++++++++++++------- src/storage/Backend.cc | 27 ++++++++-- src/storage/Backend.h | 24 +++++---- src/storage/Manager.cc | 5 +- src/storage/Manager.h | 15 ++++-- src/storage/storage.bif | 13 ++--- .../Baseline/plugins.storage/zeek-stderr | 4 +- .../storage-plugin/src/StorageDummy.cc | 4 +- .../plugins/storage-plugin/src/StorageDummy.h | 2 +- testing/btest/plugins/storage.zeek | 10 ++-- 10 files changed, 100 insertions(+), 54 deletions(-) diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index e879c64504..b575b823e9 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -7,13 +7,22 @@ module Storage; export { ## Opens a new backend connection based on a configuration object. ## - ## btype: A tag indicating what type of backend should be opened. + ## btype: A tag indicating what type of backend should be opened. These are + ## defined by the backend plugins loaded. ## ## options: A record containing the configuration for the connection. ## - ## Returns: A handle to the new backend connection, or null if the - ## connection failed. - global open_backend: function(btype: Storage::Backend, options: any): opaque of Storage::BackendHandle; + ## key_type: The script-level type of keys stored in the backend. Used for + ## validation of keys passed to other framework methods. + ## + ## val_type: The script-level type of keys stored in the backend. Used for + ## validation of values passed to :zeek:see:`Storage::put` as well as + ## for type conversions for return values from :zeek:see:`Storage::get`. + ## + ## Returns: A handle to the new backend connection, or ``F`` if the connection + ## failed. + global open_backend: function(btype: Storage::Backend, options: any, key_type: any, + val_type: any): opaque of Storage::BackendHandle; ## Closes an existing backend connection. ## @@ -30,10 +39,13 @@ export { ## ## value: A corresponding value. ## - ## overwrite: A flag indicating whether this value should overwrite an - ## existing entry for the key. + ## overwrite: A flag indicating whether this value should overwrite an existing + ## entry for the key. ## - ## Returns: A boolean indicating success or failure of the operation. + ## Returns: A boolean indicating success or failure of the operation. Type + ## comparison failures against the types passed to + ## :zeek:see:`Storage::open_backend` for the backend will cause false to + ## be returned. global put: function(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool): bool; ## Gets an entry from the backend. @@ -42,12 +54,11 @@ export { ## ## key: The key to look up. ## - ## val_type: The type of the value to return. - ## - ## Returns: A boolean indicating success or failure of the - ## operation. Type conversion failures for the value will - ## return false. - global get: function(backend: opaque of Storage::BackendHandle, key: any, val_type: any): any; + ## Returns: A boolean indicating success or failure of the operation. Type + ## comparison failures against the types passed to + ## :zeek:see:`Storage::open_backend` for the backend will cause false to + ## be returned. + global get: function(backend: opaque of Storage::BackendHandle, key: any): any; ## Erases an entry from the backend. ## @@ -55,13 +66,16 @@ export { ## ## key: The key to erase. ## - ## Returns: A boolean indicating success or failure of the operation. + ## Returns: A boolean indicating success or failure of the operation. Type + ## comparison failures against the types passed to + ## :zeek:see:`Storage::open_backend` for the backend will cause false to + ## be returned. global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; } -function open_backend(btype: Storage::Backend, options: any): opaque of Storage::BackendHandle +function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle { - return Storage::__open_backend(btype, options); + return Storage::__open_backend(btype, options, key_type, val_type); } function close_backend(backend: opaque of Storage::BackendHandle): bool @@ -74,9 +88,9 @@ function put(backend: opaque of Storage::BackendHandle, key: any, value: any, ov return Storage::__put(backend, key, value, overwrite); } -function get(backend: opaque of Storage::BackendHandle, key: any, val_type: any): any +function get(backend: opaque of Storage::BackendHandle, key: any): any { - return Storage::__get(backend, key, val_type); + return Storage::__get(backend, key); } function erase(backend: opaque of Storage::BackendHandle, key: any): bool diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index fa9e3e2487..f064344d62 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -2,27 +2,48 @@ #include "zeek/storage/Backend.h" +#include "zeek/Desc.h" #include "zeek/broker/Data.h" namespace zeek::storage { -ErrorResult Backend::Open(RecordValPtr options) { return DoOpen(std::move(options)); } +ErrorResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt) { + key_type = std::move(kt); + val_type = std::move(vt); + + return DoOpen(std::move(options)); +} ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite) { // The intention for this method is to do some other heavy lifting in regard // to backends that need to pass data through the manager instead of directly // through the workers. For the first versions of the storage framework it // just calls the backend itself directly. + if ( ! same_type(key->GetType(), key_type) ) + return util::fmt("type of key passed (%s) does not match backend's key type (%s)", + obj_desc_short(key->GetType().get()).c_str(), key_type->GetName().c_str()); + if ( ! same_type(value->GetType(), val_type) ) + return util::fmt("type of value passed (%s) does not match backend's value type (%s)", + obj_desc_short(value->GetType().get()).c_str(), val_type->GetName().c_str()); + return DoPut(std::move(key), std::move(value), overwrite); } -ValResult Backend::Get(ValPtr key, TypePtr value_type) { +ValResult Backend::Get(ValPtr key) { // See the note in Put(). - return DoGet(std::move(key), std::move(value_type)); + if ( ! same_type(key->GetType(), key_type) ) + return zeek::unexpected(util::fmt("type of key passed (%s) does not match backend's key type (%s)", + key->GetType()->GetName().c_str(), key_type->GetName().c_str())); + + return DoGet(std::move(key)); } ErrorResult Backend::Erase(ValPtr key) { // See the note in Put(). + if ( ! same_type(key->GetType(), key_type) ) + return util::fmt("type of key passed (%s) does not match backend's key type (%s)", + key->GetType()->GetName().c_str(), key_type->GetName().c_str()); + return DoErase(std::move(key)); } diff --git a/src/storage/Backend.h b/src/storage/Backend.h index fedabe08c5..dccff707ee 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -41,13 +41,10 @@ public: * Retrieve a value from the backend for a provided key. * * @param key the key to lookup in the backend. - * @param value_type The script-land type to be used when retrieving values - * from the backend. - * @return A result pair containing a ValPtr with the resulting value or - * nullptr retrieval failed, and a string with the error message if the - * operation failed. + * @return A std::expected containing either a valid ValPtr with the result + * of the operation or a string containing an error message for failure. */ - ValResult Get(ValPtr key, TypePtr value_type); + ValResult Get(ValPtr key); /** * Erases the value for a key from the backend. @@ -79,10 +76,14 @@ protected: * Called by the manager system to open the backend. * * @param options A record storing configuration options for the backend. - * @return A result pair containing a bool with the success state, and a - * possible error string if the operation failed. + * @param kt The script-side type of the keys stored in the backend. Used for + * validation of types. + * @param vt The script-side type of the values stored in the backend. Used for + * validation of types and conversion during retrieval. + * @return An optional value potentially containing an error string if + * needed. Will be unset if the operation succeeded. */ - ErrorResult Open(RecordValPtr options); + ErrorResult Open(RecordValPtr options, TypePtr kt, TypePtr vt); /** * Finalizes the backend when it's being closed. Can be overridden by @@ -103,13 +104,16 @@ protected: /** * The workhorse method for Get(). */ - virtual ValResult DoGet(ValPtr key, TypePtr vt) = 0; + virtual ValResult DoGet(ValPtr key) = 0; /** * The workhorse method for Erase(). */ virtual ErrorResult DoErase(ValPtr key) = 0; + TypePtr key_type; + TypePtr val_type; + std::string tag; }; diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index 8a52c04271..5c24ff38eb 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -10,7 +10,8 @@ Manager::Manager() : plugin::ComponentManager("Storage", "Ba void Manager::InitPostScript() { detail::backend_opaque = make_intrusive("Storage::Backend"); } -zeek::expected Manager::OpenBackend(const Tag& type, RecordValPtr options) { +zeek::expected Manager::OpenBackend(const Tag& type, RecordValPtr options, TypePtr key_type, + TypePtr val_type) { Component* c = Lookup(type); if ( ! c ) { return zeek::unexpected( @@ -32,7 +33,7 @@ zeek::expected Manager::OpenBackend(const Tag& type, Re util::fmt("Failed to instantiate backend %s", GetComponentName(type).c_str())); } - if ( auto res = bp->Open(std::move(options)); res.has_value() ) { + if ( auto res = bp->Open(std::move(options), std::move(key_type), std::move(val_type)); res.has_value() ) { return zeek::unexpected( util::fmt("Failed to open backend %s: %s", GetComponentName(type).c_str(), res.value().c_str())); } diff --git a/src/storage/Manager.h b/src/storage/Manager.h index 74cbbc8bee..c0545e53a8 100644 --- a/src/storage/Manager.h +++ b/src/storage/Manager.h @@ -23,12 +23,17 @@ public: * Opens a new storage backend. * * @param type The tag for the type of backend being opened. - * @param options A record val representing the configuration for this - * type of backend. - * @return A pair containing a pointer to a backend and a string for - * returning error messages if needed. + * @param options A record val representing the configuration for this type of + * backend. + * @param key_type The script-side type of the keys stored in the backend. Used for + * validation of types. + * @param val_type The script-side type of the values stored in the backend. Used for + * validation of types and conversion during retrieval. + * @return An optional value potentially containing an error string if needed. Will be + * unset if the operation succeeded. */ - zeek::expected OpenBackend(const Tag& type, RecordValPtr options); + zeek::expected OpenBackend(const Tag& type, RecordValPtr options, TypePtr key_type, + TypePtr val_type); /** * Closes a storage backend. diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 326ae9e114..98ac5978c9 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -14,14 +14,16 @@ event Storage::backend_opened%(%); # Generated when a backend connection is lost event Storage::backend_lost%(%); -function Storage::__open_backend%(btype: Storage::Backend, options: any%): opaque of Storage::BackendHandle +function Storage::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): opaque of Storage::BackendHandle %{ auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; Tag tag{btype_val}; - auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; + auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); - auto b = storage_mgr->OpenBackend(tag, options_val); + auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; + auto b = storage_mgr->OpenBackend(tag, options_val, kt, vt); if ( ! b.has_value() ) { emit_builtin_error(b.error().c_str()); @@ -70,7 +72,7 @@ function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, va return val_mgr->Bool(true); %} -function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, val_type: any%): any +function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any%): any %{ auto b = dynamic_cast(backend); if ( ! b ) { @@ -83,8 +85,7 @@ function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, va // TODO: add support for when statements (see broker/store.bif) auto key_v = IntrusivePtr{NewRef{}, key}; - auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); - auto result = b->backend->Get(key_v, vt); + auto result = b->backend->Get(key_v); if ( ! result.has_value() ) { emit_builtin_error(util::fmt("Failed to retrieve data: %s", result.error().c_str())); return val_mgr->Bool(false); diff --git a/testing/btest/Baseline/plugins.storage/zeek-stderr b/testing/btest/Baseline/plugins.storage/zeek-stderr index 4963c5ee83..3ef9134f8a 100644 --- a/testing/btest/Baseline/plugins.storage/zeek-stderr +++ b/testing/btest/Baseline/plugins.storage/zeek-stderr @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/storage.zeek, line 37: Failed to retrieve data: Failed to find key (Storage::get(b, to_any_coerce key, to_any_coerce str)) -error in <...>/storage.zeek, line 50: Failed to open backend STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts)) +error in <...>/storage.zeek, line 37: Failed to retrieve data: Failed to find key (Storage::get(b, to_any_coerce key)) +error in <...>/storage.zeek, line 50: Failed to open backend STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts, to_any_coerce str, to_any_coerce str)) error in <...>/storage.zeek, line 51: Invalid storage handle (Storage::close_backend(b2) and F) diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc index 0f1c319fa4..eb78d7eace 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -46,13 +46,13 @@ zeek::storage::ErrorResult StorageDummy::DoPut(zeek::ValPtr key, zeek::ValPtr va /** * The workhorse method for Get(). This must be implemented for plugins. */ -zeek::storage::ValResult StorageDummy::DoGet(zeek::ValPtr key, zeek::TypePtr vt) { +zeek::storage::ValResult StorageDummy::DoGet(zeek::ValPtr key) { auto json_key = key->ToJSON(); auto it = data.find(json_key->ToStdString()); if ( it == data.end() ) return zeek::unexpected("Failed to find key"); - auto val = zeek::detail::ValFromJSON(it->second.c_str(), vt, zeek::Func::nil); + auto val = zeek::detail::ValFromJSON(it->second.c_str(), val_type, zeek::Func::nil); if ( std::holds_alternative(val) ) { zeek::ValPtr val_v = std::get(val); return val_v; diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h index cce7fb0293..fab247f1bf 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.h +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -41,7 +41,7 @@ public: /** * The workhorse method for Get(). */ - zeek::storage::ValResult DoGet(zeek::ValPtr key, zeek::TypePtr vt) override; + zeek::storage::ValResult DoGet(zeek::ValPtr key) override; /** * The workhorse method for Erase(). diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek index 8034d27cf4..a3b6e94421 100644 --- a/testing/btest/plugins/storage.zeek +++ b/testing/btest/plugins/storage.zeek @@ -26,20 +26,20 @@ event zeek_init() { # Test basic operation. The second get() should return an error # as the key should have been erased. - local b = Storage::open_backend(Storage::STORAGEDUMMY, opts); + local b = Storage::open_backend(Storage::STORAGEDUMMY, opts, str, str); local put_res = Storage::put(b, key, value, F); - local get_res = Storage::get(b, key, str); + local get_res = Storage::get(b, key); if ( get_res is bool ) { print("Got an invalid value in response!"); } local erase_res = Storage::erase(b, key); - get_res = Storage::get(b, key, str); + get_res = Storage::get(b, key); Storage::close_backend(b); # Test attempting to use the closed handle. put_res = Storage::put(b, "a", "b", F); - get_res = Storage::get(b, "a", str); + get_res = Storage::get(b, "a"); erase_res = Storage::erase(b, "a"); print(fmt("results of trying to use closed handle: get: %d, put: %d, erase: %d", @@ -47,6 +47,6 @@ event zeek_init() { # Test failing to open the handle and test closing an invalid handle. opts$open_fail = T; - local b2 = Storage::open_backend(Storage::STORAGEDUMMY, opts); + local b2 = Storage::open_backend(Storage::STORAGEDUMMY, opts, str, str); Storage::close_backend(b2); } From 8dee733a7d7cfd8fb5c188286ef51731a8ae9a8f Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 6 Dec 2024 14:28:13 -0700 Subject: [PATCH 06/52] Change args to Storage::put to be a record The number of args being passed to the put() methods was getting to be fairly long, with more on the horizon. Changing to a record means simplifying things a little bit. --- scripts/base/frameworks/storage/main.zeek | 28 +++++++++++++++-------- testing/btest/plugins/storage.zeek | 4 ++-- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index b575b823e9..8bfe4fd4e6 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -5,6 +5,19 @@ module Storage; export { + ## Record for passing arguments to :zeek:see:`Storage::put`. + type PutArgs: record { + # The key to store the value under. + key: any; + + # The value to store associated with the key. + value: any; + + # Indicates whether this value should overwrite an existing entry + # for the key. + overwrite: bool &default=T; + }; + ## Opens a new backend connection based on a configuration object. ## ## btype: A tag indicating what type of backend should be opened. These are @@ -33,20 +46,17 @@ export { ## Inserts a new entry into a backend. ## + ## ## backend: A handle to a backend connection. ## - ## key: A key value. - ## - ## value: A corresponding value. - ## - ## overwrite: A flag indicating whether this value should overwrite an existing - ## entry for the key. + ## args: A :zeek:see:`Storage::PutArgs` record containing the arguments for the + ## operation. ## ## Returns: A boolean indicating success or failure of the operation. Type ## comparison failures against the types passed to ## :zeek:see:`Storage::open_backend` for the backend will cause false to ## be returned. - global put: function(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool): bool; + global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool; ## Gets an entry from the backend. ## @@ -83,9 +93,9 @@ function close_backend(backend: opaque of Storage::BackendHandle): bool return Storage::__close_backend(backend); } -function put(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool): bool +function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool { - return Storage::__put(backend, key, value, overwrite); + return Storage::__put(backend, args$key, args$value, args$overwrite); } function get(backend: opaque of Storage::BackendHandle, key: any): any diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek index a3b6e94421..8f7ebabf46 100644 --- a/testing/btest/plugins/storage.zeek +++ b/testing/btest/plugins/storage.zeek @@ -27,7 +27,7 @@ event zeek_init() { # Test basic operation. The second get() should return an error # as the key should have been erased. local b = Storage::open_backend(Storage::STORAGEDUMMY, opts, str, str); - local put_res = Storage::put(b, key, value, F); + local put_res = Storage::put(b, [$key=key, $value=value, $overwrite=F]); local get_res = Storage::get(b, key); if ( get_res is bool ) { print("Got an invalid value in response!"); @@ -38,7 +38,7 @@ event zeek_init() { Storage::close_backend(b); # Test attempting to use the closed handle. - put_res = Storage::put(b, "a", "b", F); + put_res = Storage::put(b, [$key="a", $value="b", $overwrite=F]); get_res = Storage::get(b, "a"); erase_res = Storage::erase(b, "a"); From d07d27453af793a0944d4a83c2e8d672fe7fc054 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 6 Dec 2024 15:29:15 -0700 Subject: [PATCH 07/52] Add infrastructure for automated expiration of storage entries This is used for backends that don't support expiration natively. --- scripts/base/frameworks/storage/main.zeek | 6 ++- scripts/base/init-bare.zeek | 8 ++++ src/Timer.cc | 1 + src/Timer.h | 3 +- src/const.bif | 2 + src/storage/Backend.cc | 4 +- src/storage/Backend.h | 16 +++++-- src/storage/Manager.cc | 44 ++++++++++++++++--- src/storage/Manager.h | 22 ++++++++-- src/storage/storage.bif | 5 ++- .../storage-plugin/src/StorageDummy.cc | 3 +- .../plugins/storage-plugin/src/StorageDummy.h | 3 +- 12 files changed, 96 insertions(+), 21 deletions(-) diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index 8bfe4fd4e6..f6fe09b960 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -16,6 +16,10 @@ export { # Indicates whether this value should overwrite an existing entry # for the key. overwrite: bool &default=T; + + # An interval of time until the entry is automatically removed from the + # backend. + expire_time: interval &default=0sec; }; ## Opens a new backend connection based on a configuration object. @@ -95,7 +99,7 @@ function close_backend(backend: opaque of Storage::BackendHandle): bool function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool { - return Storage::__put(backend, args$key, args$value, args$overwrite); + return Storage::__put(backend, args$key, args$value, args$overwrite, args$expire_time); } function get(backend: opaque of Storage::BackendHandle, key: any): any diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 06fc0283d3..9593efd0f1 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -6210,6 +6210,14 @@ export { }; } +module Storage; + +export { + ## The interval used by the storage framework for automatic expiration + ## of elements in all backends that don't support it natively. + const expire_interval = 5.0 secs &redef; +} + module GLOBAL; @load base/bif/event.bif diff --git a/src/Timer.cc b/src/Timer.cc index 285e26df6a..70fa7f4ff7 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -51,6 +51,7 @@ const char* TimerNames[] = { "UnknownProtocolExpire", "LogDelayExpire", "LogFlushWriteBufferTimer", + "StorageExpire", }; const char* timer_type_to_string(TimerType type) { return TimerNames[type]; } diff --git a/src/Timer.h b/src/Timer.h index cf3f4f130d..509a6bec18 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -58,8 +58,9 @@ enum TimerType : uint8_t { TIMER_UNKNOWN_PROTOCOL_EXPIRE, TIMER_LOG_DELAY_EXPIRE, TIMER_LOG_FLUSH_WRITE_BUFFER, + TIMER_STORAGE_EXPIRE, }; -constexpr int NUM_TIMER_TYPES = int(TIMER_LOG_FLUSH_WRITE_BUFFER) + 1; +constexpr int NUM_TIMER_TYPES = int(TIMER_STORAGE_EXPIRE) + 1; extern const char* timer_type_to_string(TimerType type); diff --git a/src/const.bif b/src/const.bif index f2b0423ba6..822b0d30a6 100644 --- a/src/const.bif +++ b/src/const.bif @@ -32,3 +32,5 @@ const Threading::heartbeat_interval: interval; const Log::flush_interval: interval; const Log::write_buffer_size: count; + +const Storage::expire_interval: interval; diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index f064344d62..37e074eda3 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -14,7 +14,7 @@ ErrorResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt) { return DoOpen(std::move(options)); } -ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite) { +ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expiration_time) { // The intention for this method is to do some other heavy lifting in regard // to backends that need to pass data through the manager instead of directly // through the workers. For the first versions of the storage framework it @@ -26,7 +26,7 @@ ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite) { return util::fmt("type of value passed (%s) does not match backend's value type (%s)", obj_desc_short(value->GetType().get()).c_str(), val_type->GetName().c_str()); - return DoPut(std::move(key), std::move(value), overwrite); + return DoPut(std::move(key), std::move(value), overwrite, expiration_time); } ValResult Backend::Get(ValPtr key) { diff --git a/src/storage/Backend.h b/src/storage/Backend.h index dccff707ee..dd8aedc064 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -32,10 +32,12 @@ public: * @param key the key for the pair * @param value the value for the pair * @param overwrite whether an existing value for a key should be overwritten. - * @return A result pair containing a bool with the success state, and a - * possible error string if the operation failed. + * @param expiration_time the time when this entry should be automatically + * removed. Set to zero to disable expiration. + * @return An optional value potentially containing an error string if + * needed. Will be unset if the operation succeeded. */ - ErrorResult Put(ValPtr key, ValPtr value, bool overwrite = true); + ErrorResult Put(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0); /** * Retrieve a value from the backend for a provided key. @@ -99,7 +101,7 @@ protected: /** * The workhorse method for Put(). */ - virtual ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true) = 0; + virtual ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0) = 0; /** * The workhorse method for Get(). @@ -111,6 +113,12 @@ protected: */ virtual ErrorResult DoErase(ValPtr key) = 0; + /** + * Removes any entries in the backend that have expired. Can be overridden by + * derived classes. + */ + virtual void Expire() {} + TypePtr key_type; TypePtr val_type; diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index 5c24ff38eb..987f54f55c 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -4,11 +4,24 @@ #include "zeek/Desc.h" +#include "const.bif.netvar_h" + namespace zeek::storage { +void detail::ExpirationTimer::Dispatch(double t, bool is_expire) { + if ( is_expire ) + return; + + storage_mgr->Expire(); + storage_mgr->StartExpirationTimer(); +} + Manager::Manager() : plugin::ComponentManager("Storage", "Backend") {} -void Manager::InitPostScript() { detail::backend_opaque = make_intrusive("Storage::Backend"); } +void Manager::InitPostScript() { + detail::backend_opaque = make_intrusive("Storage::Backend"); + StartExpirationTimer(); +} zeek::expected Manager::OpenBackend(const Tag& type, RecordValPtr options, TypePtr key_type, TypePtr val_type) { @@ -40,20 +53,39 @@ zeek::expected Manager::OpenBackend(const Tag& type, Re // TODO: post Storage::backend_opened event - backends.push_back(bp); + { + std::unique_lock lk(backends_mtx); + backends.push_back(bp); + } return bp; } void Manager::CloseBackend(BackendPtr backend) { - auto it = std::find(backends.begin(), backends.end(), backend); - if ( it == backends.end() ) - return; + { + std::unique_lock lk(backends_mtx); + auto it = std::find(backends.begin(), backends.end(), backend); + if ( it == backends.end() ) + return; + + backends.erase(it); + } - backends.erase(it); backend->Close(); // TODO: post Storage::backend_lost event } +void Manager::Expire() { + DBG_LOG(DBG_STORAGE, "Expire running, have %zu backends to check", backends.size()); + std::unique_lock lk(backends_mtx); + for ( const auto& b : backends ) + b->Expire(); +} + +void Manager::StartExpirationTimer() { + zeek::detail::timer_mgr->Add( + new detail::ExpirationTimer(run_state::network_time + zeek::BifConst::Storage::expire_interval)); +} + } // namespace zeek::storage diff --git a/src/storage/Manager.h b/src/storage/Manager.h index c0545e53a8..49957a678d 100644 --- a/src/storage/Manager.h +++ b/src/storage/Manager.h @@ -2,12 +2,26 @@ #pragma once +#include + +#include "zeek/Timer.h" #include "zeek/plugin/ComponentManager.h" #include "zeek/storage/Backend.h" #include "zeek/storage/Component.h" namespace zeek::storage { +namespace detail { + +class ExpirationTimer final : public zeek::detail::Timer { +public: + ExpirationTimer(double t) : zeek::detail::Timer(t, zeek::detail::TIMER_STORAGE_EXPIRE) {} + ~ExpirationTimer() override {} + void Dispatch(double t, bool is_expire) override; +}; + +} // namespace detail + class Manager final : public plugin::ComponentManager { public: Manager(); @@ -40,12 +54,14 @@ public: */ void CloseBackend(BackendPtr backend); - // TODO: - // - Hooks for storage-backed tables? - // - Handling aggregation from workers on a single manager? +protected: + friend class storage::detail::ExpirationTimer; + void Expire(); + void StartExpirationTimer(); private: std::vector backends; + std::mutex backends_mtx; }; } // namespace zeek::storage diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 98ac5978c9..960084e666 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -49,7 +49,8 @@ function Storage::__close_backend%(backend: opaque of Storage::BackendHandle%) : return val_mgr->Bool(true); %} -function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool%): bool +function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, + overwrite: bool, expire_time: interval%): bool %{ auto b = dynamic_cast(backend); if ( ! b ) { @@ -63,7 +64,7 @@ function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, va auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; - auto result = b->backend->Put(key_v, val_v, overwrite); + auto result = b->backend->Put(key_v, val_v, overwrite, expire_time); if ( result.has_value() ) { emit_builtin_error(util::fmt("Failed to store data: %s", result.value().c_str())); return val_mgr->Bool(false); diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc index eb78d7eace..0d0fe83d3f 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -36,7 +36,8 @@ void StorageDummy::Close() { open = false; } /** * The workhorse method for Put(). This must be implemented by plugins. */ -zeek::storage::ErrorResult StorageDummy::DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite) { +zeek::storage::ErrorResult StorageDummy::DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite, + double expiration_time) { auto json_key = key->ToJSON()->ToStdString(); auto json_value = value->ToJSON()->ToStdString(); data[json_key] = json_value; diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h index fab247f1bf..1e2b6bc99e 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.h +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -36,7 +36,8 @@ public: /** * The workhorse method for Put(). */ - zeek::storage::ErrorResult DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite = true) override; + zeek::storage::ErrorResult DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite = true, + double expiration_time = 0) override; /** * The workhorse method for Get(). From 7ad6a05f5bba40e5ad19b50a3476c806d32a47f7 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 6 Dec 2024 16:05:23 -0700 Subject: [PATCH 08/52] Add infrastructure for asynchronous storage operations --- scripts/base/frameworks/storage/main.zeek | 44 +++++++--- src/storage/Backend.cc | 81 +++++++++++++++++-- src/storage/Backend.h | 70 +++++++++++++--- src/storage/storage.bif | 80 +++++++++++++++--- .../Baseline/plugins.storage/zeek-stderr | 2 +- .../storage-plugin/src/StorageDummy.cc | 6 +- .../plugins/storage-plugin/src/StorageDummy.h | 9 ++- testing/btest/plugins/storage.zeek | 8 +- 8 files changed, 251 insertions(+), 49 deletions(-) diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index f6fe09b960..e7fa684466 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -13,13 +13,19 @@ export { # The value to store associated with the key. value: any; - # Indicates whether this value should overwrite an existing entry - # for the key. + # Indicates whether this value should overwrite an existing entry for the + # key. overwrite: bool &default=T; # An interval of time until the entry is automatically removed from the # backend. expire_time: interval &default=0sec; + + # Indicates whether this operation should happen asynchronously. If this + # is true, the call to put must happen as part of a :zeek:see:`when` + # statement. This flag is overridden and set to F when reading pcaps, + # since time won't move forward the same as when caputring live traffic. + async_mode: bool &default=T; }; ## Opens a new backend connection based on a configuration object. @@ -58,7 +64,7 @@ export { ## ## Returns: A boolean indicating success or failure of the operation. Type ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause false to + ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to ## be returned. global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool; @@ -68,11 +74,18 @@ export { ## ## key: The key to look up. ## + ## async_mode: Indicates whether this operation should happen asynchronously. If + ## this is T, the call must happen as part of a :zeek:see:`when` + ## statement. This flag is overridden and set to F when reading pcaps, + ## since time won't move forward the same as when caputring live + ## traffic. + ## ## Returns: A boolean indicating success or failure of the operation. Type ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause false to + ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to ## be returned. - global get: function(backend: opaque of Storage::BackendHandle, key: any): any; + global get: function(backend: opaque of Storage::BackendHandle, key: any, + async_mode: bool &default=T): any; ## Erases an entry from the backend. ## @@ -80,11 +93,18 @@ export { ## ## key: The key to erase. ## + ## async_mode: Indicates whether this operation should happen asynchronously. If + ## this is T, the call must happen as part of a :zeek:see:`when` + ## statement. This flag is overridden and set to F when reading pcaps, + ## since time won't move forward the same as when caputring live + ## traffic. + ## ## Returns: A boolean indicating success or failure of the operation. Type ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause false to + ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to ## be returned. - global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; + global erase: function(backend: opaque of Storage::BackendHandle, key: any, + async_mode: bool &default=T): bool; } function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle @@ -99,15 +119,15 @@ function close_backend(backend: opaque of Storage::BackendHandle): bool function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool { - return Storage::__put(backend, args$key, args$value, args$overwrite, args$expire_time); + return Storage::__put(backend, args$key, args$value, args$overwrite, args$expire_time, args$async_mode); } -function get(backend: opaque of Storage::BackendHandle, key: any): any +function get(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T): any { - return Storage::__get(backend, key); + return Storage::__get(backend, key, async_mode); } -function erase(backend: opaque of Storage::BackendHandle, key: any): bool +function erase(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T): bool { - return Storage::__erase(backend, key); + return Storage::__erase(backend, key, async_mode); } diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index 37e074eda3..bf4e6b69d7 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -3,10 +3,58 @@ #include "zeek/storage/Backend.h" #include "zeek/Desc.h" +#include "zeek/RunState.h" +#include "zeek/Trigger.h" #include "zeek/broker/Data.h" namespace zeek::storage { +ErrorResultCallback::ErrorResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc) + : trigger(std::move(trigger)), assoc(assoc) {} +ErrorResultCallback::~ErrorResultCallback() {} + +void ErrorResultCallback::Complete(const ErrorResult& res) { + zeek::Val* result; + + if ( res ) + result = new StringVal(res.value()); + else + result = val_mgr->Bool(true).get(); + + trigger->Cache(assoc, result); + Unref(result); + trigger->Release(); +} + +void ErrorResultCallback::Timeout() { + auto v = make_intrusive("Timeout during request"); + trigger->Cache(assoc, v.get()); +} + +ValResultCallback::ValResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc) + : trigger(std::move(trigger)), assoc(assoc) {} +ValResultCallback::~ValResultCallback() {} + +void ValResultCallback::Complete(const ValResult& res) { + zeek::Val* result; + + if ( res ) { + result = res.value().get(); + Ref(result); + } + else + result = new StringVal(res.error()); + + trigger->Cache(assoc, result); + Unref(result); + trigger->Release(); +} + +void ValResultCallback::Timeout() { + auto v = make_intrusive("Timeout during request"); + trigger->Cache(assoc, v.get()); +} + ErrorResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt) { key_type = std::move(kt); val_type = std::move(vt); @@ -14,7 +62,7 @@ ErrorResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt) { return DoOpen(std::move(options)); } -ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expiration_time) { +ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expiration_time, ErrorResultCallback* cb) { // The intention for this method is to do some other heavy lifting in regard // to backends that need to pass data through the manager instead of directly // through the workers. For the first versions of the storage framework it @@ -26,25 +74,46 @@ ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expira return util::fmt("type of value passed (%s) does not match backend's value type (%s)", obj_desc_short(value->GetType().get()).c_str(), val_type->GetName().c_str()); - return DoPut(std::move(key), std::move(value), overwrite, expiration_time); + auto res = DoPut(std::move(key), std::move(value), overwrite, expiration_time, cb); + + if ( (! native_async || zeek::run_state::reading_traces) && cb ) { + cb->Complete(res); + delete cb; + } + + return res; } -ValResult Backend::Get(ValPtr key) { +ValResult Backend::Get(ValPtr key, ValResultCallback* cb) { // See the note in Put(). if ( ! same_type(key->GetType(), key_type) ) return zeek::unexpected(util::fmt("type of key passed (%s) does not match backend's key type (%s)", key->GetType()->GetName().c_str(), key_type->GetName().c_str())); - return DoGet(std::move(key)); + auto res = DoGet(std::move(key), cb); + + if ( (! native_async || zeek::run_state::reading_traces) && cb ) { + cb->Complete(res); + delete cb; + } + + return res; } -ErrorResult Backend::Erase(ValPtr key) { +ErrorResult Backend::Erase(ValPtr key, ErrorResultCallback* cb) { // See the note in Put(). if ( ! same_type(key->GetType(), key_type) ) return util::fmt("type of key passed (%s) does not match backend's key type (%s)", key->GetType()->GetName().c_str(), key_type->GetName().c_str()); - return DoErase(std::move(key)); + auto res = DoErase(std::move(key), cb); + + if ( (! native_async || zeek::run_state::reading_traces) && cb ) { + cb->Complete(res); + delete cb; + } + + return res; } zeek::OpaqueTypePtr detail::backend_opaque; diff --git a/src/storage/Backend.h b/src/storage/Backend.h index dd8aedc064..ea50b36c3d 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -6,6 +6,11 @@ #include "zeek/Val.h" #include "zeek/util.h" +namespace zeek::detail::trigger { +class Trigger; +using TriggerPtr = IntrusivePtr; +} // namespace zeek::detail::trigger + namespace zeek::storage { class Manager; @@ -19,6 +24,32 @@ using ErrorResult = std::optional; // string value will store an error message if the result is null. using ValResult = zeek::expected; +// A callback result that returns an ErrorResult. +class ErrorResultCallback { +public: + ErrorResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc); + ~ErrorResultCallback(); + void Complete(const ErrorResult& res); + void Timeout(); + +private: + zeek::detail::trigger::TriggerPtr trigger; + const void* assoc; +}; + +// A callback result that returns a ValResult. +class ValResultCallback { +public: + ValResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc); + ~ValResultCallback(); + void Complete(const ValResult& res); + void Timeout(); + +private: + zeek::detail::trigger::TriggerPtr trigger; + const void* assoc; +}; + class Backend : public zeek::Obj { public: /** @@ -29,39 +60,50 @@ public: /** * Store a new key/value pair in the backend. * - * @param key the key for the pair - * @param value the value for the pair + * @param key the key for the pair. + * @param value the value for the pair. * @param overwrite whether an existing value for a key should be overwritten. * @param expiration_time the time when this entry should be automatically * removed. Set to zero to disable expiration. + * @param cb An optional callback object if being called via an async context. * @return An optional value potentially containing an error string if - * needed. Will be unset if the operation succeeded. + * needed Will be unset if the operation succeeded. */ - ErrorResult Put(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0); + ErrorResult Put(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + ErrorResultCallback* cb = nullptr); /** * Retrieve a value from the backend for a provided key. * * @param key the key to lookup in the backend. + * @param cb An optional callback object if being called via an async context. * @return A std::expected containing either a valid ValPtr with the result * of the operation or a string containing an error message for failure. */ - ValResult Get(ValPtr key); + ValResult Get(ValPtr key, ValResultCallback* cb = nullptr); /** * Erases the value for a key from the backend. * + * @param key the key to erase + * @param cb An optional callback object if being called via an async context. * @return An optional value potentially containing an error string if * needed. Will be unset if the operation succeeded. * possible error string if the operation failed. */ - ErrorResult Erase(ValPtr key); + ErrorResult Erase(ValPtr key, ErrorResultCallback* cb = nullptr); /** * Returns whether the backend is opened. */ virtual bool IsOpen() = 0; + /** + * Returns whether the backend's connection supports asynchronous commands. + * Defaults to true, but can be overridden by backends. + */ + virtual bool SupportsAsync() { return true; } + protected: // Allow the manager to call Open/Close. friend class storage::Manager; @@ -69,10 +111,14 @@ protected: /** * Constructor * + * @param native_async Denotes whether this backend can handle async request + * natively. If set to false, the Put/Get/Erase methods will call the + * callback after their corresponding Do methods return. If set to true, the + * backend needs to call the callback itself. * @param tag A string representation of the tag for this backend. This * is passed from the Manager through the component factory. */ - Backend(std::string_view tag) : tag(tag) {} + Backend(bool native_async, std::string_view tag) : tag(tag), native_async(native_async) {} /** * Called by the manager system to open the backend. @@ -101,17 +147,18 @@ protected: /** * The workhorse method for Put(). */ - virtual ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0) = 0; + virtual ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + ErrorResultCallback* cb = nullptr) = 0; /** * The workhorse method for Get(). */ - virtual ValResult DoGet(ValPtr key) = 0; + virtual ValResult DoGet(ValPtr key, ValResultCallback* cb = nullptr) = 0; /** * The workhorse method for Erase(). */ - virtual ErrorResult DoErase(ValPtr key) = 0; + virtual ErrorResult DoErase(ValPtr key, ErrorResultCallback* cb = nullptr) = 0; /** * Removes any entries in the backend that have expired. Can be overridden by @@ -123,6 +170,9 @@ protected: TypePtr val_type; std::string tag; + +private: + bool native_async = false; }; using BackendPtr = zeek::IntrusivePtr; diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 960084e666..56a18a49aa 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -1,9 +1,35 @@ %%{ #include "zeek/storage/Backend.h" #include "zeek/storage/Manager.h" +#include "zeek/Trigger.h" +#include "zeek/Frame.h" using namespace zeek; using namespace zeek::storage; + +static zeek::detail::trigger::TriggerPtr init_trigger(zeek::detail::Frame* frame, const BackendPtr& b) { + if ( ! b->SupportsAsync() ) { + emit_builtin_error("Async mode requested but backend does not support async operations"); + return nullptr; + } + + auto trigger = frame->GetTrigger(); + + if ( ! trigger ) { + emit_builtin_error("Async storage operations can only be called inside when-conditions"); + return nullptr; + } + + if ( auto timeout = trigger->TimeoutValue(); timeout < 0 ) { + emit_builtin_error("Async Storage operations must specify a timeout block"); + return nullptr; + } + + frame->SetDelayed(); + trigger->Hold(); + + return {NewRef{}, trigger}; +} %%} module Storage; @@ -50,7 +76,7 @@ function Storage::__close_backend%(backend: opaque of Storage::BackendHandle%) : %} function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, - overwrite: bool, expire_time: interval%): bool + overwrite: bool, expire_time: interval, async_mode: bool &default=T%): bool %{ auto b = dynamic_cast(backend); if ( ! b ) { @@ -60,11 +86,23 @@ function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, va else if ( ! b->backend->IsOpen() ) return val_mgr->Bool(false); - // TODO: add support for when statements (see broker/store.bif) + ErrorResultCallback* cb = nullptr; + + if ( async_mode ) { + auto trigger = init_trigger(frame, b->backend); + if ( ! trigger ) + return val_mgr->Bool(false); + + cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); + } auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; - auto result = b->backend->Put(key_v, val_v, overwrite, expire_time); + auto result = b->backend->Put(key_v, val_v, overwrite, expire_time, cb); + + if ( async_mode ) + return nullptr; + if ( result.has_value() ) { emit_builtin_error(util::fmt("Failed to store data: %s", result.value().c_str())); return val_mgr->Bool(false); @@ -73,7 +111,7 @@ function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, va return val_mgr->Bool(true); %} -function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any%): any +function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T%): any %{ auto b = dynamic_cast(backend); if ( ! b ) { @@ -83,10 +121,22 @@ function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any%): else if ( ! b->backend->IsOpen() ) return val_mgr->Bool(false); - // TODO: add support for when statements (see broker/store.bif) + ValResultCallback* cb = nullptr; + + if ( async_mode ) { + auto trigger = init_trigger(frame, b->backend); + if ( ! trigger ) + return val_mgr->Bool(false); + + cb = new ValResultCallback(trigger, frame->GetTriggerAssoc()); + } auto key_v = IntrusivePtr{NewRef{}, key}; - auto result = b->backend->Get(key_v); + auto result = b->backend->Get(key_v, cb); + + if ( async_mode ) + return nullptr; + if ( ! result.has_value() ) { emit_builtin_error(util::fmt("Failed to retrieve data: %s", result.error().c_str())); return val_mgr->Bool(false); @@ -95,7 +145,7 @@ function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any%): return result.value(); %} -function Storage::__erase%(backend: opaque of Storage::BackendHandle, key: any%): bool +function Storage::__erase%(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T%): bool %{ auto b = dynamic_cast(backend); if ( ! b ) { @@ -105,10 +155,22 @@ function Storage::__erase%(backend: opaque of Storage::BackendHandle, key: any%) else if ( ! b->backend->IsOpen() ) return val_mgr->Bool(false); - // TODO: add support for when statements (see broker/store.bif) + ErrorResultCallback* cb = nullptr; + + if ( async_mode ) { + auto trigger = init_trigger(frame, b->backend); + if ( ! trigger ) + return val_mgr->Bool(false); + + cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); + } auto key_v = IntrusivePtr{NewRef{}, key}; - auto result = b->backend->Erase(key_v); + auto result = b->backend->Erase(key_v, cb); + + if ( async_mode ) + return nullptr; + if ( result.has_value() ) { emit_builtin_error(util::fmt("Failed to erase data for key: %s", result.value().c_str())); return val_mgr->Bool(false); diff --git a/testing/btest/Baseline/plugins.storage/zeek-stderr b/testing/btest/Baseline/plugins.storage/zeek-stderr index 3ef9134f8a..a1151e1362 100644 --- a/testing/btest/Baseline/plugins.storage/zeek-stderr +++ b/testing/btest/Baseline/plugins.storage/zeek-stderr @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/storage.zeek, line 37: Failed to retrieve data: Failed to find key (Storage::get(b, to_any_coerce key)) +error in <...>/storage.zeek, line 37: Failed to retrieve data: Failed to find key (Storage::get(b, to_any_coerce key, F)) error in <...>/storage.zeek, line 50: Failed to open backend STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts, to_any_coerce str, to_any_coerce str)) error in <...>/storage.zeek, line 51: Invalid storage handle (Storage::close_backend(b2) and F) diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc index 0d0fe83d3f..ce71dd5c1c 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -37,7 +37,7 @@ void StorageDummy::Close() { open = false; } * The workhorse method for Put(). This must be implemented by plugins. */ zeek::storage::ErrorResult StorageDummy::DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite, - double expiration_time) { + double expiration_time, zeek::storage::ErrorResultCallback* cb) { auto json_key = key->ToJSON()->ToStdString(); auto json_value = value->ToJSON()->ToStdString(); data[json_key] = json_value; @@ -47,7 +47,7 @@ zeek::storage::ErrorResult StorageDummy::DoPut(zeek::ValPtr key, zeek::ValPtr va /** * The workhorse method for Get(). This must be implemented for plugins. */ -zeek::storage::ValResult StorageDummy::DoGet(zeek::ValPtr key) { +zeek::storage::ValResult StorageDummy::DoGet(zeek::ValPtr key, zeek::storage::ValResultCallback* cb) { auto json_key = key->ToJSON(); auto it = data.find(json_key->ToStdString()); if ( it == data.end() ) @@ -65,7 +65,7 @@ zeek::storage::ValResult StorageDummy::DoGet(zeek::ValPtr key) { /** * The workhorse method for Erase(). This must be implemented for plugins. */ -zeek::storage::ErrorResult StorageDummy::DoErase(zeek::ValPtr key) { +zeek::storage::ErrorResult StorageDummy::DoErase(zeek::ValPtr key, zeek::storage::ErrorResultCallback* cb) { auto json_key = key->ToJSON(); auto it = data.find(json_key->ToStdString()); if ( it == data.end() ) diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h index 1e2b6bc99e..b770080dca 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.h +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -13,7 +13,7 @@ namespace btest::storage::backend { */ class StorageDummy : public zeek::storage::Backend { public: - StorageDummy(std::string_view tag) : Backend(tag) {} + StorageDummy(std::string_view tag) : Backend(false, tag) {} ~StorageDummy() override = default; static zeek::storage::BackendPtr Instantiate(std::string_view tag); @@ -37,17 +37,18 @@ public: * The workhorse method for Put(). */ zeek::storage::ErrorResult DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite = true, - double expiration_time = 0) override; + double expiration_time = 0, + zeek::storage::ErrorResultCallback* cb = nullptr) override; /** * The workhorse method for Get(). */ - zeek::storage::ValResult DoGet(zeek::ValPtr key) override; + zeek::storage::ValResult DoGet(zeek::ValPtr key, zeek::storage::ValResultCallback* cb = nullptr) override; /** * The workhorse method for Erase(). */ - zeek::storage::ErrorResult DoErase(zeek::ValPtr key) override; + zeek::storage::ErrorResult DoErase(zeek::ValPtr key, zeek::storage::ErrorResultCallback* cb = nullptr) override; private: std::map data; diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek index 8f7ebabf46..d759e4b7da 100644 --- a/testing/btest/plugins/storage.zeek +++ b/testing/btest/plugins/storage.zeek @@ -27,14 +27,14 @@ event zeek_init() { # Test basic operation. The second get() should return an error # as the key should have been erased. local b = Storage::open_backend(Storage::STORAGEDUMMY, opts, str, str); - local put_res = Storage::put(b, [$key=key, $value=value, $overwrite=F]); - local get_res = Storage::get(b, key); + local put_res = Storage::put(b, [$key=key, $value=value, $overwrite=F, $async_mode=F]); + local get_res = Storage::get(b, key, F); if ( get_res is bool ) { print("Got an invalid value in response!"); } - local erase_res = Storage::erase(b, key); - get_res = Storage::get(b, key); + local erase_res = Storage::erase(b, key, F); + get_res = Storage::get(b, key, F); Storage::close_backend(b); # Test attempting to use the closed handle. From 9d1eef3fbc51b92c4d9667db8a479220d699fc10 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sat, 27 Jan 2024 13:34:20 -0700 Subject: [PATCH 09/52] Add basic SQLite storage backend --- .../storage/backend/sqlite/__load__.zeek | 1 + .../storage/backend/sqlite/main.zeek | 22 +++ scripts/test-all-policy.zeek | 2 + src/storage/backend/CMakeLists.txt | 2 +- src/storage/backend/sqlite/CMakeLists.txt | 3 + src/storage/backend/sqlite/Plugin.cc | 22 +++ src/storage/backend/sqlite/SQLite.cc | 176 ++++++++++++++++++ src/storage/backend/sqlite/SQLite.h | 61 ++++++ .../Baseline/language.expire_func-copy/output | 18 +- .../.stderr | 2 + .../out | 8 + .../.stderr | 3 + .../out | 1 + .../base/frameworks/storage/sqlite-basic.zeek | 56 ++++++ .../storage/sqlite-error-handling.zeek | 21 +++ 15 files changed, 388 insertions(+), 10 deletions(-) create mode 100644 scripts/policy/frameworks/storage/backend/sqlite/__load__.zeek create mode 100644 scripts/policy/frameworks/storage/backend/sqlite/main.zeek create mode 100644 src/storage/backend/sqlite/CMakeLists.txt create mode 100644 src/storage/backend/sqlite/Plugin.cc create mode 100644 src/storage/backend/sqlite/SQLite.cc create mode 100644 src/storage/backend/sqlite/SQLite.h create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out create mode 100644 testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek create mode 100644 testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek diff --git a/scripts/policy/frameworks/storage/backend/sqlite/__load__.zeek b/scripts/policy/frameworks/storage/backend/sqlite/__load__.zeek new file mode 100644 index 0000000000..6086306018 --- /dev/null +++ b/scripts/policy/frameworks/storage/backend/sqlite/__load__.zeek @@ -0,0 +1 @@ +@load ./main.zeek \ No newline at end of file diff --git a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek new file mode 100644 index 0000000000..100d97239e --- /dev/null +++ b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek @@ -0,0 +1,22 @@ +##! SQLite storage backend support + +@load base/frameworks/storage/main + +module Storage::Backend::SQLite; + +export { + ## Options record for the built-in SQLite backend. + type Options: record { + ## Path to the database file on disk. Setting this to ":memory:" + ## will tell SQLite to use an in-memory database. Relative paths + ## will be opened relative to the directory where Zeek was + ## started from. Zeek will not create intermediate directories + ## if they do not already exist. See + ## https://www.sqlite.org/c3ref/open.html for more rules on + ## paths that can be passed here. + database_path: string; + + ## Name of the table used for storing data. + table_name: string; + }; +} diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 79db7e235b..34968b2ed6 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -83,6 +83,8 @@ # @load frameworks/spicy/record-spicy-batch.zeek # @load frameworks/spicy/resource-usage.zeek @load frameworks/software/windows-version-detection.zeek +@load frameworks/storage/backend/sqlite/__load__.zeek +@load frameworks/storage/backend/sqlite/main.zeek @load frameworks/telemetry/log.zeek @load integration/collective-intel/__load__.zeek @load integration/collective-intel/main.zeek diff --git a/src/storage/backend/CMakeLists.txt b/src/storage/backend/CMakeLists.txt index 8b13789179..a5af49edbb 100644 --- a/src/storage/backend/CMakeLists.txt +++ b/src/storage/backend/CMakeLists.txt @@ -1 +1 @@ - +add_subdirectory(sqlite) diff --git a/src/storage/backend/sqlite/CMakeLists.txt b/src/storage/backend/sqlite/CMakeLists.txt new file mode 100644 index 0000000000..56ba56b1ed --- /dev/null +++ b/src/storage/backend/sqlite/CMakeLists.txt @@ -0,0 +1,3 @@ +zeek_add_plugin( + Zeek Storage_Backend_SQLite + SOURCES SQLite.cc Plugin.cc) diff --git a/src/storage/backend/sqlite/Plugin.cc b/src/storage/backend/sqlite/Plugin.cc new file mode 100644 index 0000000000..f6d56e77fb --- /dev/null +++ b/src/storage/backend/sqlite/Plugin.cc @@ -0,0 +1,22 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/plugin/Plugin.h" + +#include "zeek/storage/Component.h" +#include "zeek/storage/backend/sqlite/SQLite.h" + +namespace zeek::storage::backend::sqlite { + +class Plugin : public plugin::Plugin { +public: + plugin::Configuration Configure() override { + AddComponent(new storage::Component("SQLITE", backend::sqlite::SQLite::Instantiate)); + + plugin::Configuration config; + config.name = "Zeek::Storage_Backend_SQLite"; + config.description = "SQLite backend for storage framework"; + return config; + } +} plugin; + +} // namespace zeek::storage::backend::sqlite diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc new file mode 100644 index 0000000000..8c593a69b8 --- /dev/null +++ b/src/storage/backend/sqlite/SQLite.cc @@ -0,0 +1,176 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/backend/sqlite/SQLite.h" + +#include "zeek/3rdparty/sqlite3.h" +#include "zeek/Func.h" +#include "zeek/Val.h" + +namespace zeek::storage::backend::sqlite { + +storage::BackendPtr SQLite::Instantiate(std::string_view tag) { return make_intrusive(tag); } + +/** + * Called by the manager system to open the backend. + */ +ErrorResult SQLite::DoOpen(RecordValPtr options) { + if ( sqlite3_threadsafe() == 0 ) { + std::string res = + "SQLite reports that it is not threadsafe. Zeek needs a threadsafe version of " + "SQLite. Aborting"; + Error(res.c_str()); + return res; + } + + // Allow connections to same DB to use single data/schema cache. Also + // allows simultaneous writes to one file. +#ifndef ZEEK_TSAN + sqlite3_enable_shared_cache(1); +#endif + + StringValPtr path = options->GetField("database_path"); + full_path = zeek::filesystem::path(path->ToStdString()).string(); + table_name = options->GetField("table_name")->ToStdString(); + + auto open_res = + checkError(sqlite3_open_v2(full_path.c_str(), &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, NULL)); + if ( open_res.has_value() ) { + sqlite3_close_v2(db); + db = nullptr; + return open_res; + } + + std::string create = "create table if not exists " + table_name + " ("; + create.append("key_str text primary key, value_str text not null);"); + + char* errorMsg = nullptr; + int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); + if ( res != SQLITE_OK ) { + std::string err = util::fmt("Error executing table creation statement: %s", errorMsg); + Error(err.c_str()); + sqlite3_free(errorMsg); + sqlite3_close(db); + db = nullptr; + return err; + } + + return std::nullopt; +} + +/** + * Finalizes the backend when it's being closed. + */ +void SQLite::Close() { + if ( db ) { + if ( int res = sqlite3_close_v2(db); res != SQLITE_OK ) + Error("Sqlite could not close connection"); + + db = nullptr; + } +} + +/** + * The workhorse method for Put(). This must be implemented by plugins. + */ +ErrorResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, ErrorResultCallback* cb) { + if ( ! db ) + return "Database was not open"; + + auto json_key = key->ToJSON(); + auto json_value = value->ToJSON(); + + std::string stmt = "INSERT INTO "; + stmt.append(table_name); + stmt.append("(key_str, value_str) VALUES('"); + stmt.append(json_key->ToStdStringView()); + stmt.append("', '"); + stmt.append(json_value->ToStdStringView()); + if ( ! overwrite ) + stmt.append("');"); + else { + // if overwriting, add an UPSERT conflict resolution block + stmt.append("') ON CONFLICT(key_str) DO UPDATE SET value_str='"); + stmt.append(json_value->ToStdStringView()); + stmt.append("';"); + } + + char* errorMsg = nullptr; + int res = sqlite3_exec(db, stmt.c_str(), NULL, NULL, &errorMsg); + if ( res != SQLITE_OK ) { + return errorMsg; + } + + return std::nullopt; +} + +/** + * The workhorse method for Get(). This must be implemented for plugins. + */ +ValResult SQLite::DoGet(ValPtr key, ValResultCallback* cb) { + if ( ! db ) + return zeek::unexpected("Database was not open"); + + auto json_key = key->ToJSON(); + + std::string stmt = "SELECT value_str from " + table_name + " where key_str = '"; + stmt.append(json_key->ToStdStringView()); + stmt.append("';"); + + char* errorMsg = nullptr; + sqlite3_stmt* st; + auto res = checkError(sqlite3_prepare_v2(db, stmt.c_str(), static_cast(stmt.size() + 1), &st, NULL)); + if ( res.has_value() ) + return zeek::unexpected(util::fmt("Failed to prepare select statement: %s", res.value().c_str())); + + int errorcode = sqlite3_step(st); + if ( errorcode == SQLITE_ROW ) { + // Column 1 is the value + const char* text = (const char*)sqlite3_column_text(st, 0); + auto val = zeek::detail::ValFromJSON(text, val_type, Func::nil); + if ( std::holds_alternative(val) ) { + ValPtr val_v = std::get(val); + return val_v; + } + else { + return zeek::unexpected(std::get(val)); + } + } + + return zeek::unexpected(util::fmt("Failed to find row for key: %s", sqlite3_errstr(errorcode))); +} + +/** + * The workhorse method for Erase(). This must be implemented for plugins. + */ +ErrorResult SQLite::DoErase(ValPtr key, ErrorResultCallback* cb) { + if ( ! db ) + return "Database was not open"; + + auto json_key = key->ToJSON(); + + std::string stmt = "DELETE from " + table_name + " where key_str = \'"; + stmt.append(json_key->ToStdStringView()); + stmt.append("\'"); + + char* errorMsg = nullptr; + int res = sqlite3_exec(db, stmt.c_str(), NULL, NULL, &errorMsg); + if ( res != SQLITE_OK ) { + return errorMsg; + } + + return std::nullopt; +} + +// returns true in case of error +ErrorResult SQLite::checkError(int code) { + if ( code != SQLITE_OK && code != SQLITE_DONE ) { + std::string msg = util::fmt("SQLite call failed: %s", sqlite3_errmsg(db)); + Error(msg.c_str()); + return msg; + } + + return std::nullopt; +} + +} // namespace zeek::storage::backend::sqlite diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h new file mode 100644 index 0000000000..4aca4e1439 --- /dev/null +++ b/src/storage/backend/sqlite/SQLite.h @@ -0,0 +1,61 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/storage/Backend.h" + +// Forward declare these to avoid including sqlite3.h here +struct sqlite3; +struct sqlite3_stmt; + +namespace zeek::storage::backend::sqlite { + +class SQLite : public Backend { +public: + SQLite(std::string_view tag) : Backend(false, tag) {} + ~SQLite() override = default; + + static BackendPtr Instantiate(std::string_view tag); + + /** + * Called by the manager system to open the backend. + */ + ErrorResult DoOpen(RecordValPtr options) override; + + /** + * Finalizes the backend when it's being closed. + */ + void Close() override; + + /** + * Returns whether the backend is opened. + */ + bool IsOpen() override { return db != nullptr; } + + /** + * The workhorse method for Put(). + */ + ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + ErrorResultCallback* cb = nullptr) override; + + /** + * The workhorse method for Get(). + */ + ValResult DoGet(ValPtr key, ValResultCallback* cb = nullptr) override; + + /** + * The workhorse method for Erase(). + */ + ErrorResult DoErase(ValPtr key, ErrorResultCallback* cb = nullptr) override; + + // TODO: add support for checking for expired data + +private: + ErrorResult checkError(int code); + + sqlite3* db = nullptr; + std::string full_path; + std::string table_name; +}; + +} // namespace zeek::storage::backend::sqlite diff --git a/testing/btest/Baseline/language.expire_func-copy/output b/testing/btest/Baseline/language.expire_func-copy/output index 60db90a1b2..17b843d3f7 100644 --- a/testing/btest/Baseline/language.expire_func-copy/output +++ b/testing/btest/Baseline/language.expire_func-copy/output @@ -1,5 +1,14 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. @XXXXXXXXXX.XXXXXX expired a +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49656/tcp, resp_h=172.16.238.131, resp_p=22/tcp, proto=6] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49658/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=17500/udp, resp_h=172.16.238.255, resp_p=17500/udp, proto=17] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49657/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=37975/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] +@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] +@XXXXXXXXXX.XXXXXX expired [orig_h=fe80::20c:29ff:febd:6f01, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp, proto=17] +@XXXXXXXXXX.XXXXXX expired a @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] @XXXXXXXXXX.XXXXXX expired copy [orig_h=fe80::20c:29ff:febd:6f01, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp, proto=17] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49657/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] @@ -9,15 +18,6 @@ @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49658/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=37975/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired a -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49656/tcp, resp_h=172.16.238.131, resp_p=22/tcp, proto=6] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49658/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=17500/udp, resp_h=172.16.238.255, resp_p=17500/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49657/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=37975/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired [orig_h=fe80::20c:29ff:febd:6f01, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp, proto=17] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49659/tcp, resp_h=172.16.238.131, resp_p=21/tcp, proto=6] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=45126/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] @XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=45126/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr new file mode 100644 index 0000000000..d172ece6f0 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in /Users/tim/Desktop/projects/storage-framework/testing/btest/.tmp/scripts.base.frameworks.storage.sqlite-basic/sqlite-basic.zeek, line 42: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(b, to_any_coerce key, F)) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out new file mode 100644 index 0000000000..c54d0a144d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +T +value +T +value2 +T +got empty result +value2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr new file mode 100644 index 0000000000..6ed2d3c5d8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in : SQLite call failed: unable to open database file () +error in <...>/sqlite-error-handling.zeek, line 20: Failed to open backend SQLITE: SQLite call failed: unable to open database file (Storage::open_backend(Storage::SQLITE, to_any_coerce opts, to_any_coerce str, to_any_coerce str)) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek new file mode 100644 index 0000000000..c28bb5041b --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -0,0 +1,56 @@ +# @TEST-DOC: Basic functionality for storage: opening/closing an sqlite backend, storing/retrieving/erasing basic data +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff .stderr + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/sqlite + +# Create a typename here that can be passed down into open_backend. +type str: string; + +event zeek_init() { + # Create a database file in the .tmp directory with a 'testing' table + local opts : Storage::Backend::SQLite::Options; + opts$database_path = "test.sqlite"; + opts$table_name = "testing"; + + local key = "key1111"; + local value = "value"; + + # Test inserting/retrieving a key/value pair that we know won't be in + # the backend yet. + local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + local res = Storage::put(b, [$key=key, $value=value, $overwrite=T, $async_mode=F]); + print res; + + local res2 = Storage::get(b, key, F); + print res2; + + # Test overwriting a value with put() + local value2 = "value2"; + local res3 = Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]); + print res3; + + local res4 = Storage::get(b, key, F); + print res4; + + # Test erasing a key and getting a "false" result + local res5 = Storage::erase(b, key, F); + print res5; + + local res6 = Storage::get(b, key, F); + if ( ! res6 as bool ) { + print "got empty result"; + } + + # Insert something back into the database to test reopening + Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]); + + Storage::close_backend(b); + + # Test reopening the same database and getting the data back out of it + local b2 = Storage::open_backend(Storage::SQLITE, opts, str, str); + local res7 = Storage::get(b2, key, F); + print res7; +} diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek new file mode 100644 index 0000000000..159972c48b --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek @@ -0,0 +1,21 @@ +# @TEST-DOC: Tests various error handling scenarios for the storage framework +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +@load base/frameworks/storage +@load base/frameworks/reporter +@load policy/frameworks/storage/backend/sqlite + +# Create a typename here that can be passed down into open_backend. +type str: string; + +event zeek_init() { + # Test opening a database with an invalid path + local opts : Storage::Backend::SQLite::Options; + opts$database_path = "/this/path/should/not/exist/test.sqlite"; + opts$table_name = "testing"; + + # This should report an error in .stderr and reporter.log + local b = Storage::open_backend(Storage::SQLITE, opts, str, str); +} From e95784db16560e3a106893698c239a17f0825ae2 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 9 Dec 2024 14:38:06 -0700 Subject: [PATCH 10/52] SQLite: Store/lookup prepared statements instead of recreating --- src/storage/backend/sqlite/SQLite.cc | 118 ++++++++++++------ src/storage/backend/sqlite/SQLite.h | 2 + .../.stderr | 1 - 3 files changed, 79 insertions(+), 42 deletions(-) diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index 8c593a69b8..79b9328981 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -45,16 +45,33 @@ ErrorResult SQLite::DoOpen(RecordValPtr options) { create.append("key_str text primary key, value_str text not null);"); char* errorMsg = nullptr; - int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); - if ( res != SQLITE_OK ) { + if ( int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); res != SQLITE_OK ) { std::string err = util::fmt("Error executing table creation statement: %s", errorMsg); Error(err.c_str()); sqlite3_free(errorMsg); - sqlite3_close(db); - db = nullptr; + Close(); return err; } + static std::map statements = + {{"put", util::fmt("insert into %s (key_str, value_str) values(?, ?)", table_name.c_str())}, + {"put_update", util::fmt("insert into %s (key_str, value_str) values(?, ?) ON CONFLICT(key_str) " + "DO UPDATE SET value_str=?", + table_name.c_str())}, + {"get", util::fmt("select value_str from %s where key_str=?", table_name.c_str())}, + {"erase", util::fmt("delete from %s where key_str=?", table_name.c_str())}}; + + for ( const auto& [key, stmt] : statements ) { + sqlite3_stmt* ps; + if ( auto prep_res = checkError(sqlite3_prepare_v2(db, stmt.c_str(), stmt.size(), &ps, NULL)); + prep_res.has_value() ) { + Close(); + return prep_res; + } + + prepared_stmts.insert({key, ps}); + } + return std::nullopt; } @@ -63,6 +80,12 @@ ErrorResult SQLite::DoOpen(RecordValPtr options) { */ void SQLite::Close() { if ( db ) { + for ( const auto& [k, stmt] : prepared_stmts ) { + sqlite3_finalize(stmt); + } + + prepared_stmts.clear(); + if ( int res = sqlite3_close_v2(db); res != SQLITE_OK ) Error("Sqlite could not close connection"); @@ -80,27 +103,41 @@ ErrorResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expir auto json_key = key->ToJSON(); auto json_value = value->ToJSON(); - std::string stmt = "INSERT INTO "; - stmt.append(table_name); - stmt.append("(key_str, value_str) VALUES('"); - stmt.append(json_key->ToStdStringView()); - stmt.append("', '"); - stmt.append(json_value->ToStdStringView()); + sqlite3_stmt* stmt; if ( ! overwrite ) - stmt.append("');"); - else { - // if overwriting, add an UPSERT conflict resolution block - stmt.append("') ON CONFLICT(key_str) DO UPDATE SET value_str='"); - stmt.append(json_value->ToStdStringView()); - stmt.append("';"); + stmt = prepared_stmts["put"]; + else + stmt = prepared_stmts["put_update"]; + + auto key_str = json_key->ToStdStringView(); + if ( auto res = checkError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); + res.has_value() ) { + sqlite3_reset(stmt); + return res; } - char* errorMsg = nullptr; - int res = sqlite3_exec(db, stmt.c_str(), NULL, NULL, &errorMsg); - if ( res != SQLITE_OK ) { - return errorMsg; + auto value_str = json_value->ToStdStringView(); + if ( auto res = checkError(sqlite3_bind_text(stmt, 2, value_str.data(), value_str.size(), SQLITE_STATIC)); + res.has_value() ) { + sqlite3_reset(stmt); + return res; } + if ( overwrite ) { + if ( auto res = checkError(sqlite3_bind_text(stmt, 3, value_str.data(), value_str.size(), SQLITE_STATIC)); + res.has_value() ) { + sqlite3_reset(stmt); + return res; + } + } + + if ( auto res = checkError(sqlite3_step(stmt)); res.has_value() ) { + sqlite3_reset(stmt); + return res; + } + + sqlite3_reset(stmt); + return std::nullopt; } @@ -112,22 +149,21 @@ ValResult SQLite::DoGet(ValPtr key, ValResultCallback* cb) { return zeek::unexpected("Database was not open"); auto json_key = key->ToJSON(); + auto stmt = prepared_stmts["get"]; - std::string stmt = "SELECT value_str from " + table_name + " where key_str = '"; - stmt.append(json_key->ToStdStringView()); - stmt.append("';"); + auto key_str = json_key->ToStdStringView(); + if ( auto res = checkError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); + res.has_value() ) { + sqlite3_reset(stmt); + return nonstd::unexpected(res.value()); + } - char* errorMsg = nullptr; - sqlite3_stmt* st; - auto res = checkError(sqlite3_prepare_v2(db, stmt.c_str(), static_cast(stmt.size() + 1), &st, NULL)); - if ( res.has_value() ) - return zeek::unexpected(util::fmt("Failed to prepare select statement: %s", res.value().c_str())); - - int errorcode = sqlite3_step(st); + int errorcode = sqlite3_step(stmt); if ( errorcode == SQLITE_ROW ) { // Column 1 is the value - const char* text = (const char*)sqlite3_column_text(st, 0); + const char* text = (const char*)sqlite3_column_text(stmt, 0); auto val = zeek::detail::ValFromJSON(text, val_type, Func::nil); + sqlite3_reset(stmt); if ( std::holds_alternative(val) ) { ValPtr val_v = std::get(val); return val_v; @@ -148,15 +184,17 @@ ErrorResult SQLite::DoErase(ValPtr key, ErrorResultCallback* cb) { return "Database was not open"; auto json_key = key->ToJSON(); + auto stmt = prepared_stmts["erase"]; - std::string stmt = "DELETE from " + table_name + " where key_str = \'"; - stmt.append(json_key->ToStdStringView()); - stmt.append("\'"); + auto key_str = json_key->ToStdStringView(); + if ( auto res = checkError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); + res.has_value() ) { + sqlite3_reset(stmt); + return res; + } - char* errorMsg = nullptr; - int res = sqlite3_exec(db, stmt.c_str(), NULL, NULL, &errorMsg); - if ( res != SQLITE_OK ) { - return errorMsg; + if ( auto res = checkError(sqlite3_step(stmt)); res.has_value() ) { + return res; } return std::nullopt; @@ -165,9 +203,7 @@ ErrorResult SQLite::DoErase(ValPtr key, ErrorResultCallback* cb) { // returns true in case of error ErrorResult SQLite::checkError(int code) { if ( code != SQLITE_OK && code != SQLITE_DONE ) { - std::string msg = util::fmt("SQLite call failed: %s", sqlite3_errmsg(db)); - Error(msg.c_str()); - return msg; + return util::fmt("SQLite call failed: %s", sqlite3_errmsg(db)); } return std::nullopt; diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index 4aca4e1439..7899d7108b 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -54,6 +54,8 @@ private: ErrorResult checkError(int code); sqlite3* db = nullptr; + std::unordered_map prepared_stmts; + std::string full_path; std::string table_name; }; diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr index 6ed2d3c5d8..4cb5a5b2d9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr @@ -1,3 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in : SQLite call failed: unable to open database file () error in <...>/sqlite-error-handling.zeek, line 20: Failed to open backend SQLITE: SQLite call failed: unable to open database file (Storage::open_backend(Storage::SQLITE, to_any_coerce opts, to_any_coerce str, to_any_coerce str)) From ec49f5d550f4e0c0a0e79fe2cec03ba6cc3d470e Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 12 Aug 2024 13:23:24 -0700 Subject: [PATCH 11/52] SQLite: Handle automated expiration --- src/storage/backend/sqlite/SQLite.cc | 41 ++++++++++++++---- src/storage/backend/sqlite/SQLite.h | 6 ++- .../.stderr | 3 ++ .../out | 5 +++ .../base/frameworks/storage/expiration.zeek | 42 +++++++++++++++++++ 5 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out create mode 100644 testing/btest/scripts/base/frameworks/storage/expiration.zeek diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index 79b9328981..e4fc4afd9b 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -42,7 +42,7 @@ ErrorResult SQLite::DoOpen(RecordValPtr options) { } std::string create = "create table if not exists " + table_name + " ("; - create.append("key_str text primary key, value_str text not null);"); + create.append("key_str text primary key, value_str text not null, expire_time real);"); char* errorMsg = nullptr; if ( int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); res != SQLITE_OK ) { @@ -54,12 +54,15 @@ ErrorResult SQLite::DoOpen(RecordValPtr options) { } static std::map statements = - {{"put", util::fmt("insert into %s (key_str, value_str) values(?, ?)", table_name.c_str())}, - {"put_update", util::fmt("insert into %s (key_str, value_str) values(?, ?) ON CONFLICT(key_str) " - "DO UPDATE SET value_str=?", - table_name.c_str())}, + {{"put", util::fmt("insert into %s (key_str, value_str, expire_time) values(?, ?, ?)", table_name.c_str())}, + {"put_update", + util::fmt("insert into %s (key_str, value_str, expire_time) values(?, ?, ?) ON CONFLICT(key_str) " + "DO UPDATE SET value_str=?", + table_name.c_str())}, {"get", util::fmt("select value_str from %s where key_str=?", table_name.c_str())}, - {"erase", util::fmt("delete from %s where key_str=?", table_name.c_str())}}; + {"erase", util::fmt("delete from %s where key_str=?", table_name.c_str())}, + {"expire", util::fmt("delete from %s where expire_time > 0 and expire_time != 0 and expire_time <= ?", + table_name.c_str())}}; for ( const auto& [key, stmt] : statements ) { sqlite3_stmt* ps; @@ -123,8 +126,13 @@ ErrorResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expir return res; } + if ( auto res = checkError(sqlite3_bind_double(stmt, 3, expiration_time)); res.has_value() ) { + sqlite3_reset(stmt); + return res; + } + if ( overwrite ) { - if ( auto res = checkError(sqlite3_bind_text(stmt, 3, value_str.data(), value_str.size(), SQLITE_STATIC)); + if ( auto res = checkError(sqlite3_bind_text(stmt, 4, value_str.data(), value_str.size(), SQLITE_STATIC)); res.has_value() ) { sqlite3_reset(stmt); return res; @@ -155,7 +163,7 @@ ValResult SQLite::DoGet(ValPtr key, ValResultCallback* cb) { if ( auto res = checkError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); res.has_value() ) { sqlite3_reset(stmt); - return nonstd::unexpected(res.value()); + return zeek::unexpected(res.value()); } int errorcode = sqlite3_step(stmt); @@ -200,6 +208,23 @@ ErrorResult SQLite::DoErase(ValPtr key, ErrorResultCallback* cb) { return std::nullopt; } +/** + * Removes any entries in the backend that have expired. Can be overridden by + * derived classes. + */ +void SQLite::Expire() { + auto stmt = prepared_stmts["expire"]; + + if ( auto res = checkError(sqlite3_bind_double(stmt, 1, util::current_time())); res.has_value() ) { + sqlite3_reset(stmt); + // TODO: do something with the error here? + } + + if ( auto res = checkError(sqlite3_step(stmt)); res.has_value() ) { + // TODO: do something with the error here? + } +} + // returns true in case of error ErrorResult SQLite::checkError(int code) { if ( code != SQLITE_OK && code != SQLITE_DONE ) { diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index 7899d7108b..1a3a713909 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -48,7 +48,11 @@ public: */ ErrorResult DoErase(ValPtr key, ErrorResultCallback* cb = nullptr) override; - // TODO: add support for checking for expired data + /** + * Removes any entries in the backend that have expired. Can be overridden by + * derived classes. + */ + void Expire() override; private: ErrorResult checkError(int code); diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr new file mode 100644 index 0000000000..0494e4d891 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in /Users/tim/Desktop/projects/storage-framework/testing/btest/.tmp/scripts.base.frameworks.storage.expiration/expiration.zeek, line 20: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(backend, to_any_coerce key, F)) +received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out new file mode 100644 index 0000000000..f75c9fb9cc --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, value7890 +get result same as inserted, T +get result, F diff --git a/testing/btest/scripts/base/frameworks/storage/expiration.zeek b/testing/btest/scripts/base/frameworks/storage/expiration.zeek new file mode 100644 index 0000000000..30caade463 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/expiration.zeek @@ -0,0 +1,42 @@ +# @TEST-DOC: Automatic expiration of stored data +# @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b %INPUT > out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff .stderr + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/sqlite + +redef Storage::expire_interval = 5 secs; +redef exit_only_after_terminate = T; + +# Create a typename here that can be passed down into get(). +type str: string; + +global backend: opaque of Storage::BackendHandle; +global key: string = "key1234"; +global value: string = "value7890"; + +event check_removed() { + local res2 = Storage::get(backend, key, F); + print "get result", res2; + + Storage::close_backend(backend); + terminate(); +} + +event zeek_init() { + local opts : Storage::Backend::SQLite::Options; + opts$database_path = "storage-test.sqlite"; + opts$table_name = "testing"; + + backend = Storage::open_backend(Storage::SQLITE, opts, str, str); + + local res = Storage::put(backend, [$key=key, $value=value, $overwrite=T, $expire_time=2 secs, $async_mode=F]); + print "put result", res; + + local res2 = Storage::get(backend, key, F); + print "get result", res2; + print "get result same as inserted", value == (res2 as string); + + schedule 5 secs { check_removed() }; +} From 3e8ff836aac6097079cc81a403bbd0965682c887 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 19 Aug 2024 17:27:12 -0700 Subject: [PATCH 12/52] SQLite: Add tuning options to configuration --- .../storage/backend/sqlite/main.zeek | 9 ++++++++ src/storage/backend/sqlite/SQLite.cc | 21 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek index 100d97239e..6cab5911f2 100644 --- a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek +++ b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek @@ -18,5 +18,14 @@ export { ## Name of the table used for storing data. table_name: string; + + ## Key/value table for passing tuning parameters when opening + ## the database. These must be pairs that can be passed to the + ## ``pragma`` command in sqlite. + tuning_params: table[string] of string &default=table( + ["journal_mode"] = "WAL", + ["synchronous"] = "normal", + ["temp_store"] = "memory" + ); }; } diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index e4fc4afd9b..e166e861c8 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -53,6 +53,21 @@ ErrorResult SQLite::DoOpen(RecordValPtr options) { return err; } + auto tuning_params = options->GetField("tuning_params")->ToMap(); + for ( const auto& [k, v] : tuning_params ) { + auto ks = k->AsListVal()->Idx(0)->AsStringVal(); + auto vs = v->AsStringVal(); + std::string cmd = util::fmt("pragma %s = %s", ks->ToStdStringView().data(), vs->ToStdStringView().data()); + + if ( int res = sqlite3_exec(db, cmd.c_str(), NULL, NULL, &errorMsg); res != SQLITE_OK ) { + std::string err = util::fmt("Error executing tuning pragma statement: %s", errorMsg); + Error(err.c_str()); + sqlite3_free(errorMsg); + Close(); + return err; + } + } + static std::map statements = {{"put", util::fmt("insert into %s (key_str, value_str, expire_time) values(?, ?, ?)", table_name.c_str())}, {"put_update", @@ -89,6 +104,12 @@ void SQLite::Close() { prepared_stmts.clear(); + char* errmsg; + if ( int res = sqlite3_exec(db, "pragma optimize", NULL, NULL, &errmsg); res != SQLITE_OK ) { + Error(util::fmt("Sqlite failed to optimize at shutdown: %s", errmsg)); + sqlite3_free(&errmsg); + } + if ( int res = sqlite3_close_v2(db); res != SQLITE_OK ) Error("Sqlite could not close connection"); From b2bcb19b22abc9613979ba2ab445c33b940ce616 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 21 Aug 2024 12:33:43 -0700 Subject: [PATCH 13/52] SQLite: Add pragma integrity_check --- src/storage/backend/sqlite/SQLite.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index e166e861c8..db43201a1d 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -53,6 +53,14 @@ ErrorResult SQLite::DoOpen(RecordValPtr options) { return err; } + if ( int res = sqlite3_exec(db, "pragma integrity_check", NULL, NULL, &errorMsg); res != SQLITE_OK ) { + std::string err = util::fmt("Error executing integrity check: %s", errorMsg); + Error(err.c_str()); + sqlite3_free(errorMsg); + Close(); + return err; + } + auto tuning_params = options->GetField("tuning_params")->ToMap(); for ( const auto& [k, v] : tuning_params ) { auto ks = k->AsListVal()->Idx(0)->AsStringVal(); From 6bc5f70236ec251576d64b759c982d0b8805610d Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 9 Dec 2024 20:14:36 -0700 Subject: [PATCH 14/52] SQLite: Add additional btests, which also cover general storage functionality - New erase/overwrite tests - Change existing sqlite-basic test to use async - Test passing bad keys to validate backend type checking - New test for compound keys and values --- .../.stderr | 1 + .../out | 7 ++ .../.stderr | 2 + .../scripts.base.frameworks.storage.erase/out | 3 + .../.stderr | 1 + .../out | 4 + .../.stderr | 3 + .../out | 4 + .../.stderr | 2 +- .../out | 10 +-- .../.stderr | 1 + .../out | 1 + testing/btest/Files/storage-test.sqlite | Bin 0 -> 16384 bytes .../frameworks/storage/compound-types.zeek | 74 ++++++++++++++++++ .../base/frameworks/storage/erase.zeek | 34 ++++++++ .../base/frameworks/storage/overwriting.zeek | 31 ++++++++ .../storage/sqlite-basic-reading-pcap.zeek | 47 +++++++++++ .../base/frameworks/storage/sqlite-basic.zeek | 53 ++++++------- .../storage/sqlite-error-handling.zeek | 13 +++ 19 files changed, 252 insertions(+), 39 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/.stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.erase/.stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.erase/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/.stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/.stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out create mode 100644 testing/btest/Files/storage-test.sqlite create mode 100644 testing/btest/scripts/base/frameworks/storage/compound-types.zeek create mode 100644 testing/btest/scripts/base/frameworks/storage/erase.zeek create mode 100644 testing/btest/scripts/base/frameworks/storage/overwriting.zeek create mode 100644 testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out b/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out new file mode 100644 index 0000000000..e8f3abe594 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, { +[2] = b, +[1] = a, +[3] = c +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.erase/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.erase/.stderr new file mode 100644 index 0000000000..886fb29d21 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.erase/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/erase.zeek, line 28: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(b, to_any_coerce key, F)) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out b/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out new file mode 100644 index 0000000000..056ba93c85 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +erase result, T +get result, got empty result diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out b/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out new file mode 100644 index 0000000000..809959c28f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, value7890 +get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/.stderr new file mode 100644 index 0000000000..5d6ddc9db9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/.stderr @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1362692526.869344 received termination signal +1362692526.869344 warning in <...>/find-filtered-trace.zeek, line 69: The analyzed trace file was determined to contain only TCP control packets, which may indicate it's been pre-filtered. By default, Zeek reports the missing segments for this type of trace, but the 'detect_filtered_trace' option may be toggled if that's not desired. diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out new file mode 100644 index 0000000000..5125c8494e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, value5678 +get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr index d172ece6f0..e3f6131b1d 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/.stderr @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in /Users/tim/Desktop/projects/storage-framework/testing/btest/.tmp/scripts.base.frameworks.storage.sqlite-basic/sqlite-basic.zeek, line 42: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(b, to_any_coerce key, F)) +received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out index c54d0a144d..5125c8494e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out @@ -1,8 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -T -value -T -value2 -T -got empty result -value2 +put result, T +get result, value5678 +get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr index 4cb5a5b2d9..c6fae04e3a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr @@ -1,2 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. error in <...>/sqlite-error-handling.zeek, line 20: Failed to open backend SQLITE: SQLite call failed: unable to open database file (Storage::open_backend(Storage::SQLITE, to_any_coerce opts, to_any_coerce str, to_any_coerce str)) +error in <...>/sqlite-error-handling.zeek, line 28: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::put(b, (coerce [$key=bad_key, $value=value, $async_mode=F] to Storage::PutArgs))) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out index 49d861c74c..c3ff9662ac 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out @@ -1 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Put result on closed handle: 0 diff --git a/testing/btest/Files/storage-test.sqlite b/testing/btest/Files/storage-test.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..d92a0a5df57f5faff39ec325be186703a16e102c GIT binary patch literal 16384 zcmeI(&q~8E90%~Et#gW_f_=g+K^2NGV3nQR#kre@(y=JPuA5ydoi`uP7uZ91 z_2SWVb}|oo73cRyLXw8&&*#>(y}Im4CFo|7jblY8Y@ZnhJ0)U_nL1lKufouk^)_F1 z{7{-jD7*2tWV=5P$##AOHafKmY;|_#J@@v*xr~{947sM9l6J z{X`tiRIGyax^>>~hJ!8*!n0nN)_dA&nD}y3_~kvFoNqg>%b&vKXiChKOmC~g_%cRy zk=l1+uII~0i9*pdljAs>lfLNFL!9L0TB^QuqA1Oi#3xZqWhNpe$AU5uC!Hgs=7cV1 zG98JcT3wv0$+8_)Q$;~lDWC1@C3Zt+Zoe>{5D1Rwwb2tWV=5P$##AOHaf{CR;x zZZ+Myr;il^9iF#5TJU!DzPSC$bV5J?0uX=z1Rwwb2tWV=5P$##Ah0 out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff .stderr + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/sqlite + +type Color: enum { + Red = 10, + White = 20, + Blue = 30 +}; + +type Rec: record +{ + hello: string; + t: bool; + f: bool; + n: count &optional; + m: count &optional; # not in input + def: count &default = 123; + i: int; + pi: double; + a: string_vec; + c1: Color; + p: port; + ti: time; + it: interval; + ad: addr; + s: subnet; + re: pattern; + su: subnet_set; +}; + +type tbl: table[count] of string; + +event zeek_init() { + # Create a database file in the .tmp directory with a 'testing' table + local opts : Storage::Backend::SQLite::Options; + opts$database_path = "types_test.sqlite"; + opts$table_name = "types_testing"; + + local key : Rec; + key$hello = "hello"; + key$t = T; + key$f = F; + key$n = 1234; + key$m = 5678; + key$i = -2345; + key$pi = 345.0; + key$a = ["a","b","c"]; + key$c1 = Red; + key$p = 1234/tcp; + key$ti = current_time(); + key$it = 15sec; + key$ad = 1.2.3.4; + key$s = 255.255.255.0/24; + key$re = /.*/; + key$su = [255.255.255.0/24]; + + local value : tbl; + value[1] = "a"; + value[2] = "b"; + value[3] = "c"; + + local b = Storage::open_backend(Storage::SQLITE, opts, Rec, tbl); + + local res = Storage::put(b, [$key=key, $value=value, $async_mode=F]); + print "put result", res; + + local res2 = Storage::get(b, key, F); + print "get result", res2; +} diff --git a/testing/btest/scripts/base/frameworks/storage/erase.zeek b/testing/btest/scripts/base/frameworks/storage/erase.zeek new file mode 100644 index 0000000000..515f2ab2ea --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/erase.zeek @@ -0,0 +1,34 @@ +# @TEST-DOC: Erase existing data in a SQLite backend +# @TEST-EXEC: cp $FILES/storage-test.sqlite ./storage-test.sqlite +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/sqlite + +# Create a typename here that can be passed down into get(). +type str: string; + +event zeek_init() { + # Create a database file in the .tmp directory with a 'testing' table + local opts : Storage::Backend::SQLite::Options; + opts$database_path = "storage-test.sqlite"; + opts$table_name = "testing"; + + local key = "key1234"; + + # Test inserting/retrieving a key/value pair that we know won't be in + # the backend yet. + local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + + local res = Storage::erase(b, key, F); + print "erase result", res; + + local res2 = Storage::get(b, key, F); + if ( ! res2 as bool ) { + print "get result, got empty result"; + } + + Storage::close_backend(b); +} diff --git a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek new file mode 100644 index 0000000000..82b667b2b9 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek @@ -0,0 +1,31 @@ +# @TEST-DOC: Overwriting existing data in a SQLite backend +# @TEST-EXEC: cp $FILES/storage-test.sqlite ./storage-test.sqlite +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff .stderr + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/sqlite + +# Create a typename here that can be passed down into get(). +type str: string; + +event zeek_init() { + local opts : Storage::Backend::SQLite::Options; + opts$database_path = "storage-test.sqlite"; + opts$table_name = "testing"; + + local key = "key1234"; + local value = "value7890"; + + local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + + local res = Storage::put(b, [$key=key, $value=value, $async_mode=F]); + print "put result", res; + + local res2 = Storage::get(b, key, F); + print "get result", res2; + print "get result same as inserted", value == (res2 as string); + + Storage::close_backend(b); +} diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek new file mode 100644 index 0000000000..dbd00662b3 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek @@ -0,0 +1,47 @@ +# @TEST-DOC: Tests that sqlite async works fine while reading pcaps +# @TEST-EXEC: zeek -C -r $TRACES/http/get.trace %INPUT > out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/sqlite + +redef exit_only_after_terminate = T; + +# Create a typename here that can be passed down into get(). +type str: string; + +event zeek_init() { + # Create a database file in the .tmp directory with a 'testing' table + local opts : Storage::Backend::SQLite::Options; + opts$database_path = "test.sqlite"; + opts$table_name = "testing"; + + local key = "key1234"; + local value = "value5678"; + + # Test inserting/retrieving a key/value pair that we know won't be in + # the backend yet. + local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + + when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) { + print "put result", res; + + when [b, key, value] ( local res2 = Storage::get(b, key) ) { + print "get result", res2; + print "get result same as inserted", value == (res2 as string); + + Storage::close_backend(b); + + terminate(); + } + timeout 5 sec { + print "get requeest timed out"; + terminate(); + } + } + timeout 5 sec { + print "put request timed out"; + terminate(); + } +} diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index c28bb5041b..47c7d1d846 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -1,4 +1,4 @@ -# @TEST-DOC: Basic functionality for storage: opening/closing an sqlite backend, storing/retrieving/erasing basic data +# @TEST-DOC: Basic functionality for storage: opening/closing an sqlite backend, storing/retrieving data, using async methods # @TEST-EXEC: zeek -b %INPUT > out # @TEST-EXEC: btest-diff out # @TEST-EXEC: btest-diff .stderr @@ -6,7 +6,9 @@ @load base/frameworks/storage @load policy/frameworks/storage/backend/sqlite -# Create a typename here that can be passed down into open_backend. +redef exit_only_after_terminate = T; + +# Create a typename here that can be passed down into get(). type str: string; event zeek_init() { @@ -15,42 +17,31 @@ event zeek_init() { opts$database_path = "test.sqlite"; opts$table_name = "testing"; - local key = "key1111"; - local value = "value"; + local key = "key1234"; + local value = "value5678"; # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. local b = Storage::open_backend(Storage::SQLITE, opts, str, str); - local res = Storage::put(b, [$key=key, $value=value, $overwrite=T, $async_mode=F]); - print res; - local res2 = Storage::get(b, key, F); - print res2; + when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) { + print "put result", res; - # Test overwriting a value with put() - local value2 = "value2"; - local res3 = Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]); - print res3; + when [b, key, value] ( local res2 = Storage::get(b, key) ) { + print "get result", res2; + print "get result same as inserted", value == (res2 as string); - local res4 = Storage::get(b, key, F); - print res4; + Storage::close_backend(b); - # Test erasing a key and getting a "false" result - local res5 = Storage::erase(b, key, F); - print res5; - - local res6 = Storage::get(b, key, F); - if ( ! res6 as bool ) { - print "got empty result"; + terminate(); + } + timeout 5 sec { + print "get requeest timed out"; + terminate(); + } + } + timeout 5 sec { + print "put request timed out"; + terminate(); } - - # Insert something back into the database to test reopening - Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]); - - Storage::close_backend(b); - - # Test reopening the same database and getting the data back out of it - local b2 = Storage::open_backend(Storage::SQLITE, opts, str, str); - local res7 = Storage::get(b2, key, F); - print res7; } diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek index 159972c48b..4d7d248fc4 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek @@ -18,4 +18,17 @@ event zeek_init() { # This should report an error in .stderr and reporter.log local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + + # Open a valid database file + opts$database_path = "test.sqlite"; + b = Storage::open_backend(Storage::SQLITE, opts, str, str); + + local bad_key: count = 12345; + local value = "abcde"; + Storage::put(b, [$key=bad_key, $value=value, $async_mode=F]); + + # Close the backend and then attempt to use the closed handle + Storage::close_backend(b); + local res = Storage::put(b, [$key="a", $value="b", $async_mode=F]); + print fmt("Put result on closed handle: %d", res); } From 6289eb8e1544abc5781f086541662153ceeb0a53 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 20 Dec 2024 15:20:12 -0700 Subject: [PATCH 15/52] SQLite: Fix some issues with expiration, including in the btest --- src/storage/backend/sqlite/SQLite.cc | 5 ++++- .../.stderr | 4 ++-- .../base/frameworks/storage/expiration.zeek | 19 +++++++++++++------ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index db43201a1d..ece7eb1aff 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -155,6 +155,9 @@ ErrorResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expir return res; } + if ( expiration_time != 0 ) + expiration_time += run_state::network_time; + if ( auto res = checkError(sqlite3_bind_double(stmt, 3, expiration_time)); res.has_value() ) { sqlite3_reset(stmt); return res; @@ -244,7 +247,7 @@ ErrorResult SQLite::DoErase(ValPtr key, ErrorResultCallback* cb) { void SQLite::Expire() { auto stmt = prepared_stmts["expire"]; - if ( auto res = checkError(sqlite3_bind_double(stmt, 1, util::current_time())); res.has_value() ) { + if ( auto res = checkError(sqlite3_bind_double(stmt, 1, run_state::network_time)); res.has_value() ) { sqlite3_reset(stmt); // TODO: do something with the error here? } diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr index 0494e4d891..65a0a4d3ed 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in /Users/tim/Desktop/projects/storage-framework/testing/btest/.tmp/scripts.base.frameworks.storage.expiration/expiration.zeek, line 20: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(backend, to_any_coerce key, F)) -received termination signal +1627225025.686472 error in <...>/expiration.zeek, line 20: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(backend, to_any_coerce key, F)) +1627225025.686472 received termination signal diff --git a/testing/btest/scripts/base/frameworks/storage/expiration.zeek b/testing/btest/scripts/base/frameworks/storage/expiration.zeek index 30caade463..46fb2be5a8 100644 --- a/testing/btest/scripts/base/frameworks/storage/expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/expiration.zeek @@ -1,12 +1,12 @@ # @TEST-DOC: Automatic expiration of stored data -# @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b %INPUT > out -# @TEST-EXEC: btest-diff out -# @TEST-EXEC: btest-diff .stderr +# @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b -Cr - %INPUT > out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr @load base/frameworks/storage @load policy/frameworks/storage/backend/sqlite -redef Storage::expire_interval = 5 secs; +redef Storage::expire_interval = 2 secs; redef exit_only_after_terminate = T; # Create a typename here that can be passed down into get(). @@ -24,14 +24,14 @@ event check_removed() { terminate(); } -event zeek_init() { +event setup_test() { local opts : Storage::Backend::SQLite::Options; opts$database_path = "storage-test.sqlite"; opts$table_name = "testing"; backend = Storage::open_backend(Storage::SQLITE, opts, str, str); - local res = Storage::put(backend, [$key=key, $value=value, $overwrite=T, $expire_time=2 secs, $async_mode=F]); + local res = Storage::put(backend, [$key=key, $value=value, $expire_time=2 secs, $async_mode=F]); print "put result", res; local res2 = Storage::get(backend, key, F); @@ -40,3 +40,10 @@ event zeek_init() { schedule 5 secs { check_removed() }; } + +event zeek_init() { + # We need network time to be set to something other than zero for the + # expiration time to be set correctly. Schedule an event on a short + # timer so packets start getting read and do the setup there. + schedule 100 msecs { setup_test() }; +} From 31e146b16d0868ee8ee1ff4f89add7f821b567a2 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 12 Sep 2024 10:37:22 +0200 Subject: [PATCH 16/52] Redis: Add new backend --- CMakeLists.txt | 4 + .../storage/backend/redis/__load__.zeek | 1 + .../storage/backend/redis/main.zeek | 37 ++ scripts/test-all-policy.zeek | 2 + src/storage/backend/CMakeLists.txt | 1 + src/storage/backend/redis/CMakeLists.txt | 22 + src/storage/backend/redis/Plugin.cc | 22 + src/storage/backend/redis/Redis.cc | 412 ++++++++++++++++++ src/storage/backend/redis/Redis.h | 91 ++++ .../backend/redis/cmake/FindHiredis.cmake | 42 ++ 10 files changed, 634 insertions(+) create mode 100644 scripts/policy/frameworks/storage/backend/redis/__load__.zeek create mode 100644 scripts/policy/frameworks/storage/backend/redis/main.zeek create mode 100644 src/storage/backend/redis/CMakeLists.txt create mode 100644 src/storage/backend/redis/Plugin.cc create mode 100644 src/storage/backend/redis/Redis.cc create mode 100644 src/storage/backend/redis/Redis.h create mode 100644 src/storage/backend/redis/cmake/FindHiredis.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 88d90578a2..c3520a4dfa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1511,6 +1511,10 @@ message( "\n - Broker: ON" "\n - ZeroMQ: ${ENABLE_CLUSTER_BACKEND_ZEROMQ}" "\n" + "\nStorage backends:" + "\n - SQLite: ON" + "\n - Redis: ${ENABLE_STORAGE_BACKEND_REDIS}" + "\n" "\nFuzz Targets: ${ZEEK_ENABLE_FUZZERS}" "\nFuzz Engine: ${ZEEK_FUZZING_ENGINE}" "\n" diff --git a/scripts/policy/frameworks/storage/backend/redis/__load__.zeek b/scripts/policy/frameworks/storage/backend/redis/__load__.zeek new file mode 100644 index 0000000000..6086306018 --- /dev/null +++ b/scripts/policy/frameworks/storage/backend/redis/__load__.zeek @@ -0,0 +1 @@ +@load ./main.zeek \ No newline at end of file diff --git a/scripts/policy/frameworks/storage/backend/redis/main.zeek b/scripts/policy/frameworks/storage/backend/redis/main.zeek new file mode 100644 index 0000000000..09746eddfb --- /dev/null +++ b/scripts/policy/frameworks/storage/backend/redis/main.zeek @@ -0,0 +1,37 @@ +##! Redis storage backend support + +@load base/frameworks/storage/main + +module Storage::Backend::Redis; + +export { + ## Options record for the built-in Redis backend. + type Options: record { + # Address or hostname of the server + server_host: string &optional; + + # Port for the server + server_port: port &default=6379/tcp; + + # Server unix socket file. This can be used instead of the + # address and port above to connect to a local server. + server_unix_socket: string &optional; + + # Prefix used in key values stored to differentiate varying + # types of data on the same server. Defaults to an empty string, + # but preferably should be set to a unique value per Redis + # backend opened. + key_prefix: string &default=""; + + # Redis only supports sync and async separately. You cannot do + # both with the same connection. If this flag is true, the + # connection will be async and will only allow commands via + # ``when`` commands. You will still need to set the + # ``async_mode`` flags of the put, get, and erase methods to + # match this flag. This flag is overridden when reading pcaps + # and the backend will be forced into synchronous mode, since + # time won't move forward the same as when capturing live + # traffic. + async_mode: bool &default=F; + }; +} diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 34968b2ed6..8031df2610 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -83,6 +83,8 @@ # @load frameworks/spicy/record-spicy-batch.zeek # @load frameworks/spicy/resource-usage.zeek @load frameworks/software/windows-version-detection.zeek +@load frameworks/storage/backend/redis/__load__.zeek +@load frameworks/storage/backend/redis/main.zeek @load frameworks/storage/backend/sqlite/__load__.zeek @load frameworks/storage/backend/sqlite/main.zeek @load frameworks/telemetry/log.zeek diff --git a/src/storage/backend/CMakeLists.txt b/src/storage/backend/CMakeLists.txt index a5af49edbb..5e90740211 100644 --- a/src/storage/backend/CMakeLists.txt +++ b/src/storage/backend/CMakeLists.txt @@ -1 +1,2 @@ add_subdirectory(sqlite) +add_subdirectory(redis) diff --git a/src/storage/backend/redis/CMakeLists.txt b/src/storage/backend/redis/CMakeLists.txt new file mode 100644 index 0000000000..c9f5f1b38d --- /dev/null +++ b/src/storage/backend/redis/CMakeLists.txt @@ -0,0 +1,22 @@ +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +find_package(Hiredis) + +# Default to building Reids only if the hiredis library was found. +# +# If a user enabled the backend explicitly (-D ENABLE_STORAGE_BACKEND_REDIS:bool=ON), +# but hiredis wasn't found, hard bail. +option(ENABLE_STORAGE_BACKEND_REDIS "Enable the Redis storage backend" ${HIREDIS_FOUND}) + +if (ENABLE_STORAGE_BACKEND_REDIS) + if (NOT HIREDIS_FOUND) + message(STATUS "ENABLE_STORAGE_BACKEND_REDIS set, but hiredis library not available.") + endif () + + zeek_add_plugin( + Zeek Storage_Backend_Redis + INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${HIREDIS_INCLUDE_DIRS} + DEPENDENCIES ${HIREDIS_LIBRARIES} + SOURCES Plugin.cc Redis.cc) + +endif () diff --git a/src/storage/backend/redis/Plugin.cc b/src/storage/backend/redis/Plugin.cc new file mode 100644 index 0000000000..751a3996c9 --- /dev/null +++ b/src/storage/backend/redis/Plugin.cc @@ -0,0 +1,22 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/plugin/Plugin.h" + +#include "zeek/storage/Component.h" +#include "zeek/storage/backend/redis/Redis.h" + +namespace zeek::storage::backend::redis { + +class Plugin : public plugin::Plugin { +public: + plugin::Configuration Configure() override { + AddComponent(new storage::Component("REDIS", backend::redis::Redis::Instantiate)); + + plugin::Configuration config; + config.name = "Zeek::Storage_Backend_Redis"; + config.description = "Redis backend for storage framework"; + return config; + } +} plugin; + +} // namespace zeek::storage::backend::redis diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc new file mode 100644 index 0000000000..40e6397ed0 --- /dev/null +++ b/src/storage/backend/redis/Redis.cc @@ -0,0 +1,412 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/backend/redis/Redis.h" + +#include "zeek/Func.h" +#include "zeek/RunState.h" +#include "zeek/Val.h" +#include "zeek/iosource/Manager.h" + +#include "hiredis/async.h" +#include "hiredis/hiredis.h" + +// Anonymous callback handler methods for the hiredis async API. +namespace { + +class Tracer { +public: + Tracer(const std::string& where) : where(where) { /*printf("%s\n", where.c_str());*/ } + ~Tracer() { /* printf("%s done\n", where.c_str()); */ } + std::string where; +}; + +void redisOnConnect(const redisAsyncContext* ctx, int status) { + auto t = Tracer("connect"); + auto backend = static_cast(ctx->data); + backend->OnConnect(status); +} + +void redisOnDisconnect(const redisAsyncContext* ctx, int status) { + auto t = Tracer("disconnect"); + auto backend = static_cast(ctx->data); + backend->OnDisconnect(status); +} + +void redisPut(redisAsyncContext* ctx, void* reply, void* privdata) { + auto t = Tracer("put"); + auto backend = static_cast(ctx->data); + auto callback = static_cast(privdata); + backend->HandlePutResult(static_cast(reply), callback); +} + +void redisGet(redisAsyncContext* ctx, void* reply, void* privdata) { + auto t = Tracer("get"); + auto backend = static_cast(ctx->data); + auto callback = static_cast(privdata); + backend->HandleGetResult(static_cast(reply), callback); +} + +void redisErase(redisAsyncContext* ctx, void* reply, void* privdata) { + auto t = Tracer("erase"); + auto backend = static_cast(ctx->data); + auto callback = static_cast(privdata); + backend->HandleEraseResult(static_cast(reply), callback); +} + +void redisAddRead(void* privdata) { + auto t = Tracer("addread"); + auto backend = static_cast(privdata); + backend->OnAddRead(); +} +void redisDelRead(void* privdata) { + auto t = Tracer("delread"); + auto backend = static_cast(privdata); + backend->OnDelRead(); +} +void redisAddWrite(void* privdata) { + auto t = Tracer("addwrite"); + auto backend = static_cast(privdata); + backend->OnAddWrite(); +} +void redisDelWrite(void* privdata) { + auto t = Tracer("delwrite"); + auto backend = static_cast(privdata); + backend->OnDelWrite(); +} + +} // namespace + +namespace zeek::storage::backend::redis { + +storage::BackendPtr Redis::Instantiate(std::string_view tag) { return make_intrusive(tag); } + +/** + * Called by the manager system to open the backend. + * + * Derived classes must implement this method. If successful, the + * implementation must call \a Opened(); if not, it must call Error() + * with a corresponding message. + */ +ErrorResult Redis::DoOpen(RecordValPtr options) { + async_mode = options->GetField("async_mode")->Get(); + key_prefix = options->GetField("key_prefix")->ToStdString(); + + redisOptions opt = {0}; + + StringValPtr host = options->GetField("server_host"); + if ( host ) { + PortValPtr port = options->GetField("server_port"); + server_addr = util::fmt("%s:%d", host->ToStdStringView().data(), port->Port()); + REDIS_OPTIONS_SET_TCP(&opt, host->ToStdStringView().data(), port->Port()); + } + else { + StringValPtr unix_sock = options->GetField("server_unix_socket"); + if ( ! unix_sock ) + return util::fmt( + "Either server_host/server_port or server_unix_socket must be set in Redis options record"); + + server_addr = unix_sock->ToStdString(); + REDIS_OPTIONS_SET_UNIX(&opt, server_addr.c_str()); + } + + opt.options |= REDIS_OPT_PREFER_IPV4; + opt.options |= REDIS_OPT_NOAUTOFREEREPLIES; + + struct timeval timeout = {5, 0}; + opt.connect_timeout = &timeout; + + if ( async_mode ) { + async_ctx = redisAsyncConnectWithOptions(&opt); + if ( async_ctx == nullptr || async_ctx->err ) { + // This block doesn't necessarily mean the connection failed. It means + // that hiredis failed to set up the async context. Connection failure + // is returned later via the OnConnect callback. + std::string errmsg = util::fmt("Failed to open connection to Redis server at %s", server_addr.c_str()); + if ( async_ctx ) { + errmsg.append(": "); + errmsg.append(async_ctx->errstr); + } + + redisAsyncFree(async_ctx); + async_ctx = nullptr; + return errmsg; + } + + // The context is passed to the handler methods. Setting this data object + // pointer allows us to look up the backend in the handlers. + async_ctx->data = this; + async_ctx->ev.data = this; + + redisAsyncSetConnectCallback(async_ctx, redisOnConnect); + redisAsyncSetDisconnectCallback(async_ctx, redisOnDisconnect); + + // These four callbacks handle the file descriptor coming and going for read + // and write operations for hiredis. Their subsequent callbacks will + // register/unregister with iosource_mgr as needed. I tried just registering + // full time for both read and write but it leads to weird syncing issues + // within the hiredis code. This is safer in regards to the library, even if + // it results in waking up our IO loop more frequently. + async_ctx->ev.addRead = redisAddRead; + async_ctx->ev.delRead = redisDelRead; + async_ctx->ev.addWrite = redisAddWrite; + async_ctx->ev.delWrite = redisDelWrite; + } + else { + ctx = redisConnectWithOptions(&opt); + if ( ctx == nullptr || ctx->err ) { + if ( ctx ) + return util::fmt("Failed to open connection to Redis server at %s", server_addr.c_str()); + else + return util::fmt("Failed to open connection to Redis server at %s: %s", server_addr.c_str(), + ctx->errstr); + } + + connected = true; + } + + return std::nullopt; +} + +/** + * Finalizes the backend when it's being closed. + */ +void Redis::Close() { + connected = false; + + if ( async_mode ) { + // This will probably result in an error since hiredis should have + // already removed the file descriptor via the delRead and delWrite + // callbacks, but do it anyways just to be sure. + iosource_mgr->UnregisterFd(async_ctx->c.fd, this, IOSource::READ | IOSource::WRITE); + redisAsyncDisconnect(async_ctx); + redisAsyncFree(async_ctx); + async_ctx = nullptr; + } + else { + redisFree(ctx); + ctx = nullptr; + } +} + +/** + * The workhorse method for Put(). This must be implemented by plugins. + */ +ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, ErrorResultCallback* cb) { + // The async context will queue operations until it's connected fully. + if ( ! connected && ! async_ctx ) + return "Connection is not open"; + + std::string format = "SET %s:%s %s"; + if ( ! overwrite ) + format.append(" NX"); + + // Use built-in expiration if reading live data, since time will move + // forward consistently. If reading pcaps, we'll do something else. + if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) + format.append(" PXAT %d"); + + double expire_time = expiration_time + run_state::network_time; + + auto json_key = key->ToJSON()->ToStdString(); + auto json_value = value->ToJSON()->ToStdString(); + + if ( async_mode ) { + int status; + if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) + status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), + json_value.data(), static_cast(expire_time * 1e6)); + else + status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), + json_value.data()); + + if ( connected && status == REDIS_ERR ) + return util::fmt("Failed to queue async put operation: %s", async_ctx->errstr); + } + else { + redisReply* reply; + if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) + reply = (redisReply*)redisCommand(ctx, format.c_str(), key_prefix.data(), json_key.data(), + json_value.data(), static_cast(expire_time * 1e6)); + else + reply = + (redisReply*)redisCommand(ctx, format.c_str(), key_prefix.data(), json_key.data(), json_value.data()); + + if ( ! reply ) + return util::fmt("Put operation failed: %s", ctx->errstr); + + freeReplyObject(reply); + } + + return std::nullopt; +} + +/** + * The workhorse method for Get(). This must be implemented for plugins. + */ +ValResult Redis::DoGet(ValPtr key, ValResultCallback* cb) { + // The async context will queue operations until it's connected fully. + if ( ! connected && ! async_ctx ) + return zeek::unexpected("Connection is not open"); + + if ( async_mode ) { + int status = redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%s", key_prefix.data(), + key->ToJSON()->ToStdStringView().data()); + + if ( connected && status == REDIS_ERR ) + return zeek::unexpected( + util::fmt("Failed to queue async get operation: %s", async_ctx->errstr)); + + // There isn't a result to return here. That happens in HandleGetResult. + return zeek::unexpected(""); + } + else { + auto reply = + (redisReply*)redisCommand(ctx, "GET %s:%s", key_prefix.data(), key->ToJSON()->ToStdStringView().data()); + + if ( ! reply ) + return zeek::unexpected(util::fmt("Get operation failed: %s", ctx->errstr)); + + return ParseGetReply(reply); + } +} + +/** + * The workhorse method for Erase(). This must be implemented for plugins. + */ +ErrorResult Redis::DoErase(ValPtr key, ErrorResultCallback* cb) { + // The async context will queue operations until it's connected fully. + if ( ! connected && ! async_ctx ) + return "Connection is not open"; + + if ( async_mode ) { + int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%s", key_prefix.data(), + key->ToJSON()->ToStdStringView().data()); + + if ( connected && status == REDIS_ERR ) + return util::fmt("Failed to queue async erase operation failed: %s", async_ctx->errstr); + } + else { + redisReply* reply = + (redisReply*)redisCommand(ctx, "DEL %s:%s", key_prefix.data(), key->ToJSON()->ToStdStringView().data()); + + if ( ! reply ) + return util::fmt("Put operation failed: %s", ctx->errstr); + + freeReplyObject(reply); + } + + return std::nullopt; +} + +void Redis::HandlePutResult(redisReply* reply, ErrorResultCallback* callback) { + ErrorResult res; + if ( ! connected ) + res = util::fmt("Connection is not open"); + else if ( ! reply ) + res = util::fmt("Async put operation returned null reply"); + else if ( reply && reply->type == REDIS_REPLY_ERROR ) + res = util::fmt("Async put operation failed: %s", reply->str); + + freeReplyObject(reply); + callback->Complete(res); + delete callback; +} + +void Redis::HandleGetResult(redisReply* reply, ValResultCallback* callback) { + ValResult res; + if ( ! connected ) + res = zeek::unexpected("Connection is not open"); + else + res = ParseGetReply(reply); + + callback->Complete(res); + delete callback; +} + +void Redis::HandleEraseResult(redisReply* reply, ErrorResultCallback* callback) { + ErrorResult res; + if ( ! connected ) + res = "Connection is not open"; + else if ( ! reply ) + res = util::fmt("Async erase operation returned null reply"); + else if ( reply && reply->type == REDIS_REPLY_ERROR ) + res = util::fmt("Async erase operation failed: %s", reply->str); + + freeReplyObject(reply); + + callback->Complete(res); + delete callback; +} + +void Redis::OnConnect(int status) { + if ( status == REDIS_OK ) { + connected = true; + return; + } + + // TODO: we could attempt to reconnect here +} + +void Redis::OnDisconnect(int status) { + if ( status == REDIS_OK ) { + // TODO: this was an intentional disconnect, nothing to do? + } + else { + // TODO: this was unintentional, should we reconnect? + } + + connected = false; +} + +void Redis::OnAddRead() { + if ( ! async_ctx ) + return; + + iosource_mgr->RegisterFd(async_ctx->c.fd, this, IOSource::READ); +} +void Redis::OnDelRead() { + if ( ! async_ctx ) + return; + + iosource_mgr->UnregisterFd(async_ctx->c.fd, this, IOSource::READ); +} +void Redis::OnAddWrite() { + if ( ! async_ctx ) + return; + + iosource_mgr->RegisterFd(async_ctx->c.fd, this, IOSource::WRITE); +} +void Redis::OnDelWrite() { + if ( ! async_ctx ) + return; + + iosource_mgr->UnregisterFd(async_ctx->c.fd, this, IOSource::WRITE); +} + +void Redis::ProcessFd(int fd, int flags) { + if ( (flags & IOSource::ProcessFlags::READ) != 0 ) + redisAsyncHandleRead(async_ctx); + if ( (flags & IOSource::ProcessFlags::WRITE) != 0 ) + redisAsyncHandleWrite(async_ctx); +} + +ValResult Redis::ParseGetReply(redisReply* reply) const { + ValResult res; + + if ( ! reply ) + res = zeek::unexpected("GET returned null reply"); + else if ( ! reply->str ) + res = zeek::unexpected("GET returned key didn't exist"); + else { + auto val = zeek::detail::ValFromJSON(reply->str, val_type, Func::nil); + if ( std::holds_alternative(val) ) + res = std::get(val); + else + res = zeek::unexpected(std::get(val)); + } + + freeReplyObject(reply); + return res; +} + +} // namespace zeek::storage::backend::redis diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h new file mode 100644 index 0000000000..68f1969489 --- /dev/null +++ b/src/storage/backend/redis/Redis.h @@ -0,0 +1,91 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/iosource/IOSource.h" +#include "zeek/storage/Backend.h" + +// Forward declare some types from hiredis to avoid including the header +struct redisContext; +struct redisAsyncContext; +struct redisReply; + +namespace zeek::storage::backend::redis { + +class Redis : public Backend, public iosource::IOSource { +public: + Redis(std::string_view tag) : Backend(true, tag), IOSource(true) {} + ~Redis() override = default; + + static BackendPtr Instantiate(std::string_view tag); + + /** + * Returns a descriptive tag representing the source for debugging. + * This has to be overloaded for Redis because IOSource requires it. + * + * @return The debugging name. + */ + const char* Tag() override { return tag.c_str(); } + + /** + * Called by the manager system to open the backend. + */ + ErrorResult DoOpen(RecordValPtr options) override; + + /** + * Finalizes the backend when it's being closed. + */ + void Close() override; + + /** + * Returns whether the backend is opened. + */ + bool IsOpen() override { return connected; } + + /** + * The workhorse method for Retrieve(). + */ + ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + ErrorResultCallback* cb = nullptr) override; + + /** + * The workhorse method for Get(). + */ + ValResult DoGet(ValPtr key, ValResultCallback* cb = nullptr) override; + + /** + * The workhorse method for Erase(). + */ + ErrorResult DoErase(ValPtr key, ErrorResultCallback* cb = nullptr) override; + + // IOSource interface + double GetNextTimeout() override { return -1; } + void Process() override {} + void ProcessFd(int fd, int flags) override; + + // Hiredis async interface + void OnConnect(int status); + void OnDisconnect(int status); + + void OnAddRead(); + void OnDelRead(); + void OnAddWrite(); + void OnDelWrite(); + + void HandlePutResult(redisReply* reply, ErrorResultCallback* callback); + void HandleGetResult(redisReply* reply, ValResultCallback* callback); + void HandleEraseResult(redisReply* reply, ErrorResultCallback* callback); + +private: + ValResult ParseGetReply(redisReply* reply) const; + + redisContext* ctx = nullptr; + redisAsyncContext* async_ctx = nullptr; + bool connected = true; + + std::string server_addr; + std::string key_prefix; + bool async_mode = false; +}; + +} // namespace zeek::storage::backend::redis diff --git a/src/storage/backend/redis/cmake/FindHiredis.cmake b/src/storage/backend/redis/cmake/FindHiredis.cmake new file mode 100644 index 0000000000..fa3b71e5a1 --- /dev/null +++ b/src/storage/backend/redis/cmake/FindHiredis.cmake @@ -0,0 +1,42 @@ +include(FindPackageHandleStandardArgs) + +find_library( + HIREDIS_LIBRARY NAMES "libhiredis${CMAKE_SHARED_LIBRARY_SUFFIX}" + "libhiredis${CMAKE_STATIC_LIBRARY_SUFFIX}" HINTS ${HIREDIS_ROOT_DIR}/lib) + +find_path(HIREDIS_INCLUDE_DIR NAMES hiredis/hiredis.h HINTS ${HIREDIS_ROOT_DIR}/include) + +find_package_handle_standard_args(Hiredis FOUND_VAR HIREDIS_FOUND REQUIRED_VARS HIREDIS_LIBRARY + HIREDIS_INCLUDE_DIR) + +if (HIREDIS_FOUND) + + # The hiredis library must be at least v1.0.0 to have all of the API bits that + # we need. We can scrape that out of the header. + file(STRINGS "${HIREDIS_INCLUDE_DIR}/hiredis/hiredis.h" HIREDIS_MAJOR_VERSION_H + REGEX "^#define HIREDIS_MAJOR [0-9]+$") + file(STRINGS "${HIREDIS_INCLUDE_DIR}/hiredis/hiredis.h" HIREDIS_MINOR_VERSION_H + REGEX "^#define HIREDIS_MINOR [0-9]+$") + file(STRINGS "${HIREDIS_INCLUDE_DIR}/hiredis/hiredis.h" HIREDIS_PATCH_VERSION_H + REGEX "^#define HIREDIS_PATCH [0-9]+$") + string(REGEX REPLACE "^.*MAJOR ([0-9]+)$" "\\1" HIREDIS_MAJOR_VERSION + "${HIREDIS_MAJOR_VERSION_H}") + string(REGEX REPLACE "^.*MINOR ([0-9]+)$" "\\1" HIREDIS_MINOR_VERSION + "${HIREDIS_MINOR_VERSION_H}") + string(REGEX REPLACE "^.*PATCH ([0-9]+)$" "\\1" HIREDIS_PATCH_VERSION + "${HIREDIS_PATCH_VERSION_H}") + + set(HIREDIS_VERSION + "${HIREDIS_MAJOR_VERSION}.${HIREDIS_MINOR_VERSION}.${HIREDIS_PATCH_VERSION}") + + if (HIREDIS_VERSION VERSION_LESS "1.0.0") + message( + STATUS "Hiredis library version ${HIREDIS_VERSION} is too old, need v1.0.0 or later.") + unset(HIREDIS_FOUND) + + else () + set(HIREDIS_LIBRARIES ${HIREDIS_LIBRARY}) + set(HIREDIS_INCLUDE_DIRS ${HIREDIS_INCLUDE_DIR}) + set(HIREDIS_FOUND ${HIREDIS_FOUND}) + endif () +endif () From 52d94b781a5b0f81a55765cb2bf7c944f67041dc Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 17 Dec 2024 16:20:25 -0700 Subject: [PATCH 17/52] Redis: Force storage sync mode when reading pcaps, default to async mode --- scripts/policy/frameworks/storage/backend/redis/main.zeek | 2 +- src/storage/backend/redis/Redis.cc | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/policy/frameworks/storage/backend/redis/main.zeek b/scripts/policy/frameworks/storage/backend/redis/main.zeek index 09746eddfb..413c201083 100644 --- a/scripts/policy/frameworks/storage/backend/redis/main.zeek +++ b/scripts/policy/frameworks/storage/backend/redis/main.zeek @@ -32,6 +32,6 @@ export { # and the backend will be forced into synchronous mode, since # time won't move forward the same as when capturing live # traffic. - async_mode: bool &default=F; + async_mode: bool &default=T; }; } diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 40e6397ed0..29a893d704 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -88,7 +88,9 @@ storage::BackendPtr Redis::Instantiate(std::string_view tag) { return make_intru * with a corresponding message. */ ErrorResult Redis::DoOpen(RecordValPtr options) { - async_mode = options->GetField("async_mode")->Get(); + // When reading traces we disable storage async mode globally (see src/storage/Backend.cc) since + // time moves forward based on the pcap and not based on real time. + async_mode = options->GetField("async_mode")->Get() && ! zeek::run_state::reading_traces; key_prefix = options->GetField("key_prefix")->ToStdString(); redisOptions opt = {0}; From 08bebaa426e454d6cfe2d7012ef00173b75751a4 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 18 Dec 2024 10:28:31 -0700 Subject: [PATCH 18/52] Redis: Add btests for the redis backend --- ci/ubuntu-24.04/Dockerfile | 2 + src/storage/backend/redis/Redis.cc | 5 ++ .../out | 4 + .../out | 4 + .../out | 7 ++ testing/btest/Files/redis.conf | 73 +++++++++++++++++++ .../storage/redis-async-reading-pcap.zeek | 49 +++++++++++++ .../base/frameworks/storage/redis-async.zeek | 55 ++++++++++++++ .../base/frameworks/storage/redis-sync.zeek | 49 +++++++++++++ testing/scripts/have-redis | 4 + 10 files changed, 252 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out create mode 100644 testing/btest/Files/redis.conf create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-async.zeek create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-sync.zeek create mode 100755 testing/scripts/have-redis diff --git a/ci/ubuntu-24.04/Dockerfile b/ci/ubuntu-24.04/Dockerfile index 85f0e36d7f..1a7fec855e 100644 --- a/ci/ubuntu-24.04/Dockerfile +++ b/ci/ubuntu-24.04/Dockerfile @@ -24,6 +24,7 @@ RUN apt-get update && apt-get -y install \ jq \ lcov \ libkrb5-dev \ + libhiredis-dev \ libmaxminddb-dev \ libpcap-dev \ libssl-dev \ @@ -31,6 +32,7 @@ RUN apt-get update && apt-get -y install \ python3 \ python3-dev \ python3-pip \ + redis-server \ ruby \ sqlite3 \ swig \ diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 29a893d704..3fded1452a 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -2,6 +2,7 @@ #include "zeek/storage/backend/redis/Redis.h" +#include "zeek/DebugLogger.h" #include "zeek/Func.h" #include "zeek/RunState.h" #include "zeek/Val.h" @@ -93,6 +94,8 @@ ErrorResult Redis::DoOpen(RecordValPtr options) { async_mode = options->GetField("async_mode")->Get() && ! zeek::run_state::reading_traces; key_prefix = options->GetField("key_prefix")->ToStdString(); + DBG_LOG(DBG_STORAGE, "Redis backend: running in async mode? %d", async_mode); + redisOptions opt = {0}; StringValPtr host = options->GetField("server_host"); @@ -341,6 +344,7 @@ void Redis::HandleEraseResult(redisReply* reply, ErrorResultCallback* callback) } void Redis::OnConnect(int status) { + DBG_LOG(DBG_STORAGE, "Redis backend: connection event"); if ( status == REDIS_OK ) { connected = true; return; @@ -350,6 +354,7 @@ void Redis::OnConnect(int status) { } void Redis::OnDisconnect(int status) { + DBG_LOG(DBG_STORAGE, "Redis backend: disconnection event"); if ( status == REDIS_OK ) { // TODO: this was an intentional disconnect, nothing to do? } diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out new file mode 100644 index 0000000000..5125c8494e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, value5678 +get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out new file mode 100644 index 0000000000..5125c8494e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, value5678 +get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out new file mode 100644 index 0000000000..e7d5445bc7 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, value1234 +get result same as inserted, T +overwrite put result, T +get result, value5678 +get result same as inserted after overwrite, F diff --git a/testing/btest/Files/redis.conf b/testing/btest/Files/redis.conf new file mode 100644 index 0000000000..875c46f9f7 --- /dev/null +++ b/testing/btest/Files/redis.conf @@ -0,0 +1,73 @@ +bind 127.0.0.1 +port %REDIS_PORT% +pidfile %RUN_PATH%/redis.pid +loglevel verbose +logfile %RUN_PATH%/redis.log +dir %RUN_PATH% +daemonize no +databases 1 + +# All of the values from here down are the default values in the sample config file +# that comes with redis-server v7.2.7. +protected-mode yes +tcp-backlog 511 +timeout 0 +always-show-logo no +set-proc-title yes +proc-title-template "{title} {listen-addr} {server-mode}" +stop-writes-on-bgsave-error yes +rdbcompression yes +rdbchecksum yes +dbfilename dump.rdb +rdb-del-sync-files no +replica-serve-stale-data yes +replica-read-only yes +repl-diskless-sync yes +repl-diskless-sync-delay 5 +repl-diskless-sync-max-replicas 0 +repl-diskless-load disabled +repl-disable-tcp-nodelay no +replica-priority 100 +acllog-max-len 128 +lazyfree-lazy-eviction no +lazyfree-lazy-expire no +lazyfree-lazy-server-del no +replica-lazy-flush no +lazyfree-lazy-user-del no +lazyfree-lazy-user-flush no +oom-score-adj no +oom-score-adj-values 0 200 800 +disable-thp yes +appendonly no +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no +slowlog-log-slower-than 10000 +slowlog-max-len 128 +latency-monitor-threshold 0 +notify-keyspace-events "" +hash-max-listpack-entries 512 +hash-max-listpack-value 64 +list-max-listpack-size -2 +list-compress-depth 0 +set-max-intset-entries 512 +zset-max-listpack-entries 128 +zset-max-listpack-value 64 +hll-sparse-max-bytes 3000 +stream-node-max-bytes 4096 +stream-node-max-entries 100 +activerehashing yes +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit replica 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 +hz 10 +dynamic-hz yes +aof-rewrite-incremental-fsync yes +rdb-save-incremental-fsync yes +jemalloc-bg-thread yes \ No newline at end of file diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek new file mode 100644 index 0000000000..0cd3dcc2b9 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -0,0 +1,49 @@ +# @TEST-DOC: Tests that Redis storage backend defaults back to sync mode reading pcaps + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT + +# Generate a redis.conf file with the port defined above, but without the /tcp at the end of +# it. This also sets some paths in the conf to the testing directory. +# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf +# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: zeek -r $TRACES/http/get.trace -b %INPUT > out +# @TEST-EXEC: btest-bg-wait -k 0 + +# @TEST-EXEC: btest-diff out + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/redis + +# Create a typename here that can be passed down into open_backend() +type str: string; + +event zeek_init() { + local opts : Storage::Backend::Redis::Options; + opts$server_host = "127.0.0.1"; + opts$server_port = to_port(getenv("REDIS_PORT")); + opts$key_prefix = "testing"; + opts$async_mode = T; + + local key = "key1234"; + local value = "value5678"; + + local b = Storage::open_backend(Storage::REDIS, opts, str, str); + + when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) { + print "put result", res; + + when [b, key, value] ( local res2 = Storage::get(b, key) ) { + print "get result", res2; + print "get result same as inserted", value == (res2 as string); + + Storage::close_backend(b); + } + timeout 5 sec { + print "get requeest timed out"; + } + } + timeout 5 sec { + print "put request timed out"; + } +} diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek new file mode 100644 index 0000000000..47f913e5ae --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek @@ -0,0 +1,55 @@ +# @TEST-DOC: Tests basic Redis storage backend functions in async mode + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT + +# Generate a redis.conf file with the port defined above, but without the /tcp at the end of +# it. This also sets some paths in the conf to the testing directory. +# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf +# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: btest-bg-wait -k 0 + +# @TEST-EXEC: btest-diff out + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/redis + +redef exit_only_after_terminate = T; + +# Create a typename here that can be passed down into open_backend() +type str: string; + +event zeek_init() { + local opts : Storage::Backend::Redis::Options; + opts$server_host = "127.0.0.1"; + opts$server_port = to_port(getenv("REDIS_PORT")); + opts$key_prefix = "testing"; + opts$async_mode = T; + + local key = "key1234"; + local value = "value5678"; + + local b = Storage::open_backend(Storage::REDIS, opts, str, str); + + when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) { + print "put result", res; + + when [b, key, value] ( local res2 = Storage::get(b, key) ) { + print "get result", res2; + print "get result same as inserted", value == (res2 as string); + + Storage::close_backend(b); + + terminate(); + } + timeout 5 sec { + print "get requeest timed out"; + terminate(); + } + } + timeout 5 sec { + print "put request timed out"; + terminate(); + } +} diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek new file mode 100644 index 0000000000..d21d5979a8 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -0,0 +1,49 @@ +# @TEST-DOC: Tests basic Redis storage backend functions in sync mode, including overwriting + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT + +# Generate a redis.conf file with the port defined above, but without the /tcp at the end of +# it. This also sets some paths in the conf to the testing directory. +# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf +# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: btest-bg-wait -k 0 + +# @TEST-EXEC: btest-diff out + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/redis + +# Create a typename here that can be passed down into open_backend() +type str: string; + +event zeek_init() { + local opts : Storage::Backend::Redis::Options; + opts$server_host = "127.0.0.1"; + opts$server_port = to_port(getenv("REDIS_PORT")); + opts$key_prefix = "testing"; + opts$async_mode = F; + + local key = "key1234"; + local value = "value1234"; + + local b = Storage::open_backend(Storage::REDIS, opts, str, str); + + local res = Storage::put(b, [$key=key, $value=value, $async_mode=F]); + print "put result", res; + + local res2 = Storage::get(b, key, F); + print "get result", res2; + print "get result same as inserted", value == (res2 as string); + + local value2 = "value5678"; + res = Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]); + print "overwrite put result", res; + + res2 = Storage::get(b, key, F); + print "get result", res2; + print "get result same as inserted after overwrite", value == (res2 as string); + + Storage::close_backend(b); +} diff --git a/testing/scripts/have-redis b/testing/scripts/have-redis new file mode 100755 index 0000000000..b00289a280 --- /dev/null +++ b/testing/scripts/have-redis @@ -0,0 +1,4 @@ +#!/bin/sh + +zeek -N Zeek::Storage_Backend_Redis && which redis-server >/dev/null +exit $? From ea87c773cdcf22d806f96249bd3aa961a7bee3b9 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 20 Dec 2024 14:07:10 -0700 Subject: [PATCH 19/52] Redis: Support non-native expiration when reading traces --- src/storage/backend/redis/Redis.cc | 62 ++++++++++++++++++- src/storage/backend/redis/Redis.h | 6 ++ .../out | 5 ++ .../frameworks/storage/redis-expiration.zeek | 60 ++++++++++++++++++ 4 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 3fded1452a..a30c17e23a 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -217,7 +217,7 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira if ( async_mode ) { int status; - if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) + if ( expiration_time > 0.0 ) status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), json_value.data(), static_cast(expire_time * 1e6)); else @@ -242,6 +242,22 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira freeReplyObject(reply); } + // If reading pcaps insert into a secondary set that's ordered by expiration + // time that gets checked by Expire(). + if ( expiration_time > 0.0 && zeek::run_state::reading_traces ) { + format = "ZADD %s_expire"; + if ( ! overwrite ) + format.append(" NX"); + format += " %f %s"; + + redisReply* reply = + (redisReply*)redisCommand(ctx, format.c_str(), key_prefix.data(), expire_time, json_key.data()); + if ( ! reply ) + return util::fmt("ZADD operation failed: %s", ctx->errstr); + + freeReplyObject(reply); + } + return std::nullopt; } @@ -303,6 +319,50 @@ ErrorResult Redis::DoErase(ValPtr key, ErrorResultCallback* cb) { return std::nullopt; } +void Redis::Expire() { + if ( ! connected || ! zeek::run_state::reading_traces ) + return; + + redisReply* reply = + (redisReply*)redisCommand(ctx, "ZRANGEBYSCORE %s_expire -inf %f", key_prefix.data(), run_state::network_time); + + if ( ! reply ) { + // TODO: do something with the error? + printf("ZRANGEBYSCORE command failed: %s\n", ctx->errstr); + return; + } + + if ( reply->elements == 0 ) { + freeReplyObject(reply); + return; + } + + // TODO: it's possible to pass multiple keys to a DEL operation but it requires + // building an array of the strings, building up the DEL command with entries, + // and passing the array as a block somehow. There's no guarantee it'd be faster + // anyways. + for ( size_t i = 0; i < reply->elements; i++ ) { + auto del_reply = (redisReply*)redisCommand(ctx, "DEL %s:%s", key_prefix.data(), reply->element[i]->str); + + // Don't bother checking the response here. The only error that would matter is if the key + // didn't exist, but that would mean it was already removed for some other reason. + + freeReplyObject(del_reply); + } + + freeReplyObject(reply); + reply = (redisReply*)redisCommand(ctx, "ZREMRANGEBYSCORE %s_expire -inf %f", key_prefix.data(), + run_state::network_time); + + if ( ! reply ) { + // TODO: do something with the error? + printf("ZREMRANGEBYSCORE command failed: %s\n", ctx->errstr); + return; + } + + freeReplyObject(reply); +} + void Redis::HandlePutResult(redisReply* reply, ErrorResultCallback* callback) { ErrorResult res; if ( ! connected ) diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index 68f1969489..2d283f2965 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -58,6 +58,12 @@ public: */ ErrorResult DoErase(ValPtr key, ErrorResultCallback* cb = nullptr) override; + /** + * Removes any entries in the backend that have expired. Can be overridden by + * derived classes. + */ + void Expire() override; + // IOSource interface double GetNextTimeout() override { return -1; } void Process() override {} diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out new file mode 100644 index 0000000000..88d16bb4e3 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +put result, T +get result, value7890 +get result same as inserted, T +get result after expiration, F diff --git a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek new file mode 100644 index 0000000000..810bbb102d --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek @@ -0,0 +1,60 @@ +# @TEST-DOC: Tests expiration of data from Redis when reading a pcap + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT + +# Generate a redis.conf file with the port defined above, but without the /tcp at the end of +# it. This also sets some paths in the conf to the testing directory. +# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf +# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -B storage -b -Cr - %INPUT > out +# @TEST-EXEC: btest-bg-wait -k 1 + +# @TEST-EXEC: btest-diff out + +@load base/frameworks/storage +@load policy/frameworks/storage/backend/redis + +redef Storage::expire_interval = 2 secs; +redef exit_only_after_terminate = T; + +# Create a typename here that can be passed down into open_backend() +type str: string; + +global b: opaque of Storage::BackendHandle; +global key: string = "key1234"; +global value: string = "value7890"; + +event check_removed() { + local res2 = Storage::get(b, key, F); + print "get result after expiration", res2; + + Storage::close_backend(b); + terminate(); +} + +event setup_test() { + local opts : Storage::Backend::Redis::Options; + opts$server_host = "127.0.0.1"; + opts$server_port = to_port(getenv("REDIS_PORT")); + opts$key_prefix = "testing"; + opts$async_mode = F; + + b = Storage::open_backend(Storage::REDIS, opts, str, str); + + local res = Storage::put(b, [$key=key, $value=value, $async_mode=F, $expire_time=2 secs]); + print "put result", res; + + local res2 = Storage::get(b, key, F); + print "get result", res2; + print "get result same as inserted", value == (res2 as string); + + schedule 5 secs { check_removed() }; +} + +event zeek_init() { + # We need network time to be set to something other than zero for the + # expiration time to be set correctly. Schedule an event on a short + # timer so packets start getting read and do the setup there. + schedule 100 msecs { setup_test() }; +} From 4695060d751881be5081bc0af0393711606d25b7 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 16 Jan 2025 17:08:23 -0700 Subject: [PATCH 20/52] Allow opening and closing backends to be async --- scripts/base/frameworks/storage/main.zeek | 24 ++++-- src/storage/Backend.cc | 77 ++++++++++++++----- src/storage/Backend.h | 65 +++++++++++----- src/storage/Manager.cc | 49 +++++++----- src/storage/Manager.h | 22 +++++- src/storage/backend/redis/Redis.cc | 9 ++- src/storage/backend/redis/Redis.h | 4 +- src/storage/backend/sqlite/SQLite.cc | 16 ++-- src/storage/backend/sqlite/SQLite.h | 4 +- src/storage/storage.bif | 56 +++++++++++--- .../Baseline/plugins.storage/zeek-stderr | 4 +- .../out | 2 + .../.stderr | 2 +- .../storage-plugin/src/StorageDummy.cc | 7 +- .../plugins/storage-plugin/src/StorageDummy.h | 5 +- .../base/frameworks/storage/sqlite-basic.zeek | 32 +++++--- 16 files changed, 271 insertions(+), 107 deletions(-) diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index e7fa684466..a1d478ca78 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -42,17 +42,29 @@ export { ## validation of values passed to :zeek:see:`Storage::put` as well as ## for type conversions for return values from :zeek:see:`Storage::get`. ## + ## async_mode: Indicates whether this operation should happen asynchronously. If + ## this is T, the call must happen as part of a :zeek:see:`when` + ## statement. This flag is overridden and set to F when reading pcaps, + ## since time won't move forward the same as when caputring live + ## traffic. + ## ## Returns: A handle to the new backend connection, or ``F`` if the connection ## failed. global open_backend: function(btype: Storage::Backend, options: any, key_type: any, - val_type: any): opaque of Storage::BackendHandle; + val_type: any, async_mode: bool &default=F): opaque of Storage::BackendHandle; ## Closes an existing backend connection. ## ## backend: A handle to a backend connection. ## + ## async_mode: Indicates whether this operation should happen asynchronously. If + ## this is T, the call must happen as part of a :zeek:see:`when` + ## statement. This flag is overridden and set to F when reading pcaps, + ## since time won't move forward the same as when caputring live + ## traffic. + ## ## Returns: A boolean indicating success or failure of the operation. - global close_backend: function(backend: opaque of Storage::BackendHandle): bool; + global close_backend: function(backend: opaque of Storage::BackendHandle, async_mode: bool &default=F): bool; ## Inserts a new entry into a backend. ## @@ -107,14 +119,14 @@ export { async_mode: bool &default=T): bool; } -function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle +function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any, async_mode: bool &default=F): opaque of Storage::BackendHandle { - return Storage::__open_backend(btype, options, key_type, val_type); + return Storage::__open_backend(btype, options, key_type, val_type, async_mode); } -function close_backend(backend: opaque of Storage::BackendHandle): bool +function close_backend(backend: opaque of Storage::BackendHandle, async_mode: bool &default=F): bool { - return Storage::__close_backend(backend); + return Storage::__close_backend(backend, async_mode); } function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index bf4e6b69d7..b4437b2f6f 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -6,12 +6,28 @@ #include "zeek/RunState.h" #include "zeek/Trigger.h" #include "zeek/broker/Data.h" +#include "zeek/storage/Manager.h" namespace zeek::storage { -ErrorResultCallback::ErrorResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc) +ResultCallback::ResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc) : trigger(std::move(trigger)), assoc(assoc) {} -ErrorResultCallback::~ErrorResultCallback() {} + +ResultCallback::~ResultCallback() {} + +void ResultCallback::Timeout() { + auto v = make_intrusive("Timeout during request"); + trigger->Cache(assoc, v.get()); +} + +void ResultCallback::ValComplete(Val* result) { + trigger->Cache(assoc, result); + Unref(result); + trigger->Release(); +} + +ErrorResultCallback::ErrorResultCallback(IntrusivePtr trigger, const void* assoc) + : ResultCallback(std::move(trigger), assoc) {} void ErrorResultCallback::Complete(const ErrorResult& res) { zeek::Val* result; @@ -21,19 +37,11 @@ void ErrorResultCallback::Complete(const ErrorResult& res) { else result = val_mgr->Bool(true).get(); - trigger->Cache(assoc, result); - Unref(result); - trigger->Release(); -} - -void ErrorResultCallback::Timeout() { - auto v = make_intrusive("Timeout during request"); - trigger->Cache(assoc, v.get()); + ValComplete(result); } ValResultCallback::ValResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc) - : trigger(std::move(trigger)), assoc(assoc) {} -ValResultCallback::~ValResultCallback() {} + : ResultCallback(std::move(trigger), assoc) {} void ValResultCallback::Complete(const ValResult& res) { zeek::Val* result; @@ -45,21 +53,50 @@ void ValResultCallback::Complete(const ValResult& res) { else result = new StringVal(res.error()); - trigger->Cache(assoc, result); - Unref(result); - trigger->Release(); + ValComplete(result); } -void ValResultCallback::Timeout() { - auto v = make_intrusive("Timeout during request"); - trigger->Cache(assoc, v.get()); +OpenResultCallback::OpenResultCallback(IntrusivePtr trigger, const void* assoc, + detail::BackendHandleVal* backend) + : ResultCallback(std::move(trigger), assoc), backend(backend) {} + +void OpenResultCallback::Complete(const ErrorResult& res) { + zeek::Val* result; + + if ( res ) { + result = new StringVal(res.value()); + } + else { + storage_mgr->AddBackendToMap(backend->backend); + result = backend; + } + + ValComplete(result); } -ErrorResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt) { +ErrorResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb) { key_type = std::move(kt); val_type = std::move(vt); - return DoOpen(std::move(options)); + auto res = DoOpen(std::move(options)); + + if ( (! native_async || zeek::run_state::reading_traces) && cb ) { + cb->Complete(res); + delete cb; + } + + return res; +} + +ErrorResult Backend::Close(ErrorResultCallback* cb) { + auto res = DoClose(cb); + + if ( (! native_async || zeek::run_state::reading_traces) && cb ) { + cb->Complete(res); + delete cb; + } + + return res; } ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expiration_time, ErrorResultCallback* cb) { diff --git a/src/storage/Backend.h b/src/storage/Backend.h index ea50b36c3d..ec5cd39f14 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -24,32 +24,39 @@ using ErrorResult = std::optional; // string value will store an error message if the result is null. using ValResult = zeek::expected; -// A callback result that returns an ErrorResult. -class ErrorResultCallback { + +// Base callback object for async operations. This is just here to allow some +// code reuse in the other callback methods. +class ResultCallback { public: - ErrorResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc); - ~ErrorResultCallback(); - void Complete(const ErrorResult& res); + ResultCallback(IntrusivePtr trigger, const void* assoc); + virtual ~ResultCallback(); void Timeout(); +protected: + void ValComplete(Val* result); + private: - zeek::detail::trigger::TriggerPtr trigger; + IntrusivePtr trigger; const void* assoc; }; +// A callback result that returns an ErrorResult. +class ErrorResultCallback : public ResultCallback { +public: + ErrorResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc); + virtual void Complete(const ErrorResult& res); +}; + // A callback result that returns a ValResult. -class ValResultCallback { +class ValResultCallback : public ResultCallback { public: ValResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc); - ~ValResultCallback(); void Complete(const ValResult& res); - void Timeout(); - -private: - zeek::detail::trigger::TriggerPtr trigger; - const void* assoc; }; +class OpenResultCallback; + class Backend : public zeek::Obj { public: /** @@ -89,7 +96,6 @@ public: * @param cb An optional callback object if being called via an async context. * @return An optional value potentially containing an error string if * needed. Will be unset if the operation succeeded. - * possible error string if the operation failed. */ ErrorResult Erase(ValPtr key, ErrorResultCallback* cb = nullptr); @@ -128,21 +134,30 @@ protected: * validation of types. * @param vt The script-side type of the values stored in the backend. Used for * validation of types and conversion during retrieval. + * @param cb An optional callback object if being called via an async context. * @return An optional value potentially containing an error string if * needed. Will be unset if the operation succeeded. */ - ErrorResult Open(RecordValPtr options, TypePtr kt, TypePtr vt); + ErrorResult Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb = nullptr); /** - * Finalizes the backend when it's being closed. Can be overridden by - * derived classes. + * Finalizes the backend when it's being closed. + * + * @param cb An optional callback object if being called via an async context. + * @return An optional value potentially containing an error string if + * needed. Will be unset if the operation succeeded. */ - virtual void Close() {} + ErrorResult Close(ErrorResultCallback* cb = nullptr); /** * The workhorse method for Open(). */ - virtual ErrorResult DoOpen(RecordValPtr options) = 0; + virtual ErrorResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) = 0; + + /** + * The workhorse method for Close(). + */ + virtual ErrorResult DoClose(ErrorResultCallback* cb = nullptr) = 0; /** * The workhorse method for Put(). @@ -196,4 +211,16 @@ protected: }; } // namespace detail + +// A callback for the Backend::Open() method that returns an error or a backend handle. +class OpenResultCallback : public ResultCallback { +public: + OpenResultCallback(IntrusivePtr trigger, const void* assoc, + detail::BackendHandleVal* backend); + void Complete(const ErrorResult& res); + +private: + detail::BackendHandleVal* backend; +}; + } // namespace zeek::storage diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index 987f54f55c..af632e8f15 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -23,8 +23,7 @@ void Manager::InitPostScript() { StartExpirationTimer(); } -zeek::expected Manager::OpenBackend(const Tag& type, RecordValPtr options, TypePtr key_type, - TypePtr val_type) { +zeek::expected Manager::Instantiate(const Tag& type) { Component* c = Lookup(type); if ( ! c ) { return zeek::unexpected( @@ -46,32 +45,37 @@ zeek::expected Manager::OpenBackend(const Tag& type, Re util::fmt("Failed to instantiate backend %s", GetComponentName(type).c_str())); } - if ( auto res = bp->Open(std::move(options), std::move(key_type), std::move(val_type)); res.has_value() ) { - return zeek::unexpected( - util::fmt("Failed to open backend %s: %s", GetComponentName(type).c_str(), res.value().c_str())); - } - - // TODO: post Storage::backend_opened event - - { - std::unique_lock lk(backends_mtx); - backends.push_back(bp); - } - return bp; } -void Manager::CloseBackend(BackendPtr backend) { +ErrorResult Manager::OpenBackend(BackendPtr backend, RecordValPtr options, TypePtr key_type, TypePtr val_type, + OpenResultCallback* cb) { + if ( auto res = backend->Open(std::move(options), std::move(key_type), std::move(val_type), cb); res.has_value() ) { + return util::fmt("Failed to open backend %s: %s", backend->Tag(), res.value().c_str()); + } + + if ( ! cb ) + AddBackendToMap(std::move(backend)); + + // TODO: post Storage::backend_opened event + + return std::nullopt; +} + +ErrorResult Manager::CloseBackend(BackendPtr backend, ErrorResultCallback* cb) { + // Remove from the list always, even if the close may fail below and even in an async context. { std::unique_lock lk(backends_mtx); auto it = std::find(backends.begin(), backends.end(), backend); - if ( it == backends.end() ) - return; - - backends.erase(it); + if ( it != backends.end() ) + backends.erase(it); } - backend->Close(); + if ( auto res = backend->Close(cb); res.has_value() ) { + return util::fmt("Failed to close backend %s: %s", backend->Tag(), res.value().c_str()); + } + + return std::nullopt; // TODO: post Storage::backend_lost event } @@ -88,4 +92,9 @@ void Manager::StartExpirationTimer() { new detail::ExpirationTimer(run_state::network_time + zeek::BifConst::Storage::expire_interval)); } +void Manager::AddBackendToMap(BackendPtr backend) { + std::unique_lock lk(backends_mtx); + backends.push_back(std::move(backend)); +} + } // namespace zeek::storage diff --git a/src/storage/Manager.h b/src/storage/Manager.h index 49957a678d..5dc9cd9ffc 100644 --- a/src/storage/Manager.h +++ b/src/storage/Manager.h @@ -34,9 +34,20 @@ public: void InitPostScript(); /** - * Opens a new storage backend. + * Instantiates a new backend object. The backend will be in a closed state, and OpenBackend() + * will need to be called to fully initialize it. * * @param type The tag for the type of backend being opened. + * @return A std::expected containing either a valid BackendPtr with the + * result of the operation or a string containing an error message for + * failure. + */ + zeek::expected Instantiate(const Tag& type); + + /** + * Opens a new storage backend. + * + * @param backend The backend object to open. * @param options A record val representing the configuration for this type of * backend. * @param key_type The script-side type of the keys stored in the backend. Used for @@ -46,19 +57,22 @@ public: * @return An optional value potentially containing an error string if needed. Will be * unset if the operation succeeded. */ - zeek::expected OpenBackend(const Tag& type, RecordValPtr options, TypePtr key_type, - TypePtr val_type); + ErrorResult OpenBackend(BackendPtr backend, RecordValPtr options, TypePtr key_type, TypePtr val_type, + OpenResultCallback* cb = nullptr); /** * Closes a storage backend. */ - void CloseBackend(BackendPtr backend); + ErrorResult CloseBackend(BackendPtr backend, ErrorResultCallback* cb = nullptr); protected: friend class storage::detail::ExpirationTimer; void Expire(); void StartExpirationTimer(); + friend class storage::OpenResultCallback; + void AddBackendToMap(BackendPtr backend); + private: std::vector backends; std::mutex backends_mtx; diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index a30c17e23a..fb215aa5ae 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -88,7 +88,7 @@ storage::BackendPtr Redis::Instantiate(std::string_view tag) { return make_intru * implementation must call \a Opened(); if not, it must call Error() * with a corresponding message. */ -ErrorResult Redis::DoOpen(RecordValPtr options) { +ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { // When reading traces we disable storage async mode globally (see src/storage/Backend.cc) since // time moves forward based on the pcap and not based on real time. async_mode = options->GetField("async_mode")->Get() && ! zeek::run_state::reading_traces; @@ -137,6 +137,9 @@ ErrorResult Redis::DoOpen(RecordValPtr options) { return errmsg; } + // TODO: Sort out how to pass the zeek callbacks for both open/done to the async + // callbacks from hiredis so they can return errors. + // The context is passed to the handler methods. Setting this data object // pointer allows us to look up the backend in the handlers. async_ctx->data = this; @@ -175,7 +178,7 @@ ErrorResult Redis::DoOpen(RecordValPtr options) { /** * Finalizes the backend when it's being closed. */ -void Redis::Close() { +ErrorResult Redis::DoClose(ErrorResultCallback* cb) { connected = false; if ( async_mode ) { @@ -191,6 +194,8 @@ void Redis::Close() { redisFree(ctx); ctx = nullptr; } + + return std::nullopt; } /** diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index 2d283f2965..ba32df6a88 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -30,12 +30,12 @@ public: /** * Called by the manager system to open the backend. */ - ErrorResult DoOpen(RecordValPtr options) override; + ErrorResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; /** * Finalizes the backend when it's being closed. */ - void Close() override; + ErrorResult DoClose(ErrorResultCallback* cb = nullptr) override; /** * Returns whether the backend is opened. diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index ece7eb1aff..8af6246a4b 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -13,7 +13,7 @@ storage::BackendPtr SQLite::Instantiate(std::string_view tag) { return make_intr /** * Called by the manager system to open the backend. */ -ErrorResult SQLite::DoOpen(RecordValPtr options) { +ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { if ( sqlite3_threadsafe() == 0 ) { std::string res = "SQLite reports that it is not threadsafe. Zeek needs a threadsafe version of " @@ -104,7 +104,9 @@ ErrorResult SQLite::DoOpen(RecordValPtr options) { /** * Finalizes the backend when it's being closed. */ -void SQLite::Close() { +ErrorResult SQLite::DoClose(ErrorResultCallback* cb) { + ErrorResult err_res; + if ( db ) { for ( const auto& [k, stmt] : prepared_stmts ) { sqlite3_finalize(stmt); @@ -114,15 +116,19 @@ void SQLite::Close() { char* errmsg; if ( int res = sqlite3_exec(db, "pragma optimize", NULL, NULL, &errmsg); res != SQLITE_OK ) { - Error(util::fmt("Sqlite failed to optimize at shutdown: %s", errmsg)); + err_res = util::fmt("Sqlite failed to optimize at shutdown: %s", errmsg); sqlite3_free(&errmsg); } - if ( int res = sqlite3_close_v2(db); res != SQLITE_OK ) - Error("Sqlite could not close connection"); + if ( int res = sqlite3_close_v2(db); res != SQLITE_OK ) { + if ( ! err_res.has_value() ) + err_res = "Sqlite could not close connection"; + } db = nullptr; } + + return err_res; } /** diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index 1a3a713909..f3eff1536a 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -20,12 +20,12 @@ public: /** * Called by the manager system to open the backend. */ - ErrorResult DoOpen(RecordValPtr options) override; + ErrorResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; /** * Finalizes the backend when it's being closed. */ - void Close() override; + ErrorResult DoClose(ErrorResultCallback* cb = nullptr) override; /** * Returns whether the backend is opened. diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 56a18a49aa..1a606f1e44 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -40,26 +40,46 @@ event Storage::backend_opened%(%); # Generated when a backend connection is lost event Storage::backend_lost%(%); -function Storage::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): opaque of Storage::BackendHandle +function Storage::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any, async_mode: bool%): opaque of Storage::BackendHandle %{ auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; Tag tag{btype_val}; - auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); - auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); - - auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; - auto b = storage_mgr->OpenBackend(tag, options_val, kt, vt); + auto b = storage_mgr->Instantiate(tag); if ( ! b.has_value() ) { emit_builtin_error(b.error().c_str()); return val_mgr->Bool(false); } - return make_intrusive(b.value()); + auto bo = make_intrusive(b.value()); + OpenResultCallback* cb = nullptr; + + if ( async_mode ) { + auto trigger = init_trigger(frame, b.value()); + if ( ! trigger ) + return val_mgr->Bool(false); + + cb = new OpenResultCallback(trigger, frame->GetTriggerAssoc(), bo.release()); + } + + auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; + auto open_res = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); + + if ( async_mode ) + return nullptr; + + if ( open_res.has_value() ) { + emit_builtin_error(open_res.value().c_str()); + return val_mgr->Bool(false); + } + + return bo; %} -function Storage::__close_backend%(backend: opaque of Storage::BackendHandle%) : bool +function Storage::__close_backend%(backend: opaque of Storage::BackendHandle, async_mode: bool%) : bool %{ auto b = dynamic_cast(backend); if ( ! b ) { @@ -70,7 +90,25 @@ function Storage::__close_backend%(backend: opaque of Storage::BackendHandle%) : // Return true here since the backend is already closed return val_mgr->Bool(true); - storage_mgr->CloseBackend(b->backend); + ErrorResultCallback* cb = nullptr; + + if ( async_mode ) { + auto trigger = init_trigger(frame, b->backend); + if ( ! trigger ) + return val_mgr->Bool(false); + + cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); + } + + auto result = storage_mgr->CloseBackend(b->backend, cb); + + if ( async_mode ) + return nullptr; + + if ( result.has_value() ) { + emit_builtin_error(result.value().c_str()); + return val_mgr->Bool(false); + } return val_mgr->Bool(true); %} diff --git a/testing/btest/Baseline/plugins.storage/zeek-stderr b/testing/btest/Baseline/plugins.storage/zeek-stderr index a1151e1362..a039e08e9f 100644 --- a/testing/btest/Baseline/plugins.storage/zeek-stderr +++ b/testing/btest/Baseline/plugins.storage/zeek-stderr @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. error in <...>/storage.zeek, line 37: Failed to retrieve data: Failed to find key (Storage::get(b, to_any_coerce key, F)) -error in <...>/storage.zeek, line 50: Failed to open backend STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts, to_any_coerce str, to_any_coerce str)) -error in <...>/storage.zeek, line 51: Invalid storage handle (Storage::close_backend(b2) and F) +error in <...>/storage.zeek, line 50: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts, to_any_coerce str, to_any_coerce str, F)) +error in <...>/storage.zeek, line 51: Invalid storage handle (Storage::close_backend(b2, F) and F) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out index 5125c8494e..e32898816e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out @@ -1,4 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +open successful put result, T get result, value5678 get result same as inserted, T +closed succesfully diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr index c6fae04e3a..e590742314 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/sqlite-error-handling.zeek, line 20: Failed to open backend SQLITE: SQLite call failed: unable to open database file (Storage::open_backend(Storage::SQLITE, to_any_coerce opts, to_any_coerce str, to_any_coerce str)) +error in <...>/sqlite-error-handling.zeek, line 20: Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file (Storage::open_backend(Storage::SQLITE, to_any_coerce opts, to_any_coerce str, to_any_coerce str, F)) error in <...>/sqlite-error-handling.zeek, line 28: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::put(b, (coerce [$key=bad_key, $value=value, $async_mode=F] to Storage::PutArgs))) diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc index ce71dd5c1c..c1460058a7 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -18,7 +18,7 @@ zeek::storage::BackendPtr StorageDummy::Instantiate(std::string_view tag) { * implementation must call \a Opened(); if not, it must call Error() * with a corresponding message. */ -zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options) { +zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options, zeek::storage::OpenResultCallback* cb) { bool open_fail = options->GetField("open_fail")->Get(); if ( open_fail ) return "open_fail was set to true, returning error"; @@ -31,7 +31,10 @@ zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options) { /** * Finalizes the backend when it's being closed. */ -void StorageDummy::Close() { open = false; } +zeek::storage::ErrorResult StorageDummy::DoClose(zeek::storage::ErrorResultCallback* cb) { + open = false; + return std::nullopt; +} /** * The workhorse method for Put(). This must be implemented by plugins. diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h index b770080dca..e45acf3335 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.h +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -21,12 +21,13 @@ public: /** * Called by the manager system to open the backend. */ - zeek::storage::ErrorResult DoOpen(zeek::RecordValPtr options) override; + zeek::storage::ErrorResult DoOpen(zeek::RecordValPtr options, + zeek::storage::OpenResultCallback* cb = nullptr) override; /** * Finalizes the backend when it's being closed. */ - void Close() override; + zeek::storage::ErrorResult DoClose(zeek::storage::ErrorResultCallback* cb = nullptr) override; /** * Returns whether the backend is opened. diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index 47c7d1d846..224eda3223 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -22,26 +22,36 @@ event zeek_init() { # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. - local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + when [opts, key, value] ( local b = Storage::open_backend(Storage::SQLITE, opts, str, str, T) ) { + print "open successful"; - when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) { - print "put result", res; + when [b, key, value] ( local put_res = Storage::put(b, [$key=key, $value=value]) ) { + print "put result", put_res; - when [b, key, value] ( local res2 = Storage::get(b, key) ) { - print "get result", res2; - print "get result same as inserted", value == (res2 as string); + when [b, key, value] ( local get_res = Storage::get(b, key) ) { + print "get result", get_res; + print "get result same as inserted", value == (get_res as string); - Storage::close_backend(b); - - terminate(); + when [b] ( local close_res = Storage::close_backend(b, T) ) { + print "closed succesfully"; + terminate(); + } timeout 5 sec { + print "close request timed out"; + terminate(); + } + } + timeout 5 sec { + print "get requeest timed out"; + terminate(); + } } timeout 5 sec { - print "get requeest timed out"; + print "put request timed out"; terminate(); } } timeout 5 sec { - print "put request timed out"; + print "open request timed out"; terminate(); } } From f1a7376e0ad9e9ad23bd61f93146ba3ed810e8fd Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 22 Jan 2025 17:00:38 -0700 Subject: [PATCH 21/52] Return generic result for get operations that includes error messages --- scripts/base/frameworks/storage/main.zeek | 8 +++-- scripts/base/init-bare.zeek | 8 +++++ src/storage/Backend.cc | 11 ++++--- src/storage/storage.bif | 29 ++++++++++++------- .../Baseline/plugins.storage/zeek-stderr | 6 ++-- .../out | 4 +-- .../.stderr | 1 - .../scripts.base.frameworks.storage.erase/out | 2 +- .../.stderr | 1 - .../out | 4 +-- .../out | 2 +- .../out | 2 +- .../out | 2 +- .../out | 4 +-- .../out | 6 ++-- .../out | 2 +- .../out | 2 +- testing/btest/plugins/storage.zeek | 5 +++- .../base/frameworks/storage/erase.zeek | 5 ++-- .../base/frameworks/storage/expiration.zeek | 8 +++-- .../base/frameworks/storage/overwriting.zeek | 3 +- .../storage/redis-async-reading-pcap.zeek | 3 +- .../base/frameworks/storage/redis-async.zeek | 3 +- .../frameworks/storage/redis-expiration.zeek | 3 +- .../base/frameworks/storage/redis-sync.zeek | 6 ++-- .../storage/sqlite-basic-reading-pcap.zeek | 3 +- .../base/frameworks/storage/sqlite-basic.zeek | 3 +- 27 files changed, 83 insertions(+), 53 deletions(-) diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index a1d478ca78..b0c725f9b2 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -95,9 +95,11 @@ export { ## Returns: A boolean indicating success or failure of the operation. Type ## comparison failures against the types passed to ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to - ## be returned. + ## be returned. The caller should check the validity of the value before + ## attempting to use it. If the value is unset, an error string may be + ## available to describe the failure. global get: function(backend: opaque of Storage::BackendHandle, key: any, - async_mode: bool &default=T): any; + async_mode: bool &default=T): val_result; ## Erases an entry from the backend. ## @@ -134,7 +136,7 @@ function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): return Storage::__put(backend, args$key, args$value, args$overwrite, args$expire_time, args$async_mode); } -function get(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T): any +function get(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T): val_result { return Storage::__get(backend, key, async_mode); } diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 9593efd0f1..a94d29e006 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -356,6 +356,14 @@ type ftp_port: record { valid: bool; ##< True if format was right. Only then are *h* and *p* valid. }; +## A generic type for returning either a value or an error string from a +## function or a BIF method. This is sort of equivalent to std::expected +## in C++. +type val_result: record { + val: any &optional; + error: string &optional; +}; + ## Statistics about what a TCP endpoint sent. ## ## .. zeek:see:: conn_stats diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index b4437b2f6f..5a94ba35e0 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -44,14 +44,13 @@ ValResultCallback::ValResultCallback(zeek::detail::trigger::TriggerPtr trigger, : ResultCallback(std::move(trigger), assoc) {} void ValResultCallback::Complete(const ValResult& res) { - zeek::Val* result; + static auto val_result_type = zeek::id::find_type("val_result"); + auto* result = new zeek::RecordVal(val_result_type); - if ( res ) { - result = res.value().get(); - Ref(result); - } + if ( res ) + result->Assign(0, res.value()); else - result = new StringVal(res.error()); + result->Assign(1, zeek::make_intrusive(res.error())); ValComplete(result); } diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 1a606f1e44..a5cf473ae6 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -149,22 +149,29 @@ function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, va return val_mgr->Bool(true); %} -function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T%): any +function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T%): val_result %{ + static auto val_result_type = id::find_type("val_result"); + auto val_result = make_intrusive(val_result_type); + auto b = dynamic_cast(backend); if ( ! b ) { - emit_builtin_error("Invalid storage handle", backend); - return val_mgr->Bool(false); + val_result->Assign(1, make_intrusive("Invalid storage handlle")); + return val_result; + } + else if ( ! b->backend->IsOpen() ) { + val_result->Assign(1, make_intrusive("Backend is closed")); + return val_result; } - else if ( ! b->backend->IsOpen() ) - return val_mgr->Bool(false); ValResultCallback* cb = nullptr; if ( async_mode ) { auto trigger = init_trigger(frame, b->backend); - if ( ! trigger ) - return val_mgr->Bool(false); + if ( ! trigger ) { + val_result->Assign(1, make_intrusive("Failed to create trigger")); + return val_result; + } cb = new ValResultCallback(trigger, frame->GetTriggerAssoc()); } @@ -176,11 +183,13 @@ function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, as return nullptr; if ( ! result.has_value() ) { - emit_builtin_error(util::fmt("Failed to retrieve data: %s", result.error().c_str())); - return val_mgr->Bool(false); + val_result->Assign(1, make_intrusive( + util::fmt("Failed to retrieve data: %s", result.error().c_str()))); + return val_result; } - return result.value(); + val_result->Assign(0, result.value()); + return val_result; %} function Storage::__erase%(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T%): bool diff --git a/testing/btest/Baseline/plugins.storage/zeek-stderr b/testing/btest/Baseline/plugins.storage/zeek-stderr index a039e08e9f..590aa27896 100644 --- a/testing/btest/Baseline/plugins.storage/zeek-stderr +++ b/testing/btest/Baseline/plugins.storage/zeek-stderr @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/storage.zeek, line 37: Failed to retrieve data: Failed to find key (Storage::get(b, to_any_coerce key, F)) -error in <...>/storage.zeek, line 50: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts, to_any_coerce str, to_any_coerce str, F)) -error in <...>/storage.zeek, line 51: Invalid storage handle (Storage::close_backend(b2, F) and F) +error in <...>/storage.zeek, line 41: Failed to retrieve data: Failed to find key +error in <...>/storage.zeek, line 53: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts, to_any_coerce str, to_any_coerce str, F)) +error in <...>/storage.zeek, line 54: Invalid storage handle (Storage::close_backend(b2, F) and F) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out b/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out index e8f3abe594..0b2661d2e3 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out @@ -1,7 +1,7 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. put result, T -get result, { +get result, [val={ [2] = b, [1] = a, [3] = c -} +}, error=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.erase/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.erase/.stderr index 886fb29d21..49d861c74c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.erase/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.erase/.stderr @@ -1,2 +1 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/erase.zeek, line 28: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(b, to_any_coerce key, F)) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out b/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out index 056ba93c85..7ca5354b61 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. erase result, T -get result, got empty result +get result, Failed to retrieve data: Failed to find row for key: no more rows available diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr index 65a0a4d3ed..97188487be 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/.stderr @@ -1,3 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -1627225025.686472 error in <...>/expiration.zeek, line 20: Failed to retrieve data: Failed to find row for key: no more rows available (Storage::get(backend, to_any_coerce key, F)) 1627225025.686472 received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out index f75c9fb9cc..08758e1bf2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out @@ -1,5 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. put result, T -get result, value7890 +get result, [val=value7890, error=] get result same as inserted, T -get result, F +get result, Failed to retrieve data: Failed to find row for key: no more rows available diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out b/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out index 809959c28f..d1338d7ba9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. put result, T -get result, value7890 +get result, [val=value7890, error=] get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out index 5125c8494e..b32c104878 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. put result, T -get result, value5678 +get result, [val=value5678, error=] get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out index 5125c8494e..b32c104878 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. put result, T -get result, value5678 +get result, [val=value5678, error=] get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out index 88d16bb4e3..996cf2ebac 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out @@ -1,5 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. put result, T -get result, value7890 +get result, [val=value7890, error=] get result same as inserted, T -get result after expiration, F +get result after expiration, [val=, error=Failed to retrieve data: GET returned key didn't exist] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out index e7d5445bc7..b77104dcd8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out @@ -1,7 +1,7 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. put result, T -get result, value1234 +get result, [val=value1234, error=] get result same as inserted, T overwrite put result, T -get result, value5678 -get result same as inserted after overwrite, F +get result, [val=value5678, error=] +get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out index 5125c8494e..b32c104878 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. put result, T -get result, value5678 +get result, [val=value5678, error=] get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out index e32898816e..1eca70373d 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out @@ -1,6 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. open successful put result, T -get result, value5678 +get result, [val=value5678, error=] get result same as inserted, T closed succesfully diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek index d759e4b7da..fd57797252 100644 --- a/testing/btest/plugins/storage.zeek +++ b/testing/btest/plugins/storage.zeek @@ -37,13 +37,16 @@ event zeek_init() { get_res = Storage::get(b, key, F); Storage::close_backend(b); + if ( get_res?$error ) + Reporter::error(get_res$error); + # Test attempting to use the closed handle. put_res = Storage::put(b, [$key="a", $value="b", $overwrite=F]); get_res = Storage::get(b, "a"); erase_res = Storage::erase(b, "a"); print(fmt("results of trying to use closed handle: get: %d, put: %d, erase: %d", - get_res, put_res, erase_res)); + get_res?$val, put_res, erase_res)); # Test failing to open the handle and test closing an invalid handle. opts$open_fail = T; diff --git a/testing/btest/scripts/base/frameworks/storage/erase.zeek b/testing/btest/scripts/base/frameworks/storage/erase.zeek index 515f2ab2ea..b6a3e540fc 100644 --- a/testing/btest/scripts/base/frameworks/storage/erase.zeek +++ b/testing/btest/scripts/base/frameworks/storage/erase.zeek @@ -26,9 +26,8 @@ event zeek_init() { print "erase result", res; local res2 = Storage::get(b, key, F); - if ( ! res2 as bool ) { - print "get result, got empty result"; - } + if ( res2?$error ) + print "get result", res2$error; Storage::close_backend(b); } diff --git a/testing/btest/scripts/base/frameworks/storage/expiration.zeek b/testing/btest/scripts/base/frameworks/storage/expiration.zeek index 46fb2be5a8..0913468004 100644 --- a/testing/btest/scripts/base/frameworks/storage/expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/expiration.zeek @@ -17,8 +17,11 @@ global key: string = "key1234"; global value: string = "value7890"; event check_removed() { + # This should return an error from the sqlite backend that there aren't any more + # rows available. local res2 = Storage::get(backend, key, F); - print "get result", res2; + if ( res2?$error ) + print "get result", res2$error; Storage::close_backend(backend); terminate(); @@ -36,7 +39,8 @@ event setup_test() { local res2 = Storage::get(backend, key, F); print "get result", res2; - print "get result same as inserted", value == (res2 as string); + if ( res2?$val ) + print "get result same as inserted", value == (res2$val as string); schedule 5 secs { check_removed() }; } diff --git a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek index 82b667b2b9..60211d1e2e 100644 --- a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek +++ b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek @@ -25,7 +25,8 @@ event zeek_init() { local res2 = Storage::get(b, key, F); print "get result", res2; - print "get result same as inserted", value == (res2 as string); + if ( res2?$val ) + print "get result same as inserted", value == (res2$val as string); Storage::close_backend(b); } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek index 0cd3dcc2b9..310e98fc4c 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -35,7 +35,8 @@ event zeek_init() { when [b, key, value] ( local res2 = Storage::get(b, key) ) { print "get result", res2; - print "get result same as inserted", value == (res2 as string); + if ( res2?$val ) + print "get result same as inserted", value == (res2$val as string); Storage::close_backend(b); } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek index 47f913e5ae..ae0da73906 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek @@ -37,7 +37,8 @@ event zeek_init() { when [b, key, value] ( local res2 = Storage::get(b, key) ) { print "get result", res2; - print "get result same as inserted", value == (res2 as string); + if ( res2?$val ) + print "get result same as inserted", value == (res2$val as string); Storage::close_backend(b); diff --git a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek index 810bbb102d..dbc050debf 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek @@ -47,7 +47,8 @@ event setup_test() { local res2 = Storage::get(b, key, F); print "get result", res2; - print "get result same as inserted", value == (res2 as string); + if ( res2?$val ) + print "get result same as inserted", value == (res2$val as string); schedule 5 secs { check_removed() }; } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek index d21d5979a8..d9dd372544 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -35,7 +35,8 @@ event zeek_init() { local res2 = Storage::get(b, key, F); print "get result", res2; - print "get result same as inserted", value == (res2 as string); + if ( res2?$val ) + print "get result same as inserted", value == (res2$val as string); local value2 = "value5678"; res = Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]); @@ -43,7 +44,8 @@ event zeek_init() { res2 = Storage::get(b, key, F); print "get result", res2; - print "get result same as inserted after overwrite", value == (res2 as string); + if ( res2?$val ) + print "get result same as inserted", value2 == (res2$val as string); Storage::close_backend(b); } diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek index dbd00662b3..6f2bc2d3a5 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek @@ -29,7 +29,8 @@ event zeek_init() { when [b, key, value] ( local res2 = Storage::get(b, key) ) { print "get result", res2; - print "get result same as inserted", value == (res2 as string); + if ( res2?$val ) + print "get result same as inserted", value == (res2$val as string); Storage::close_backend(b); diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index 224eda3223..684c86bce0 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -30,7 +30,8 @@ event zeek_init() { when [b, key, value] ( local get_res = Storage::get(b, key) ) { print "get result", get_res; - print "get result same as inserted", value == (get_res as string); + if ( get_res?$val ) + print "get result same as inserted", value == (get_res$val as string); when [b] ( local close_res = Storage::close_backend(b, T) ) { print "closed succesfully"; From 42ad5bbf7d5854df7b063fd011bf66bb62eb4a9c Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 24 Jan 2025 12:03:43 -0700 Subject: [PATCH 22/52] Add btest that uses a Redis backend in a cluster --- .../worker-1..stdout | 4 + .../worker-2..stdout | 3 + .../frameworks/storage/redis-cluster.zeek | 87 +++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-1..stdout create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-2..stdout create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-1..stdout new file mode 100644 index 0000000000..01c6005755 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-1..stdout @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +worker-1, put result, T +redis_data_written +worker-1, [val=5678, error=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-2..stdout new file mode 100644 index 0000000000..76ee9cc9df --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-2..stdout @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +redis_data_written +worker-2, [val=5678, error=] diff --git a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek new file mode 100644 index 0000000000..d6e1de7af3 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek @@ -0,0 +1,87 @@ +# @TEST-DOC: Tests Redis storage in a cluster environment + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT +# @TEST-PORT: BROKER_PORT1 +# @TEST-PORT: BROKER_PORT2 +# @TEST-PORT: BROKER_PORT3 + +# Generate a redis.conf file with the port defined above, but without the /tcp at the end of +# it. This also sets some paths in the conf to the testing directory. +# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf +# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf + +# @TEST-EXEC: btest-bg-run manager-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT +# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff worker-1/.stdout +# @TEST-EXEC: btest-diff worker-2/.stdout +# @TEST-EXEC: + +@load base/frameworks/storage +@load base/frameworks/cluster +@load policy/frameworks/storage/backend/redis +@load policy/frameworks/cluster/experimental + +@TEST-START-FILE cluster-layout.zeek +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT1"))], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT2")), $manager="manager-1"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT3")), $manager="manager-1"], +}; +@TEST-END-FILE + +global redis_data_written: event() &is_used; + +@if ( Cluster::local_node_type() == Cluster::WORKER ) + +global backend: opaque of Storage::BackendHandle; +type str: string; + +event zeek_init() { + local opts : Storage::Backend::Redis::Options; + opts$server_host = "127.0.0.1"; + opts$server_port = to_port(getenv("REDIS_PORT")); + opts$key_prefix = "testing"; + opts$async_mode = F; + + backend = Storage::open_backend(Storage::REDIS, opts, str, str); +} + +event redis_data_written() { + print "redis_data_written"; + local res = Storage::get(backend, "1234", F); + print Cluster::node, res; + Storage::close_backend(backend); + terminate(); +} + +@else + +global node_count: count = 0; + +event Cluster::node_down(name: string, id: string) { + ++node_count; + if ( node_count == 2 ) + terminate(); +} + +event redis_data_written() { + local e = Cluster::make_event(redis_data_written); + Cluster::publish(Cluster::worker_topic, e); +} + +@endif + +@if ( Cluster::node == "worker-1" ) + +event Cluster::Experimental::cluster_started() { + local res = Storage::put(backend, [$key="1234", $value="5678", $async_mode=F]); + print Cluster::node, "put result", res; + + local e = Cluster::make_event(redis_data_written); + Cluster::publish(Cluster::manager_topic, e); +} + +@endif From e8074c40d4562ebfa84d3a9dc984a9861c60e139 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 30 Jan 2025 19:11:28 -0700 Subject: [PATCH 23/52] Remove Backend::SupportsAsync --- src/storage/Backend.h | 6 ------ src/storage/storage.bif | 5 ----- 2 files changed, 11 deletions(-) diff --git a/src/storage/Backend.h b/src/storage/Backend.h index ec5cd39f14..23331800e0 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -104,12 +104,6 @@ public: */ virtual bool IsOpen() = 0; - /** - * Returns whether the backend's connection supports asynchronous commands. - * Defaults to true, but can be overridden by backends. - */ - virtual bool SupportsAsync() { return true; } - protected: // Allow the manager to call Open/Close. friend class storage::Manager; diff --git a/src/storage/storage.bif b/src/storage/storage.bif index a5cf473ae6..ae037378f5 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -8,11 +8,6 @@ using namespace zeek; using namespace zeek::storage; static zeek::detail::trigger::TriggerPtr init_trigger(zeek::detail::Frame* frame, const BackendPtr& b) { - if ( ! b->SupportsAsync() ) { - emit_builtin_error("Async mode requested but backend does not support async operations"); - return nullptr; - } - auto trigger = frame->GetTrigger(); if ( ! trigger ) { From 28951dccf128d87b9dbf73a8ae9568a20916d697 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sat, 1 Mar 2025 15:16:57 -0700 Subject: [PATCH 24/52] Split sync and async into separate script-land namespaces --- scripts/base/frameworks/storage/__load__.zeek | 4 +- scripts/base/frameworks/storage/async.zeek | 100 +++++++ scripts/base/frameworks/storage/main.zeek | 123 --------- scripts/base/frameworks/storage/sync.zeek | 98 +++++++ src/script_opt/FuncInfo.cc | 15 +- src/storage/storage.bif | 244 +++++++++++------- .../canonified_loaded_scripts.log | 4 +- .../Baseline/opt.ZAM-bif-tracking/output | 2 +- .../Baseline/plugins.storage/zeek-stderr | 4 +- .../.stderr | 4 +- testing/btest/opt/ZAM-bif-tracking.zeek | 15 +- testing/btest/plugins/storage.zeek | 24 +- .../frameworks/storage/compound-types.zeek | 8 +- .../base/frameworks/storage/erase.zeek | 10 +- .../base/frameworks/storage/expiration.zeek | 12 +- .../base/frameworks/storage/overwriting.zeek | 10 +- .../storage/redis-async-reading-pcap.zeek | 11 +- .../base/frameworks/storage/redis-async.zeek | 11 +- .../frameworks/storage/redis-cluster.zeek | 10 +- .../frameworks/storage/redis-expiration.zeek | 12 +- .../base/frameworks/storage/redis-sync.zeek | 14 +- .../storage/sqlite-basic-reading-pcap.zeek | 11 +- .../base/frameworks/storage/sqlite-basic.zeek | 10 +- .../storage/sqlite-error-handling.zeek | 12 +- 24 files changed, 465 insertions(+), 303 deletions(-) create mode 100644 scripts/base/frameworks/storage/async.zeek create mode 100644 scripts/base/frameworks/storage/sync.zeek diff --git a/scripts/base/frameworks/storage/__load__.zeek b/scripts/base/frameworks/storage/__load__.zeek index d551be57d3..0fbdc3168f 100644 --- a/scripts/base/frameworks/storage/__load__.zeek +++ b/scripts/base/frameworks/storage/__load__.zeek @@ -1 +1,3 @@ -@load ./main \ No newline at end of file +@load ./async +@load ./main +@load ./sync \ No newline at end of file diff --git a/scripts/base/frameworks/storage/async.zeek b/scripts/base/frameworks/storage/async.zeek new file mode 100644 index 0000000000..290b5fe424 --- /dev/null +++ b/scripts/base/frameworks/storage/async.zeek @@ -0,0 +1,100 @@ +##! Asynchronous operation methods for the storage framework. These methods must +##! be called as part of a :zeek:see:`when` statement. An error will be returned +##! otherwise. + +@load ./main + +module Storage::Async; + +export { + ## Opens a new backend connection based on a configuration object asynchronously. + ## + ## btype: A tag indicating what type of backend should be opened. These are + ## defined by the backend plugins loaded. + ## + ## options: A record containing the configuration for the connection. + ## + ## key_type: The script-level type of keys stored in the backend. Used for + ## validation of keys passed to other framework methods. + ## + ## val_type: The script-level type of keys stored in the backend. Used for + ## validation of values passed to :zeek:see:`Storage::Async::put` as + ## well as for type conversions for return values from + ## :zeek:see:`Storage::Async::get`. + ## + ## Returns: A handle to the new backend connection, or ``F`` if the connection + ## failed. + global open_backend: function(btype: Storage::Backend, options: any, key_type: any, + val_type: any): opaque of Storage::BackendHandle; + + ## Closes an existing backend connection asynchronously. + ## + ## backend: A handle to a backend connection. + ## + ## Returns: A boolean indicating success or failure of the operation. + global close_backend: function(backend: opaque of Storage::BackendHandle): bool; + + ## Inserts a new entry into a backend asynchronously. + ## + ## backend: A handle to a backend connection. + ## + ## args: A :zeek:see:`Storage::PutArgs` record containing the arguments for the + ## operation. + ## + ## Returns: A boolean indicating success or failure of the operation. Type + ## comparison failures against the types passed to + ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to + ## be returned. + global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool; + + ## Gets an entry from the backend asynchronously. + ## + ## backend: A handle to a backend connection. + ## + ## key: The key to look up. + ## + ## Returns: A boolean indicating success or failure of the operation. Type + ## comparison failures against the types passed to + ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to + ## be returned. The caller should check the validity of the value before + ## attempting to use it. If the value is unset, an error string may be + ## available to describe the failure. + global get: function(backend: opaque of Storage::BackendHandle, key: any): val_result; + + ## Erases an entry from the backend asynchronously. + ## + ## backend: A handle to a backend connection. + ## + ## key: The key to erase. + ## + ## Returns: A boolean indicating success or failure of the operation. Type + ## comparison failures against the types passed to + ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to + ## be returned. + global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; +} + +function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle +{ + return Storage::Async::__open_backend(btype, options, key_type, val_type); +} + +function close_backend(backend: opaque of Storage::BackendHandle): bool +{ + return Storage::Async::__close_backend(backend); +} + +function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool +{ + return Storage::Async::__put(backend, args$key, args$value, args$overwrite, args$expire_time); +} + +function get(backend: opaque of Storage::BackendHandle, key: any): val_result +{ + return Storage::Async::__get(backend, key); +} + +function erase(backend: opaque of Storage::BackendHandle, key: any): bool +{ + return Storage::Async::__erase(backend, key); +} diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index b0c725f9b2..cd46583aa9 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -20,128 +20,5 @@ export { # An interval of time until the entry is automatically removed from the # backend. expire_time: interval &default=0sec; - - # Indicates whether this operation should happen asynchronously. If this - # is true, the call to put must happen as part of a :zeek:see:`when` - # statement. This flag is overridden and set to F when reading pcaps, - # since time won't move forward the same as when caputring live traffic. - async_mode: bool &default=T; }; - - ## Opens a new backend connection based on a configuration object. - ## - ## btype: A tag indicating what type of backend should be opened. These are - ## defined by the backend plugins loaded. - ## - ## options: A record containing the configuration for the connection. - ## - ## key_type: The script-level type of keys stored in the backend. Used for - ## validation of keys passed to other framework methods. - ## - ## val_type: The script-level type of keys stored in the backend. Used for - ## validation of values passed to :zeek:see:`Storage::put` as well as - ## for type conversions for return values from :zeek:see:`Storage::get`. - ## - ## async_mode: Indicates whether this operation should happen asynchronously. If - ## this is T, the call must happen as part of a :zeek:see:`when` - ## statement. This flag is overridden and set to F when reading pcaps, - ## since time won't move forward the same as when caputring live - ## traffic. - ## - ## Returns: A handle to the new backend connection, or ``F`` if the connection - ## failed. - global open_backend: function(btype: Storage::Backend, options: any, key_type: any, - val_type: any, async_mode: bool &default=F): opaque of Storage::BackendHandle; - - ## Closes an existing backend connection. - ## - ## backend: A handle to a backend connection. - ## - ## async_mode: Indicates whether this operation should happen asynchronously. If - ## this is T, the call must happen as part of a :zeek:see:`when` - ## statement. This flag is overridden and set to F when reading pcaps, - ## since time won't move forward the same as when caputring live - ## traffic. - ## - ## Returns: A boolean indicating success or failure of the operation. - global close_backend: function(backend: opaque of Storage::BackendHandle, async_mode: bool &default=F): bool; - - ## Inserts a new entry into a backend. - ## - ## - ## backend: A handle to a backend connection. - ## - ## args: A :zeek:see:`Storage::PutArgs` record containing the arguments for the - ## operation. - ## - ## Returns: A boolean indicating success or failure of the operation. Type - ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to - ## be returned. - global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool; - - ## Gets an entry from the backend. - ## - ## backend: A handle to a backend connection. - ## - ## key: The key to look up. - ## - ## async_mode: Indicates whether this operation should happen asynchronously. If - ## this is T, the call must happen as part of a :zeek:see:`when` - ## statement. This flag is overridden and set to F when reading pcaps, - ## since time won't move forward the same as when caputring live - ## traffic. - ## - ## Returns: A boolean indicating success or failure of the operation. Type - ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to - ## be returned. The caller should check the validity of the value before - ## attempting to use it. If the value is unset, an error string may be - ## available to describe the failure. - global get: function(backend: opaque of Storage::BackendHandle, key: any, - async_mode: bool &default=T): val_result; - - ## Erases an entry from the backend. - ## - ## backend: A handle to a backend connection. - ## - ## key: The key to erase. - ## - ## async_mode: Indicates whether this operation should happen asynchronously. If - ## this is T, the call must happen as part of a :zeek:see:`when` - ## statement. This flag is overridden and set to F when reading pcaps, - ## since time won't move forward the same as when caputring live - ## traffic. - ## - ## Returns: A boolean indicating success or failure of the operation. Type - ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to - ## be returned. - global erase: function(backend: opaque of Storage::BackendHandle, key: any, - async_mode: bool &default=T): bool; -} - -function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any, async_mode: bool &default=F): opaque of Storage::BackendHandle -{ - return Storage::__open_backend(btype, options, key_type, val_type, async_mode); -} - -function close_backend(backend: opaque of Storage::BackendHandle, async_mode: bool &default=F): bool -{ - return Storage::__close_backend(backend, async_mode); -} - -function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool -{ - return Storage::__put(backend, args$key, args$value, args$overwrite, args$expire_time, args$async_mode); -} - -function get(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T): val_result -{ - return Storage::__get(backend, key, async_mode); -} - -function erase(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T): bool -{ - return Storage::__erase(backend, key, async_mode); } diff --git a/scripts/base/frameworks/storage/sync.zeek b/scripts/base/frameworks/storage/sync.zeek new file mode 100644 index 0000000000..b50859cef0 --- /dev/null +++ b/scripts/base/frameworks/storage/sync.zeek @@ -0,0 +1,98 @@ +##! Synchronous operation methods for the storage framework. + +@load ./main + +module Storage::Sync; + +export { + ## Opens a new backend connection based on a configuration object. + ## + ## btype: A tag indicating what type of backend should be opened. These are + ## defined by the backend plugins loaded. + ## + ## options: A record containing the configuration for the connection. + ## + ## key_type: The script-level type of keys stored in the backend. Used for + ## validation of keys passed to other framework methods. + ## + ## val_type: The script-level type of keys stored in the backend. Used for + ## validation of values passed to :zeek:see:`Storage::Sync::put` as well + ## as for type conversions for return values from + ## :zeek:see:`Storage::Sync::get`. + ## + ## Returns: A handle to the new backend connection, or ``F`` if the connection + ## failed. + global open_backend: function(btype: Storage::Backend, options: any, key_type: any, + val_type: any): opaque of Storage::BackendHandle; + + ## Closes an existing backend connection. + ## + ## backend: A handle to a backend connection. + ## + ## Returns: A boolean indicating success or failure of the operation. + global close_backend: function(backend: opaque of Storage::BackendHandle): bool; + + ## Inserts a new entry into a backend. + ## + ## backend: A handle to a backend connection. + ## + ## args: A :zeek:see:`Storage::PutArgs` record containing the arguments for the + ## operation. + ## + ## Returns: A boolean indicating success or failure of the operation. Type + ## comparison failures against the types passed to + ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to + ## be returned. + global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool; + + ## Gets an entry from the backend. + ## + ## backend: A handle to a backend connection. + ## + ## key: The key to look up. + ## + ## Returns: A boolean indicating success or failure of the operation. Type + ## comparison failures against the types passed to + ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to + ## be returned. The caller should check the validity of the value before + ## attempting to use it. If the value is unset, an error string may be + ## available to describe the failure. + global get: function(backend: opaque of Storage::BackendHandle, key: any): val_result; + + ## Erases an entry from the backend. + ## + ## backend: A handle to a backend connection. + ## + ## key: The key to erase. + ## + ## Returns: A boolean indicating success or failure of the operation. Type + ## comparison failures against the types passed to + ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to + ## be returned. + global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; +} + +function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle +{ + return Storage::Sync::__open_backend(btype, options, key_type, val_type); +} + +function close_backend(backend: opaque of Storage::BackendHandle): bool +{ + return Storage::Sync::__close_backend(backend); +} + +function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool +{ + return Storage::Sync::__put(backend, args$key, args$value, args$overwrite, args$expire_time); +} + +function get(backend: opaque of Storage::BackendHandle, key: any): val_result +{ + return Storage::Sync::__get(backend, key); +} + +function erase(backend: opaque of Storage::BackendHandle, key: any): bool +{ + return Storage::Sync::__erase(backend, key); +} diff --git a/src/script_opt/FuncInfo.cc b/src/script_opt/FuncInfo.cc index dea2ba7524..50d17440ca 100644 --- a/src/script_opt/FuncInfo.cc +++ b/src/script_opt/FuncInfo.cc @@ -148,11 +148,16 @@ static std::unordered_map func_attrs = { {"Reporter::warning", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Spicy::__resource_usage", ATTR_NO_ZEEK_SIDE_EFFECTS}, {"Spicy::__toggle_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS}, - {"Storage::__close_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS}, - {"Storage::__erase", ATTR_NO_SCRIPT_SIDE_EFFECTS}, - {"Storage::__get", ATTR_NO_SCRIPT_SIDE_EFFECTS}, - {"Storage::__open_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS}, - {"Storage::__put", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::Async::__close_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::Async::__erase", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::Async::__get", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::Async::__open_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::Async::__put", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::Sync::__close_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::Sync::__erase", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::Sync::__get", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::Sync::__open_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::Sync::__put", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Supervisor::__create", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Supervisor::__destroy", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Supervisor::__is_supervised", ATTR_IDEMPOTENT}, diff --git a/src/storage/storage.bif b/src/storage/storage.bif index ae037378f5..7035260232 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -7,13 +7,13 @@ using namespace zeek; using namespace zeek::storage; -static zeek::detail::trigger::TriggerPtr init_trigger(zeek::detail::Frame* frame, const BackendPtr& b) { - auto trigger = frame->GetTrigger(); +static zeek::detail::trigger::TriggerPtr init_trigger(zeek::detail::Frame* frame) { + auto trigger = frame->GetTrigger(); - if ( ! trigger ) { - emit_builtin_error("Async storage operations can only be called inside when-conditions"); - return nullptr; - } + if ( ! trigger ) { + emit_builtin_error("Asynchronous storage operations must be called via a when-condition"); + return nullptr; + } if ( auto timeout = trigger->TimeoutValue(); timeout < 0 ) { emit_builtin_error("Async Storage operations must specify a timeout block"); @@ -25,6 +25,21 @@ static zeek::detail::trigger::TriggerPtr init_trigger(zeek::detail::Frame* frame return {NewRef{}, trigger}; } + +static IntrusivePtr make_backend_handle(Val* btype) { + auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; + Tag tag{btype_val}; + + auto b = storage_mgr->Instantiate(tag); + + if ( ! b.has_value() ) { + emit_builtin_error(b.error().c_str()); + return nullptr; + } + + return make_intrusive(b.value()); +} + %%} module Storage; @@ -35,7 +50,130 @@ event Storage::backend_opened%(%); # Generated when a backend connection is lost event Storage::backend_lost%(%); -function Storage::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any, async_mode: bool%): opaque of Storage::BackendHandle +module Storage::Async; + +function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): opaque of Storage::BackendHandle + %{ + auto trigger = init_trigger(frame); + if ( ! trigger ) + return val_mgr->Bool(false); + + auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; + Tag tag{btype_val}; + + auto b = storage_mgr->Instantiate(tag); + + if ( ! b.has_value() ) { + emit_builtin_error(b.error().c_str()); + return nullptr; + } + + auto bh = make_intrusive(b.value()); + + auto cb = new OpenResultCallback(trigger, frame->GetTriggerAssoc(), bh.release()); + auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; + storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); + + return nullptr; + %} + +function Storage::Async::__close_backend%(backend: opaque of Storage::BackendHandle%) : bool + %{ + auto trigger = init_trigger(frame); + if ( ! trigger ) + return val_mgr->Bool(false); + + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + return val_mgr->Bool(true); + + auto cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); + storage_mgr->CloseBackend(b->backend, cb); + + return nullptr; + %} + +function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, + overwrite: bool, expire_time: interval%): bool + %{ + auto trigger = init_trigger(frame); + if ( ! trigger ) + return val_mgr->Bool(false); + + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + return val_mgr->Bool(false); + + auto cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); + auto key_v = IntrusivePtr{NewRef{}, key}; + auto val_v = IntrusivePtr{NewRef{}, value}; + b->backend->Put(key_v, val_v, overwrite, expire_time, cb); + + return nullptr; + %} + +function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: any%): val_result + %{ + static auto val_result_type = id::find_type("val_result"); + auto val_result = make_intrusive(val_result_type); + + auto trigger = init_trigger(frame); + if ( ! trigger ) { + val_result->Assign(1, make_intrusive("Failed to create trigger")); + return val_result; + } + + auto b = dynamic_cast(backend); + if ( ! b ) { + val_result->Assign(1, make_intrusive("Invalid storage handlle")); + return val_result; + } + else if ( ! b->backend->IsOpen() ) { + val_result->Assign(1, make_intrusive("Backend is closed")); + return val_result; + } + + auto cb = new ValResultCallback(trigger, frame->GetTriggerAssoc()); + auto key_v = IntrusivePtr{NewRef{}, key}; + auto result = b->backend->Get(key_v, cb); + + return nullptr; + %} + +function Storage::Async::__erase%(backend: opaque of Storage::BackendHandle, key: any%): bool + %{ + auto trigger = init_trigger(frame); + if ( ! trigger ) + return val_mgr->Bool(false); + + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + return val_mgr->Bool(false); + + auto cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); + auto key_v = IntrusivePtr{NewRef{}, key}; + b->backend->Erase(key_v, cb); + + return nullptr; + %} + +module Storage::Sync; + +function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): opaque of Storage::BackendHandle %{ auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; Tag tag{btype_val}; @@ -47,34 +185,20 @@ function Storage::__open_backend%(btype: Storage::Backend, options: any, key_typ return val_mgr->Bool(false); } - auto bo = make_intrusive(b.value()); - OpenResultCallback* cb = nullptr; - - if ( async_mode ) { - auto trigger = init_trigger(frame, b.value()); - if ( ! trigger ) - return val_mgr->Bool(false); - - cb = new OpenResultCallback(trigger, frame->GetTriggerAssoc(), bo.release()); - } - auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; - auto open_res = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); - - if ( async_mode ) - return nullptr; + auto open_res = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, nullptr); if ( open_res.has_value() ) { emit_builtin_error(open_res.value().c_str()); return val_mgr->Bool(false); } - return bo; + return make_intrusive(b.value()); %} -function Storage::__close_backend%(backend: opaque of Storage::BackendHandle, async_mode: bool%) : bool +function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHandle%) : bool %{ auto b = dynamic_cast(backend); if ( ! b ) { @@ -85,20 +209,7 @@ function Storage::__close_backend%(backend: opaque of Storage::BackendHandle, as // Return true here since the backend is already closed return val_mgr->Bool(true); - ErrorResultCallback* cb = nullptr; - - if ( async_mode ) { - auto trigger = init_trigger(frame, b->backend); - if ( ! trigger ) - return val_mgr->Bool(false); - - cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); - } - - auto result = storage_mgr->CloseBackend(b->backend, cb); - - if ( async_mode ) - return nullptr; + auto result = storage_mgr->CloseBackend(b->backend, nullptr); if ( result.has_value() ) { emit_builtin_error(result.value().c_str()); @@ -108,8 +219,8 @@ function Storage::__close_backend%(backend: opaque of Storage::BackendHandle, as return val_mgr->Bool(true); %} -function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, - overwrite: bool, expire_time: interval, async_mode: bool &default=T%): bool +function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, + overwrite: bool, expire_time: interval%): bool %{ auto b = dynamic_cast(backend); if ( ! b ) { @@ -119,22 +230,9 @@ function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, va else if ( ! b->backend->IsOpen() ) return val_mgr->Bool(false); - ErrorResultCallback* cb = nullptr; - - if ( async_mode ) { - auto trigger = init_trigger(frame, b->backend); - if ( ! trigger ) - return val_mgr->Bool(false); - - cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); - } - auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; - auto result = b->backend->Put(key_v, val_v, overwrite, expire_time, cb); - - if ( async_mode ) - return nullptr; + auto result = b->backend->Put(key_v, val_v, overwrite, expire_time, nullptr); if ( result.has_value() ) { emit_builtin_error(util::fmt("Failed to store data: %s", result.value().c_str())); @@ -144,7 +242,7 @@ function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, va return val_mgr->Bool(true); %} -function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T%): val_result +function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: any%): val_result %{ static auto val_result_type = id::find_type("val_result"); auto val_result = make_intrusive(val_result_type); @@ -159,27 +257,12 @@ function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, as return val_result; } - ValResultCallback* cb = nullptr; - - if ( async_mode ) { - auto trigger = init_trigger(frame, b->backend); - if ( ! trigger ) { - val_result->Assign(1, make_intrusive("Failed to create trigger")); - return val_result; - } - - cb = new ValResultCallback(trigger, frame->GetTriggerAssoc()); - } - auto key_v = IntrusivePtr{NewRef{}, key}; - auto result = b->backend->Get(key_v, cb); - - if ( async_mode ) - return nullptr; + auto result = b->backend->Get(key_v, nullptr); if ( ! result.has_value() ) { val_result->Assign(1, make_intrusive( - util::fmt("Failed to retrieve data: %s", result.error().c_str()))); + util::fmt("Failed to retrieve data: %s", result.error().c_str()))); return val_result; } @@ -187,7 +270,7 @@ function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, as return val_result; %} -function Storage::__erase%(backend: opaque of Storage::BackendHandle, key: any, async_mode: bool &default=T%): bool +function Storage::Sync::__erase%(backend: opaque of Storage::BackendHandle, key: any%): bool %{ auto b = dynamic_cast(backend); if ( ! b ) { @@ -197,21 +280,8 @@ function Storage::__erase%(backend: opaque of Storage::BackendHandle, key: any, else if ( ! b->backend->IsOpen() ) return val_mgr->Bool(false); - ErrorResultCallback* cb = nullptr; - - if ( async_mode ) { - auto trigger = init_trigger(frame, b->backend); - if ( ! trigger ) - return val_mgr->Bool(false); - - cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); - } - auto key_v = IntrusivePtr{NewRef{}, key}; - auto result = b->backend->Erase(key_v, cb); - - if ( async_mode ) - return nullptr; + auto result = b->backend->Erase(key_v, nullptr); if ( result.has_value() ) { emit_builtin_error(util::fmt("Failed to erase data for key: %s", result.value().c_str())); diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index ebf298afa4..c6a813054b 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -369,7 +369,9 @@ scripts/base/init-default.zeek scripts/base/frameworks/telemetry/main.zeek scripts/base/misc/version.zeek scripts/base/frameworks/storage/__load__.zeek - scripts/base/frameworks/storage/main.zeek + scripts/base/frameworks/storage/async.zeek + scripts/base/frameworks/storage/main.zeek + scripts/base/frameworks/storage/sync.zeek scripts/base/frameworks/spicy/__load__.zeek scripts/base/frameworks/spicy/main.zeek scripts/base/protocols/conn/__load__.zeek diff --git a/testing/btest/Baseline/opt.ZAM-bif-tracking/output b/testing/btest/Baseline/opt.ZAM-bif-tracking/output index 4fab29b90a..972665cbdb 100644 --- a/testing/btest/Baseline/opt.ZAM-bif-tracking/output +++ b/testing/btest/Baseline/opt.ZAM-bif-tracking/output @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -551 seen BiFs, 0 unseen BiFs (), 0 new BiFs () +556 seen BiFs, 0 unseen BiFs (), 0 new BiFs () diff --git a/testing/btest/Baseline/plugins.storage/zeek-stderr b/testing/btest/Baseline/plugins.storage/zeek-stderr index 590aa27896..e114ceba55 100644 --- a/testing/btest/Baseline/plugins.storage/zeek-stderr +++ b/testing/btest/Baseline/plugins.storage/zeek-stderr @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. error in <...>/storage.zeek, line 41: Failed to retrieve data: Failed to find key -error in <...>/storage.zeek, line 53: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts, to_any_coerce str, to_any_coerce str, F)) -error in <...>/storage.zeek, line 54: Invalid storage handle (Storage::close_backend(b2, F) and F) +error in <...>/sync.zeek, line 74: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::Sync::__open_backend(Storage::Sync::btype, Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) +error in <...>/sync.zeek, line 79: Invalid storage handle (Storage::Sync::__close_backend(Storage::Sync::backend) and F) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr index e590742314..e54152627d 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/sqlite-error-handling.zeek, line 20: Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file (Storage::open_backend(Storage::SQLITE, to_any_coerce opts, to_any_coerce str, to_any_coerce str, F)) -error in <...>/sqlite-error-handling.zeek, line 28: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::put(b, (coerce [$key=bad_key, $value=value, $async_mode=F] to Storage::PutArgs))) +error in <...>/sync.zeek, line 74: Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file (Storage::Sync::__open_backend(Storage::Sync::btype, Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) +error in <...>/sync.zeek, line 84: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::Sync::__put(Storage::Sync::backend, Storage::Sync::args$key, Storage::Sync::args$value, Storage::Sync::args$overwrite, Storage::Sync::args$expire_time)) diff --git a/testing/btest/opt/ZAM-bif-tracking.zeek b/testing/btest/opt/ZAM-bif-tracking.zeek index 2a906c365f..1d11c555c5 100644 --- a/testing/btest/opt/ZAM-bif-tracking.zeek +++ b/testing/btest/opt/ZAM-bif-tracking.zeek @@ -177,11 +177,16 @@ global known_BiFs = set( "Reporter::warning", "Spicy::__resource_usage", "Spicy::__toggle_analyzer", - "Storage::__close_backend", - "Storage::__erase", - "Storage::__get", - "Storage::__open_backend", - "Storage::__put", + "Storage::Async::__close_backend", + "Storage::Async::__erase", + "Storage::Async::__get", + "Storage::Async::__open_backend", + "Storage::Async::__put", + "Storage::Sync::__close_backend", + "Storage::Sync::__erase", + "Storage::Sync::__get", + "Storage::Sync::__open_backend", + "Storage::Sync::__put", "Supervisor::__create", "Supervisor::__destroy", "Supervisor::__is_supervised", diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek index fd57797252..92a598586f 100644 --- a/testing/btest/plugins/storage.zeek +++ b/testing/btest/plugins/storage.zeek @@ -8,7 +8,7 @@ # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff zeek-stderr -@load base/frameworks/storage +@load base/frameworks/storage/sync # Create a typename here that can be passed down into get(). type str: string; @@ -26,30 +26,30 @@ event zeek_init() { # Test basic operation. The second get() should return an error # as the key should have been erased. - local b = Storage::open_backend(Storage::STORAGEDUMMY, opts, str, str); - local put_res = Storage::put(b, [$key=key, $value=value, $overwrite=F, $async_mode=F]); - local get_res = Storage::get(b, key, F); + local b = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, str, str); + local put_res = Storage::Sync::put(b, [$key=key, $value=value, $overwrite=F]); + local get_res = Storage::Sync::get(b, key); if ( get_res is bool ) { print("Got an invalid value in response!"); } - local erase_res = Storage::erase(b, key, F); - get_res = Storage::get(b, key, F); - Storage::close_backend(b); + local erase_res = Storage::Sync::erase(b, key); + get_res = Storage::Sync::get(b, key); + Storage::Sync::close_backend(b); if ( get_res?$error ) Reporter::error(get_res$error); # Test attempting to use the closed handle. - put_res = Storage::put(b, [$key="a", $value="b", $overwrite=F]); - get_res = Storage::get(b, "a"); - erase_res = Storage::erase(b, "a"); + put_res = Storage::Sync::put(b, [$key="a", $value="b", $overwrite=F]); + get_res = Storage::Sync::get(b, "a"); + erase_res = Storage::Sync::erase(b, "a"); print(fmt("results of trying to use closed handle: get: %d, put: %d, erase: %d", get_res?$val, put_res, erase_res)); # Test failing to open the handle and test closing an invalid handle. opts$open_fail = T; - local b2 = Storage::open_backend(Storage::STORAGEDUMMY, opts, str, str); - Storage::close_backend(b2); + local b2 = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, str, str); + Storage::Sync::close_backend(b2); } diff --git a/testing/btest/scripts/base/frameworks/storage/compound-types.zeek b/testing/btest/scripts/base/frameworks/storage/compound-types.zeek index 4a90422c87..7b37289179 100644 --- a/testing/btest/scripts/base/frameworks/storage/compound-types.zeek +++ b/testing/btest/scripts/base/frameworks/storage/compound-types.zeek @@ -3,7 +3,7 @@ # @TEST-EXEC: btest-diff out # @TEST-EXEC: btest-diff .stderr -@load base/frameworks/storage +@load base/frameworks/storage/sync @load policy/frameworks/storage/backend/sqlite type Color: enum { @@ -64,11 +64,11 @@ event zeek_init() { value[2] = "b"; value[3] = "c"; - local b = Storage::open_backend(Storage::SQLITE, opts, Rec, tbl); + local b = Storage::Sync::open_backend(Storage::SQLITE, opts, Rec, tbl); - local res = Storage::put(b, [$key=key, $value=value, $async_mode=F]); + local res = Storage::Sync::put(b, [$key=key, $value=value]); print "put result", res; - local res2 = Storage::get(b, key, F); + local res2 = Storage::Sync::get(b, key); print "get result", res2; } diff --git a/testing/btest/scripts/base/frameworks/storage/erase.zeek b/testing/btest/scripts/base/frameworks/storage/erase.zeek index b6a3e540fc..25594d9123 100644 --- a/testing/btest/scripts/base/frameworks/storage/erase.zeek +++ b/testing/btest/scripts/base/frameworks/storage/erase.zeek @@ -4,7 +4,7 @@ # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr -@load base/frameworks/storage +@load base/frameworks/storage/sync @load policy/frameworks/storage/backend/sqlite # Create a typename here that can be passed down into get(). @@ -20,14 +20,14 @@ event zeek_init() { # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. - local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); - local res = Storage::erase(b, key, F); + local res = Storage::Sync::erase(b, key); print "erase result", res; - local res2 = Storage::get(b, key, F); + local res2 = Storage::Sync::get(b, key); if ( res2?$error ) print "get result", res2$error; - Storage::close_backend(b); + Storage::Sync::close_backend(b); } diff --git a/testing/btest/scripts/base/frameworks/storage/expiration.zeek b/testing/btest/scripts/base/frameworks/storage/expiration.zeek index 0913468004..8332987a38 100644 --- a/testing/btest/scripts/base/frameworks/storage/expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/expiration.zeek @@ -3,7 +3,7 @@ # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr -@load base/frameworks/storage +@load base/frameworks/storage/sync @load policy/frameworks/storage/backend/sqlite redef Storage::expire_interval = 2 secs; @@ -19,11 +19,11 @@ global value: string = "value7890"; event check_removed() { # This should return an error from the sqlite backend that there aren't any more # rows available. - local res2 = Storage::get(backend, key, F); + local res2 = Storage::Sync::get(backend, key); if ( res2?$error ) print "get result", res2$error; - Storage::close_backend(backend); + Storage::Sync::close_backend(backend); terminate(); } @@ -32,12 +32,12 @@ event setup_test() { opts$database_path = "storage-test.sqlite"; opts$table_name = "testing"; - backend = Storage::open_backend(Storage::SQLITE, opts, str, str); + backend = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); - local res = Storage::put(backend, [$key=key, $value=value, $expire_time=2 secs, $async_mode=F]); + local res = Storage::Sync::put(backend, [$key=key, $value=value, $expire_time=2 secs]); print "put result", res; - local res2 = Storage::get(backend, key, F); + local res2 = Storage::Sync::get(backend, key); print "get result", res2; if ( res2?$val ) print "get result same as inserted", value == (res2$val as string); diff --git a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek index 60211d1e2e..8aeabecaf6 100644 --- a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek +++ b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek @@ -4,7 +4,7 @@ # @TEST-EXEC: btest-diff out # @TEST-EXEC: btest-diff .stderr -@load base/frameworks/storage +@load base/frameworks/storage/sync @load policy/frameworks/storage/backend/sqlite # Create a typename here that can be passed down into get(). @@ -18,15 +18,15 @@ event zeek_init() { local key = "key1234"; local value = "value7890"; - local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); - local res = Storage::put(b, [$key=key, $value=value, $async_mode=F]); + local res = Storage::Sync::put(b, [$key=key, $value=value]); print "put result", res; - local res2 = Storage::get(b, key, F); + local res2 = Storage::Sync::get(b, key); print "get result", res2; if ( res2?$val ) print "get result same as inserted", value == (res2$val as string); - Storage::close_backend(b); + Storage::Sync::close_backend(b); } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek index 310e98fc4c..758b0e409f 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -12,7 +12,8 @@ # @TEST-EXEC: btest-diff out -@load base/frameworks/storage +@load base/frameworks/storage/sync +@load base/frameworks/storage/async @load policy/frameworks/storage/backend/redis # Create a typename here that can be passed down into open_backend() @@ -28,17 +29,17 @@ event zeek_init() { local key = "key1234"; local value = "value5678"; - local b = Storage::open_backend(Storage::REDIS, opts, str, str); + local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); - when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) { + when [b, key, value] ( local res = Storage::Async::put(b, [$key=key, $value=value]) ) { print "put result", res; - when [b, key, value] ( local res2 = Storage::get(b, key) ) { + when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) { print "get result", res2; if ( res2?$val ) print "get result same as inserted", value == (res2$val as string); - Storage::close_backend(b); + Storage::Sync::close_backend(b); } timeout 5 sec { print "get requeest timed out"; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek index ae0da73906..6d4b06383f 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek @@ -12,7 +12,8 @@ # @TEST-EXEC: btest-diff out -@load base/frameworks/storage +@load base/frameworks/storage/async +@load base/frameworks/storage/sync @load policy/frameworks/storage/backend/redis redef exit_only_after_terminate = T; @@ -30,17 +31,17 @@ event zeek_init() { local key = "key1234"; local value = "value5678"; - local b = Storage::open_backend(Storage::REDIS, opts, str, str); + local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); - when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) { + when [b, key, value] ( local res = Storage::Async::put(b, [$key=key, $value=value]) ) { print "put result", res; - when [b, key, value] ( local res2 = Storage::get(b, key) ) { + when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) { print "get result", res2; if ( res2?$val ) print "get result same as inserted", value == (res2$val as string); - Storage::close_backend(b); + Storage::Sync::close_backend(b); terminate(); } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek index d6e1de7af3..1fc9140cbb 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek @@ -19,7 +19,7 @@ # @TEST-EXEC: btest-diff worker-2/.stdout # @TEST-EXEC: -@load base/frameworks/storage +@load base/frameworks/storage/sync @load base/frameworks/cluster @load policy/frameworks/storage/backend/redis @load policy/frameworks/cluster/experimental @@ -46,14 +46,14 @@ event zeek_init() { opts$key_prefix = "testing"; opts$async_mode = F; - backend = Storage::open_backend(Storage::REDIS, opts, str, str); + backend = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); } event redis_data_written() { print "redis_data_written"; - local res = Storage::get(backend, "1234", F); + local res = Storage::Sync::get(backend, "1234"); print Cluster::node, res; - Storage::close_backend(backend); + Storage::Sync::close_backend(backend); terminate(); } @@ -77,7 +77,7 @@ event redis_data_written() { @if ( Cluster::node == "worker-1" ) event Cluster::Experimental::cluster_started() { - local res = Storage::put(backend, [$key="1234", $value="5678", $async_mode=F]); + local res = Storage::Sync::put(backend, [$key="1234", $value="5678"]); print Cluster::node, "put result", res; local e = Cluster::make_event(redis_data_written); diff --git a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek index dbc050debf..8b813d6a36 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek @@ -12,7 +12,7 @@ # @TEST-EXEC: btest-diff out -@load base/frameworks/storage +@load base/frameworks/storage/sync @load policy/frameworks/storage/backend/redis redef Storage::expire_interval = 2 secs; @@ -26,10 +26,10 @@ global key: string = "key1234"; global value: string = "value7890"; event check_removed() { - local res2 = Storage::get(b, key, F); + local res2 = Storage::Sync::get(b, key); print "get result after expiration", res2; - Storage::close_backend(b); + Storage::Sync::close_backend(b); terminate(); } @@ -40,12 +40,12 @@ event setup_test() { opts$key_prefix = "testing"; opts$async_mode = F; - b = Storage::open_backend(Storage::REDIS, opts, str, str); + b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); - local res = Storage::put(b, [$key=key, $value=value, $async_mode=F, $expire_time=2 secs]); + local res = Storage::Sync::put(b, [$key=key, $value=value, $expire_time=2 secs]); print "put result", res; - local res2 = Storage::get(b, key, F); + local res2 = Storage::Sync::get(b, key); print "get result", res2; if ( res2?$val ) print "get result same as inserted", value == (res2$val as string); diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek index d9dd372544..8579aa5a0c 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -12,7 +12,7 @@ # @TEST-EXEC: btest-diff out -@load base/frameworks/storage +@load base/frameworks/storage/sync @load policy/frameworks/storage/backend/redis # Create a typename here that can be passed down into open_backend() @@ -28,24 +28,24 @@ event zeek_init() { local key = "key1234"; local value = "value1234"; - local b = Storage::open_backend(Storage::REDIS, opts, str, str); + local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); - local res = Storage::put(b, [$key=key, $value=value, $async_mode=F]); + local res = Storage::Sync::put(b, [$key=key, $value=value]); print "put result", res; - local res2 = Storage::get(b, key, F); + local res2 = Storage::Sync::get(b, key); print "get result", res2; if ( res2?$val ) print "get result same as inserted", value == (res2$val as string); local value2 = "value5678"; - res = Storage::put(b, [$key=key, $value=value2, $overwrite=T, $async_mode=F]); + res = Storage::Sync::put(b, [$key=key, $value=value2, $overwrite=T]); print "overwrite put result", res; - res2 = Storage::get(b, key, F); + res2 = Storage::Sync::get(b, key); print "get result", res2; if ( res2?$val ) print "get result same as inserted", value2 == (res2$val as string); - Storage::close_backend(b); + Storage::Sync::close_backend(b); } diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek index 6f2bc2d3a5..30aabfaf63 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek @@ -3,7 +3,8 @@ # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr -@load base/frameworks/storage +@load base/frameworks/storage/async +@load base/frameworks/storage/sync @load policy/frameworks/storage/backend/sqlite redef exit_only_after_terminate = T; @@ -22,17 +23,17 @@ event zeek_init() { # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. - local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); - when [b, key, value] ( local res = Storage::put(b, [$key=key, $value=value]) ) { + when [b, key, value] ( local res = Storage::Async::put(b, [$key=key, $value=value]) ) { print "put result", res; - when [b, key, value] ( local res2 = Storage::get(b, key) ) { + when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) { print "get result", res2; if ( res2?$val ) print "get result same as inserted", value == (res2$val as string); - Storage::close_backend(b); + Storage::Sync::close_backend(b); terminate(); } diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index 684c86bce0..7e9e5383b9 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -3,7 +3,7 @@ # @TEST-EXEC: btest-diff out # @TEST-EXEC: btest-diff .stderr -@load base/frameworks/storage +@load base/frameworks/storage/async @load policy/frameworks/storage/backend/sqlite redef exit_only_after_terminate = T; @@ -22,18 +22,18 @@ event zeek_init() { # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. - when [opts, key, value] ( local b = Storage::open_backend(Storage::SQLITE, opts, str, str, T) ) { + when [opts, key, value] ( local b = Storage::Async::open_backend(Storage::SQLITE, opts, str, str) ) { print "open successful"; - when [b, key, value] ( local put_res = Storage::put(b, [$key=key, $value=value]) ) { + when [b, key, value] ( local put_res = Storage::Async::put(b, [$key=key, $value=value]) ) { print "put result", put_res; - when [b, key, value] ( local get_res = Storage::get(b, key) ) { + when [b, key, value] ( local get_res = Storage::Async::get(b, key) ) { print "get result", get_res; if ( get_res?$val ) print "get result same as inserted", value == (get_res$val as string); - when [b] ( local close_res = Storage::close_backend(b, T) ) { + when [b] ( local close_res = Storage::Async::close_backend(b) ) { print "closed succesfully"; terminate(); } timeout 5 sec { diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek index 4d7d248fc4..84bffa4dc3 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek @@ -3,7 +3,7 @@ # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr -@load base/frameworks/storage +@load base/frameworks/storage/sync @load base/frameworks/reporter @load policy/frameworks/storage/backend/sqlite @@ -17,18 +17,18 @@ event zeek_init() { opts$table_name = "testing"; # This should report an error in .stderr and reporter.log - local b = Storage::open_backend(Storage::SQLITE, opts, str, str); + local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); # Open a valid database file opts$database_path = "test.sqlite"; - b = Storage::open_backend(Storage::SQLITE, opts, str, str); + b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); local bad_key: count = 12345; local value = "abcde"; - Storage::put(b, [$key=bad_key, $value=value, $async_mode=F]); + Storage::Sync::put(b, [$key=bad_key, $value=value]); # Close the backend and then attempt to use the closed handle - Storage::close_backend(b); - local res = Storage::put(b, [$key="a", $value="b", $async_mode=F]); + Storage::Sync::close_backend(b); + local res = Storage::Sync::put(b, [$key="a", $value="b"]); print fmt("Put result on closed handle: %d", res); } From 64f3969434ac2d23dbe7f7f480b938e218818aa3 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 6 Feb 2025 12:50:41 -0700 Subject: [PATCH 25/52] Always register backend for expiration, check for open during loop --- src/storage/Backend.cc | 7 ++----- src/storage/Manager.cc | 11 ++++++----- src/storage/Manager.h | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index 5a94ba35e0..591d72eda7 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -62,13 +62,10 @@ OpenResultCallback::OpenResultCallback(IntrusivePtrAddBackendToMap(backend->backend); + else result = backend; - } ValComplete(result); } diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index af632e8f15..53afa52ebb 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -54,8 +54,7 @@ ErrorResult Manager::OpenBackend(BackendPtr backend, RecordValPtr options, TypeP return util::fmt("Failed to open backend %s: %s", backend->Tag(), res.value().c_str()); } - if ( ! cb ) - AddBackendToMap(std::move(backend)); + RegisterBackend(std::move(backend)); // TODO: post Storage::backend_opened event @@ -83,8 +82,10 @@ ErrorResult Manager::CloseBackend(BackendPtr backend, ErrorResultCallback* cb) { void Manager::Expire() { DBG_LOG(DBG_STORAGE, "Expire running, have %zu backends to check", backends.size()); std::unique_lock lk(backends_mtx); - for ( const auto& b : backends ) - b->Expire(); + for ( const auto& b : backends ) { + if ( b->IsOpen() ) + b->Expire(); + } } void Manager::StartExpirationTimer() { @@ -92,7 +93,7 @@ void Manager::StartExpirationTimer() { new detail::ExpirationTimer(run_state::network_time + zeek::BifConst::Storage::expire_interval)); } -void Manager::AddBackendToMap(BackendPtr backend) { +void Manager::RegisterBackend(BackendPtr backend) { std::unique_lock lk(backends_mtx); backends.push_back(std::move(backend)); } diff --git a/src/storage/Manager.h b/src/storage/Manager.h index 5dc9cd9ffc..2974f218ad 100644 --- a/src/storage/Manager.h +++ b/src/storage/Manager.h @@ -71,7 +71,7 @@ protected: void StartExpirationTimer(); friend class storage::OpenResultCallback; - void AddBackendToMap(BackendPtr backend); + void RegisterBackend(BackendPtr backend); private: std::vector backends; From a485b1d237e54c4539b25c19f67375df2a680b29 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 6 Feb 2025 14:30:10 -0700 Subject: [PATCH 26/52] Make backend options a record, move actual options to be sub-records --- scripts/base/frameworks/storage/async.zeek | 5 +++-- scripts/base/frameworks/storage/main.zeek | 7 ++++++- scripts/base/frameworks/storage/sync.zeek | 5 +++-- .../frameworks/storage/backend/redis/main.zeek | 4 ++++ .../frameworks/storage/backend/sqlite/main.zeek | 4 ++++ src/storage/backend/redis/Redis.cc | 16 +++++++--------- src/storage/backend/sqlite/SQLite.cc | 7 ++++--- .../btest/Baseline/plugins.storage/zeek-stderr | 6 +++--- .../.stderr | 4 ++-- .../plugins/storage-plugin/src/StorageDummy.cc | 3 ++- testing/btest/plugins/storage.zeek | 10 +++++++--- .../base/frameworks/storage/compound-types.zeek | 5 ++--- .../scripts/base/frameworks/storage/erase.zeek | 5 ++--- .../base/frameworks/storage/expiration.zeek | 5 ++--- .../base/frameworks/storage/overwriting.zeek | 5 ++--- .../storage/redis-async-reading-pcap.zeek | 7 ++----- .../base/frameworks/storage/redis-async.zeek | 7 ++----- .../base/frameworks/storage/redis-cluster.zeek | 7 ++----- .../frameworks/storage/redis-expiration.zeek | 7 ++----- .../base/frameworks/storage/redis-sync.zeek | 7 ++----- .../storage/sqlite-basic-reading-pcap.zeek | 5 ++--- .../base/frameworks/storage/sqlite-basic.zeek | 5 ++--- .../storage/sqlite-error-handling.zeek | 8 ++++---- 23 files changed, 71 insertions(+), 73 deletions(-) diff --git a/scripts/base/frameworks/storage/async.zeek b/scripts/base/frameworks/storage/async.zeek index 290b5fe424..3b2ba08dcd 100644 --- a/scripts/base/frameworks/storage/async.zeek +++ b/scripts/base/frameworks/storage/async.zeek @@ -24,7 +24,7 @@ export { ## ## Returns: A handle to the new backend connection, or ``F`` if the connection ## failed. - global open_backend: function(btype: Storage::Backend, options: any, key_type: any, + global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, val_type: any): opaque of Storage::BackendHandle; ## Closes an existing backend connection asynchronously. @@ -74,7 +74,8 @@ export { global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; } -function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle +function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, + val_type: any): opaque of Storage::BackendHandle { return Storage::Async::__open_backend(btype, options, key_type, val_type); } diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index cd46583aa9..66ab1c2744 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -5,7 +5,12 @@ module Storage; export { - ## Record for passing arguments to :zeek:see:`Storage::put`. + ## Base record for backend options. Backend plugins can redef this record to add + ## relevant fields to it. + type BackendOptions: record {}; + + ## Record for passing arguments to :zeek:see:`Storage::Async::put` and + ## :zeek:see:`Storage::Sync::put`. type PutArgs: record { # The key to store the value under. key: any; diff --git a/scripts/base/frameworks/storage/sync.zeek b/scripts/base/frameworks/storage/sync.zeek index b50859cef0..90aaa6a302 100644 --- a/scripts/base/frameworks/storage/sync.zeek +++ b/scripts/base/frameworks/storage/sync.zeek @@ -22,7 +22,7 @@ export { ## ## Returns: A handle to the new backend connection, or ``F`` if the connection ## failed. - global open_backend: function(btype: Storage::Backend, options: any, key_type: any, + global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, val_type: any): opaque of Storage::BackendHandle; ## Closes an existing backend connection. @@ -72,7 +72,8 @@ export { global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; } -function open_backend(btype: Storage::Backend, options: any, key_type: any, val_type: any): opaque of Storage::BackendHandle +function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, + val_type: any): opaque of Storage::BackendHandle { return Storage::Sync::__open_backend(btype, options, key_type, val_type); } diff --git a/scripts/policy/frameworks/storage/backend/redis/main.zeek b/scripts/policy/frameworks/storage/backend/redis/main.zeek index 413c201083..35e6a6f27f 100644 --- a/scripts/policy/frameworks/storage/backend/redis/main.zeek +++ b/scripts/policy/frameworks/storage/backend/redis/main.zeek @@ -34,4 +34,8 @@ export { # traffic. async_mode: bool &default=T; }; + + redef record Storage::BackendOptions += { + redis: Options &optional; + }; } diff --git a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek index 6cab5911f2..9b4fd386f8 100644 --- a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek +++ b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek @@ -28,4 +28,8 @@ export { ["temp_store"] = "memory" ); }; + + redef record Storage::BackendOptions += { + sqlite: Options &optional; + }; } diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index fb215aa5ae..576edd801e 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -83,29 +83,27 @@ storage::BackendPtr Redis::Instantiate(std::string_view tag) { return make_intru /** * Called by the manager system to open the backend. - * - * Derived classes must implement this method. If successful, the - * implementation must call \a Opened(); if not, it must call Error() - * with a corresponding message. */ ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { + RecordValPtr backend_options = options->GetField("redis"); + // When reading traces we disable storage async mode globally (see src/storage/Backend.cc) since // time moves forward based on the pcap and not based on real time. - async_mode = options->GetField("async_mode")->Get() && ! zeek::run_state::reading_traces; - key_prefix = options->GetField("key_prefix")->ToStdString(); + async_mode = backend_options->GetField("async_mode")->Get() && ! zeek::run_state::reading_traces; + key_prefix = backend_options->GetField("key_prefix")->ToStdString(); DBG_LOG(DBG_STORAGE, "Redis backend: running in async mode? %d", async_mode); redisOptions opt = {0}; - StringValPtr host = options->GetField("server_host"); + StringValPtr host = backend_options->GetField("server_host"); if ( host ) { - PortValPtr port = options->GetField("server_port"); + PortValPtr port = backend_options->GetField("server_port"); server_addr = util::fmt("%s:%d", host->ToStdStringView().data(), port->Port()); REDIS_OPTIONS_SET_TCP(&opt, host->ToStdStringView().data(), port->Port()); } else { - StringValPtr unix_sock = options->GetField("server_unix_socket"); + StringValPtr unix_sock = backend_options->GetField("server_unix_socket"); if ( ! unix_sock ) return util::fmt( "Either server_host/server_port or server_unix_socket must be set in Redis options record"); diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index 8af6246a4b..25b47cca0f 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -28,9 +28,10 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { sqlite3_enable_shared_cache(1); #endif - StringValPtr path = options->GetField("database_path"); + RecordValPtr backend_options = options->GetField("sqlite"); + StringValPtr path = backend_options->GetField("database_path"); full_path = zeek::filesystem::path(path->ToStdString()).string(); - table_name = options->GetField("table_name")->ToStdString(); + table_name = backend_options->GetField("table_name")->ToStdString(); auto open_res = checkError(sqlite3_open_v2(full_path.c_str(), &db, @@ -61,7 +62,7 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { return err; } - auto tuning_params = options->GetField("tuning_params")->ToMap(); + auto tuning_params = backend_options->GetField("tuning_params")->ToMap(); for ( const auto& [k, v] : tuning_params ) { auto ks = k->AsListVal()->Idx(0)->AsStringVal(); auto vs = v->AsStringVal(); diff --git a/testing/btest/Baseline/plugins.storage/zeek-stderr b/testing/btest/Baseline/plugins.storage/zeek-stderr index e114ceba55..8dcb29da7c 100644 --- a/testing/btest/Baseline/plugins.storage/zeek-stderr +++ b/testing/btest/Baseline/plugins.storage/zeek-stderr @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/storage.zeek, line 41: Failed to retrieve data: Failed to find key -error in <...>/sync.zeek, line 74: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::Sync::__open_backend(Storage::Sync::btype, Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) -error in <...>/sync.zeek, line 79: Invalid storage handle (Storage::Sync::__close_backend(Storage::Sync::backend) and F) +error in <...>/storage.zeek, line 45: Failed to retrieve data: Failed to find key +error in <...>/sync.zeek, line 75: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::Sync::__open_backend(Storage::Sync::btype, to_any_coerce Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) +error in <...>/sync.zeek, line 80: Invalid storage handle (Storage::Sync::__close_backend(Storage::Sync::backend) and F) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr index e54152627d..1013a7f2f4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/sync.zeek, line 74: Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file (Storage::Sync::__open_backend(Storage::Sync::btype, Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) -error in <...>/sync.zeek, line 84: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::Sync::__put(Storage::Sync::backend, Storage::Sync::args$key, Storage::Sync::args$value, Storage::Sync::args$overwrite, Storage::Sync::args$expire_time)) +error in <...>/sync.zeek, line 75: Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file (Storage::Sync::__open_backend(Storage::Sync::btype, to_any_coerce Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) +error in <...>/sync.zeek, line 85: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::Sync::__put(Storage::Sync::backend, Storage::Sync::args$key, Storage::Sync::args$value, Storage::Sync::args$overwrite, Storage::Sync::args$expire_time)) diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc index c1460058a7..8007131f3d 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -19,7 +19,8 @@ zeek::storage::BackendPtr StorageDummy::Instantiate(std::string_view tag) { * with a corresponding message. */ zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options, zeek::storage::OpenResultCallback* cb) { - bool open_fail = options->GetField("open_fail")->Get(); + zeek::RecordValPtr backend_options = options->GetField("dummy"); + bool open_fail = backend_options->GetField("open_fail")->Get(); if ( open_fail ) return "open_fail was set to true, returning error"; diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek index 92a598586f..fe876ff31a 100644 --- a/testing/btest/plugins/storage.zeek +++ b/testing/btest/plugins/storage.zeek @@ -17,9 +17,13 @@ type StorageDummyOpts : record { open_fail: bool; }; +redef record Storage::BackendOptions += { + dummy: StorageDummyOpts &optional; +}; + event zeek_init() { - local opts : StorageDummyOpts; - opts$open_fail = F; + local opts : Storage::BackendOptions; + opts$dummy = [$open_fail = F]; local key = "key1234"; local value = "value5678"; @@ -49,7 +53,7 @@ event zeek_init() { get_res?$val, put_res, erase_res)); # Test failing to open the handle and test closing an invalid handle. - opts$open_fail = T; + opts$dummy$open_fail = T; local b2 = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, str, str); Storage::Sync::close_backend(b2); } diff --git a/testing/btest/scripts/base/frameworks/storage/compound-types.zeek b/testing/btest/scripts/base/frameworks/storage/compound-types.zeek index 7b37289179..ba68c4e006 100644 --- a/testing/btest/scripts/base/frameworks/storage/compound-types.zeek +++ b/testing/btest/scripts/base/frameworks/storage/compound-types.zeek @@ -37,9 +37,8 @@ type tbl: table[count] of string; event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "types_test.sqlite"; - opts$table_name = "types_testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "types_test.sqlite", $table_name = "types_testing"]; local key : Rec; key$hello = "hello"; diff --git a/testing/btest/scripts/base/frameworks/storage/erase.zeek b/testing/btest/scripts/base/frameworks/storage/erase.zeek index 25594d9123..585da798b2 100644 --- a/testing/btest/scripts/base/frameworks/storage/erase.zeek +++ b/testing/btest/scripts/base/frameworks/storage/erase.zeek @@ -12,9 +12,8 @@ type str: string; event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "storage-test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; local key = "key1234"; diff --git a/testing/btest/scripts/base/frameworks/storage/expiration.zeek b/testing/btest/scripts/base/frameworks/storage/expiration.zeek index 8332987a38..d53e23fa36 100644 --- a/testing/btest/scripts/base/frameworks/storage/expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/expiration.zeek @@ -28,9 +28,8 @@ event check_removed() { } event setup_test() { - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "storage-test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; backend = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); diff --git a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek index 8aeabecaf6..a31360802f 100644 --- a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek +++ b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek @@ -11,9 +11,8 @@ type str: string; event zeek_init() { - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "storage-test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; local key = "key1234"; local value = "value7890"; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek index 758b0e409f..dec60fce83 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -20,11 +20,8 @@ type str: string; event zeek_init() { - local opts : Storage::Backend::Redis::Options; - opts$server_host = "127.0.0.1"; - opts$server_port = to_port(getenv("REDIS_PORT")); - opts$key_prefix = "testing"; - opts$async_mode = T; + local opts : Storage::BackendOptions; + opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = T]; local key = "key1234"; local value = "value5678"; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek index 6d4b06383f..37947d349d 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek @@ -22,11 +22,8 @@ redef exit_only_after_terminate = T; type str: string; event zeek_init() { - local opts : Storage::Backend::Redis::Options; - opts$server_host = "127.0.0.1"; - opts$server_port = to_port(getenv("REDIS_PORT")); - opts$key_prefix = "testing"; - opts$async_mode = T; + local opts : Storage::BackendOptions; + opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = T]; local key = "key1234"; local value = "value5678"; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek index 1fc9140cbb..3336c31101 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek @@ -40,11 +40,8 @@ global backend: opaque of Storage::BackendHandle; type str: string; event zeek_init() { - local opts : Storage::Backend::Redis::Options; - opts$server_host = "127.0.0.1"; - opts$server_port = to_port(getenv("REDIS_PORT")); - opts$key_prefix = "testing"; - opts$async_mode = F; + local opts : Storage::BackendOptions; + opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F]; backend = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek index 8b813d6a36..6c9b1398e8 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek @@ -34,11 +34,8 @@ event check_removed() { } event setup_test() { - local opts : Storage::Backend::Redis::Options; - opts$server_host = "127.0.0.1"; - opts$server_port = to_port(getenv("REDIS_PORT")); - opts$key_prefix = "testing"; - opts$async_mode = F; + local opts : Storage::BackendOptions; + opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F]; b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek index 8579aa5a0c..cd65443290 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -19,11 +19,8 @@ type str: string; event zeek_init() { - local opts : Storage::Backend::Redis::Options; - opts$server_host = "127.0.0.1"; - opts$server_port = to_port(getenv("REDIS_PORT")); - opts$key_prefix = "testing"; - opts$async_mode = F; + local opts : Storage::BackendOptions; + opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F]; local key = "key1234"; local value = "value1234"; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek index 30aabfaf63..76989fa1c5 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek @@ -14,9 +14,8 @@ type str: string; event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "test.sqlite", $table_name = "testing"]; local key = "key1234"; local value = "value5678"; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index 7e9e5383b9..63d262e55b 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -13,9 +13,8 @@ type str: string; event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "test.sqlite", $table_name="testing"]; local key = "key1234"; local value = "value5678"; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek index 84bffa4dc3..899ba497aa 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek @@ -12,15 +12,15 @@ type str: string; event zeek_init() { # Test opening a database with an invalid path - local opts : Storage::Backend::SQLite::Options; - opts$database_path = "/this/path/should/not/exist/test.sqlite"; - opts$table_name = "testing"; + local opts : Storage::BackendOptions; + opts$sqlite = [$database_path = "/this/path/should/not/exist/test.sqlite", + $table_name = "testing"]; # This should report an error in .stderr and reporter.log local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); # Open a valid database file - opts$database_path = "test.sqlite"; + opts$sqlite$database_path = "test.sqlite"; b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); local bad_key: count = 12345; From cad48cebd48a095688cff0f3a1e10feab91a1940 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 6 Feb 2025 15:18:24 -0700 Subject: [PATCH 27/52] Pass network-time-based expiration time to backends instead of an interval --- src/storage/Backend.h | 3 ++- src/storage/backend/redis/Redis.cc | 10 ++++------ src/storage/backend/sqlite/SQLite.cc | 3 --- src/storage/storage.bif | 3 +++ 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/storage/Backend.h b/src/storage/Backend.h index 23331800e0..071ce1f507 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -71,7 +71,8 @@ public: * @param value the value for the pair. * @param overwrite whether an existing value for a key should be overwritten. * @param expiration_time the time when this entry should be automatically - * removed. Set to zero to disable expiration. + * removed. Set to zero to disable expiration. This time is based on the current network + * time. * @param cb An optional callback object if being called via an async context. * @return An optional value potentially containing an error string if * needed Will be unset if the operation succeeded. diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 576edd801e..5e081b821f 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -209,12 +209,10 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira format.append(" NX"); // Use built-in expiration if reading live data, since time will move - // forward consistently. If reading pcaps, we'll do something else. + // forward consistently. If reading pcaps, we'll do something else. if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) format.append(" PXAT %d"); - double expire_time = expiration_time + run_state::network_time; - auto json_key = key->ToJSON()->ToStdString(); auto json_value = value->ToJSON()->ToStdString(); @@ -222,7 +220,7 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira int status; if ( expiration_time > 0.0 ) status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), - json_value.data(), static_cast(expire_time * 1e6)); + json_value.data(), static_cast(expiration_time * 1e6)); else status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), json_value.data()); @@ -234,7 +232,7 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira redisReply* reply; if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) reply = (redisReply*)redisCommand(ctx, format.c_str(), key_prefix.data(), json_key.data(), - json_value.data(), static_cast(expire_time * 1e6)); + json_value.data(), static_cast(expiration_time * 1e6)); else reply = (redisReply*)redisCommand(ctx, format.c_str(), key_prefix.data(), json_key.data(), json_value.data()); @@ -254,7 +252,7 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira format += " %f %s"; redisReply* reply = - (redisReply*)redisCommand(ctx, format.c_str(), key_prefix.data(), expire_time, json_key.data()); + (redisReply*)redisCommand(ctx, format.c_str(), key_prefix.data(), expiration_time, json_key.data()); if ( ! reply ) return util::fmt("ZADD operation failed: %s", ctx->errstr); diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index 25b47cca0f..34f1aebcfd 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -162,9 +162,6 @@ ErrorResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expir return res; } - if ( expiration_time != 0 ) - expiration_time += run_state::network_time; - if ( auto res = checkError(sqlite3_bind_double(stmt, 3, expiration_time)); res.has_value() ) { sqlite3_reset(stmt); return res; diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 7035260232..6c68e5f638 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -114,6 +114,9 @@ function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: else if ( ! b->backend->IsOpen() ) return val_mgr->Bool(false); + if ( expire_time > 0.0 ) + expire_time += run_state::network_time; + auto cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; From 40f60f26b36fde9fd050f2cef28a8cdf096a81fc Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 7 Feb 2025 16:55:40 -0700 Subject: [PATCH 28/52] Run expiration on a separate thread --- scripts/base/init-bare.zeek | 5 +++-- src/3rdparty | 2 +- src/storage/Manager.cc | 40 +++++++++++++++++++++++++++++++------ src/storage/Manager.h | 7 ++++++- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index a94d29e006..625415586f 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -6222,8 +6222,9 @@ module Storage; export { ## The interval used by the storage framework for automatic expiration - ## of elements in all backends that don't support it natively. - const expire_interval = 5.0 secs &redef; + ## of elements in all backends that don't support it natively, or if + ## using expiration while reading pcap files. + const expire_interval = 15.0 secs &redef; } module GLOBAL; diff --git a/src/3rdparty b/src/3rdparty index 0d15c8be14..059a4a369f 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 0d15c8be14851914c8fd2d378bb700dae7e7b991 +Subproject commit 059a4a369f2a52d8013f0645b69e1bf2194e97c6 diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index 53afa52ebb..18f3c9cb35 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -2,17 +2,31 @@ #include "zeek/storage/Manager.h" +#include + #include "zeek/Desc.h" +#include "zeek/RunState.h" #include "const.bif.netvar_h" +std::atomic_flag expire_running; + namespace zeek::storage { void detail::ExpirationTimer::Dispatch(double t, bool is_expire) { if ( is_expire ) return; - storage_mgr->Expire(); + // If there isn't an active thread, spin up a new one. Expiration may take + // some time to complete and we want it to get all the way done before we + // start another one running. If this causes us to skip a cycle, that's not + // a big deal as the next cycle will catch anything that should be expired + // in the interim. + if ( ! expire_running.test_and_set() ) { + DBG_LOG(DBG_STORAGE, "Starting new expiration thread"); + storage_mgr->expiration_thread = std::jthread([]() { storage_mgr->Expire(); }); + } + storage_mgr->StartExpirationTimer(); } @@ -62,7 +76,9 @@ ErrorResult Manager::OpenBackend(BackendPtr backend, RecordValPtr options, TypeP } ErrorResult Manager::CloseBackend(BackendPtr backend, ErrorResultCallback* cb) { - // Remove from the list always, even if the close may fail below and even in an async context. + // Expiration runs on a separate thread and loops over the vector of backends. The mutex + // here ensures exclusive access. This one happens in a block because we can remove the + // backend from the vector before actually closing it. { std::unique_lock lk(backends_mtx); auto it = std::find(backends.begin(), backends.end(), backend); @@ -80,22 +96,34 @@ ErrorResult Manager::CloseBackend(BackendPtr backend, ErrorResultCallback* cb) { } void Manager::Expire() { - DBG_LOG(DBG_STORAGE, "Expire running, have %zu backends to check", backends.size()); + // Expiration runs on a separate thread and loops over the vector of backends. The mutex + // here ensures exclusive access. std::unique_lock lk(backends_mtx); - for ( const auto& b : backends ) { - if ( b->IsOpen() ) - b->Expire(); + + DBG_LOG(DBG_STORAGE, "Expiration running, have %zu backends to check", backends.size()); + + for ( auto it = backends.begin(); it != backends.end() && ! run_state::terminating; ++it ) { + if ( (*it)->IsOpen() ) + (*it)->Expire(); } + + expire_running.clear(); } void Manager::StartExpirationTimer() { zeek::detail::timer_mgr->Add( new detail::ExpirationTimer(run_state::network_time + zeek::BifConst::Storage::expire_interval)); + DBG_LOG(DBG_STORAGE, "Next expiration check at %f", + run_state::network_time + zeek::BifConst::Storage::expire_interval); } void Manager::RegisterBackend(BackendPtr backend) { + // Expiration runs on a separate thread and loops over the vector of backends. The mutex + // here ensures exclusive access. std::unique_lock lk(backends_mtx); + backends.push_back(std::move(backend)); + DBG_LOG(DBG_STORAGE, "Registered backends: %zu", backends.size()); } } // namespace zeek::storage diff --git a/src/storage/Manager.h b/src/storage/Manager.h index 2974f218ad..2f240e0486 100644 --- a/src/storage/Manager.h +++ b/src/storage/Manager.h @@ -3,7 +3,9 @@ #pragma once #include +#include +#include "zeek/3rdparty/jthread.hpp" #include "zeek/Timer.h" #include "zeek/plugin/ComponentManager.h" #include "zeek/storage/Backend.h" @@ -65,10 +67,13 @@ public: */ ErrorResult CloseBackend(BackendPtr backend, ErrorResultCallback* cb = nullptr); + void Expire(); + protected: friend class storage::detail::ExpirationTimer; - void Expire(); + void RunExpireThread(); void StartExpirationTimer(); + std::jthread expiration_thread; friend class storage::OpenResultCallback; void RegisterBackend(BackendPtr backend); From c247de8ec37c3ba1216ff534984626a30f50f813 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 14 Feb 2025 14:24:55 -0800 Subject: [PATCH 29/52] Redis: Rework everything to only use async mode --- .../storage/backend/redis/main.zeek | 11 - src/storage/Backend.cc | 6 +- src/storage/backend/redis/Redis.cc | 465 ++++++++++-------- src/storage/backend/redis/Redis.h | 24 +- .../storage/redis-async-reading-pcap.zeek | 32 +- .../base/frameworks/storage/redis-async.zeek | 31 +- .../frameworks/storage/redis-cluster.zeek | 32 +- .../frameworks/storage/redis-expiration.zeek | 30 +- .../base/frameworks/storage/redis-sync.zeek | 18 +- .../storage/sqlite-basic-reading-pcap.zeek | 30 +- .../base/frameworks/storage/sqlite-basic.zeek | 48 +- 11 files changed, 410 insertions(+), 317 deletions(-) diff --git a/scripts/policy/frameworks/storage/backend/redis/main.zeek b/scripts/policy/frameworks/storage/backend/redis/main.zeek index 35e6a6f27f..f37fe0b3c8 100644 --- a/scripts/policy/frameworks/storage/backend/redis/main.zeek +++ b/scripts/policy/frameworks/storage/backend/redis/main.zeek @@ -22,17 +22,6 @@ export { # but preferably should be set to a unique value per Redis # backend opened. key_prefix: string &default=""; - - # Redis only supports sync and async separately. You cannot do - # both with the same connection. If this flag is true, the - # connection will be async and will only allow commands via - # ``when`` commands. You will still need to set the - # ``async_mode`` flags of the put, get, and erase methods to - # match this flag. This flag is overridden when reading pcaps - # and the backend will be forced into synchronous mode, since - # time won't move forward the same as when capturing live - # traffic. - async_mode: bool &default=T; }; redef record Storage::BackendOptions += { diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index 591d72eda7..cfedd8bd54 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -109,7 +109,7 @@ ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expira auto res = DoPut(std::move(key), std::move(value), overwrite, expiration_time, cb); - if ( (! native_async || zeek::run_state::reading_traces) && cb ) { + if ( ! native_async && cb ) { cb->Complete(res); delete cb; } @@ -125,7 +125,7 @@ ValResult Backend::Get(ValPtr key, ValResultCallback* cb) { auto res = DoGet(std::move(key), cb); - if ( (! native_async || zeek::run_state::reading_traces) && cb ) { + if ( ! native_async && cb ) { cb->Complete(res); delete cb; } @@ -141,7 +141,7 @@ ErrorResult Backend::Erase(ValPtr key, ErrorResultCallback* cb) { auto res = DoErase(std::move(key), cb); - if ( (! native_async || zeek::run_state::reading_traces) && cb ) { + if ( ! native_async && cb ) { cb->Complete(res); delete cb; } diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 5e081b821f..18e3b45a2e 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -8,6 +8,7 @@ #include "zeek/Val.h" #include "zeek/iosource/Manager.h" +#include "hiredis/adapters/poll.h" #include "hiredis/async.h" #include "hiredis/hiredis.h" @@ -16,8 +17,10 @@ namespace { class Tracer { public: - Tracer(const std::string& where) : where(where) { /*printf("%s\n", where.c_str());*/ } - ~Tracer() { /* printf("%s done\n", where.c_str()); */ } + Tracer(const std::string& where) : where(where) { // printf("%s\n", where.c_str()); + } + ~Tracer() { // printf("%s done\n", where.c_str()); + } std::string where; }; @@ -54,25 +57,62 @@ void redisErase(redisAsyncContext* ctx, void* reply, void* privdata) { backend->HandleEraseResult(static_cast(reply), callback); } +void redisZRANGEBYSCORE(redisAsyncContext* ctx, void* reply, void* privdata) { + auto t = Tracer("erase"); + auto backend = static_cast(ctx->data); + backend->HandleZRANGEBYSCORE(static_cast(reply)); +} + +void redisGeneric(redisAsyncContext* ctx, void* reply, void* privdata) { + auto t = Tracer("generic"); + auto backend = static_cast(ctx->data); + backend->HandleGeneric(); + freeReplyObject(reply); +} + +// Because we called redisPollAttach in DoOpen(), privdata here is a +// redisPollEvents object. We can go through that object to get the context's +// data, which contains the backend. Because we overrode these callbacks in +// DoOpen, we still want to mimic their callbacks to redisPollTick functions +// correctly. void redisAddRead(void* privdata) { auto t = Tracer("addread"); - auto backend = static_cast(privdata); - backend->OnAddRead(); + auto rpe = static_cast(privdata); + auto backend = static_cast(rpe->context->data); + + if ( rpe->reading == 0 ) + zeek::iosource_mgr->RegisterFd(rpe->fd, backend, zeek::iosource::IOSource::READ); + rpe->reading = 1; } + void redisDelRead(void* privdata) { auto t = Tracer("delread"); - auto backend = static_cast(privdata); - backend->OnDelRead(); + auto rpe = static_cast(privdata); + auto backend = static_cast(rpe->context->data); + + if ( rpe->reading == 1 ) + zeek::iosource_mgr->UnregisterFd(rpe->fd, backend, zeek::iosource::IOSource::READ); + rpe->reading = 0; } + void redisAddWrite(void* privdata) { auto t = Tracer("addwrite"); - auto backend = static_cast(privdata); - backend->OnAddWrite(); + auto rpe = static_cast(privdata); + auto backend = static_cast(rpe->context->data); + + if ( rpe->writing == 0 ) + zeek::iosource_mgr->RegisterFd(rpe->fd, backend, zeek::iosource::IOSource::WRITE); + rpe->writing = 1; } + void redisDelWrite(void* privdata) { + auto rpe = static_cast(privdata); auto t = Tracer("delwrite"); - auto backend = static_cast(privdata); - backend->OnDelWrite(); + auto backend = static_cast(rpe->context->data); + + if ( rpe->writing == 1 ) + zeek::iosource_mgr->UnregisterFd(rpe->fd, backend, zeek::iosource::IOSource::WRITE); + rpe->writing = 0; } } // namespace @@ -87,13 +127,8 @@ storage::BackendPtr Redis::Instantiate(std::string_view tag) { return make_intru ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { RecordValPtr backend_options = options->GetField("redis"); - // When reading traces we disable storage async mode globally (see src/storage/Backend.cc) since - // time moves forward based on the pcap and not based on real time. - async_mode = backend_options->GetField("async_mode")->Get() && ! zeek::run_state::reading_traces; key_prefix = backend_options->GetField("key_prefix")->ToStdString(); - DBG_LOG(DBG_STORAGE, "Redis backend: running in async mode? %d", async_mode); - redisOptions opt = {0}; StringValPtr host = backend_options->GetField("server_host"); @@ -118,57 +153,58 @@ ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { struct timeval timeout = {5, 0}; opt.connect_timeout = &timeout; - if ( async_mode ) { - async_ctx = redisAsyncConnectWithOptions(&opt); - if ( async_ctx == nullptr || async_ctx->err ) { - // This block doesn't necessarily mean the connection failed. It means - // that hiredis failed to set up the async context. Connection failure - // is returned later via the OnConnect callback. - std::string errmsg = util::fmt("Failed to open connection to Redis server at %s", server_addr.c_str()); - if ( async_ctx ) { - errmsg.append(": "); - errmsg.append(async_ctx->errstr); - } - - redisAsyncFree(async_ctx); - async_ctx = nullptr; - return errmsg; + async_ctx = redisAsyncConnectWithOptions(&opt); + if ( async_ctx == nullptr || async_ctx->err ) { + // This block doesn't necessarily mean the connection failed. It means + // that hiredis failed to set up the async context. Connection failure + // is returned later via the OnConnect callback. + std::string errmsg = util::fmt("Failed to open connection to Redis server at %s", server_addr.c_str()); + if ( async_ctx ) { + errmsg.append(": "); + errmsg.append(async_ctx->errstr); } - // TODO: Sort out how to pass the zeek callbacks for both open/done to the async - // callbacks from hiredis so they can return errors. - - // The context is passed to the handler methods. Setting this data object - // pointer allows us to look up the backend in the handlers. - async_ctx->data = this; - async_ctx->ev.data = this; - - redisAsyncSetConnectCallback(async_ctx, redisOnConnect); - redisAsyncSetDisconnectCallback(async_ctx, redisOnDisconnect); - - // These four callbacks handle the file descriptor coming and going for read - // and write operations for hiredis. Their subsequent callbacks will - // register/unregister with iosource_mgr as needed. I tried just registering - // full time for both read and write but it leads to weird syncing issues - // within the hiredis code. This is safer in regards to the library, even if - // it results in waking up our IO loop more frequently. - async_ctx->ev.addRead = redisAddRead; - async_ctx->ev.delRead = redisDelRead; - async_ctx->ev.addWrite = redisAddWrite; - async_ctx->ev.delWrite = redisDelWrite; + redisAsyncFree(async_ctx); + async_ctx = nullptr; + return errmsg; } - else { - ctx = redisConnectWithOptions(&opt); - if ( ctx == nullptr || ctx->err ) { - if ( ctx ) - return util::fmt("Failed to open connection to Redis server at %s", server_addr.c_str()); - else - return util::fmt("Failed to open connection to Redis server at %s: %s", server_addr.c_str(), - ctx->errstr); - } - connected = true; - } + ++active_ops; + + // TODO: Sort out how to pass the zeek callbacks for both open/done to the async + // callbacks from hiredis so they can return errors. + + // The context is passed to the handler methods. Setting this data object + // pointer allows us to look up the backend in the handlers. + async_ctx->data = this; + + redisPollAttach(async_ctx); + redisAsyncSetConnectCallback(async_ctx, redisOnConnect); + redisAsyncSetDisconnectCallback(async_ctx, redisOnDisconnect); + + // redisAsyncSetConnectCallback sets the flag in the redisPollEvent for writing + // so we can add this to our loop as well. + zeek::iosource_mgr->RegisterFd(async_ctx->c.fd, this, zeek::iosource::IOSource::WRITE); + + // These four callbacks handle the file descriptor coming and going for read + // and write operations for hiredis. Their subsequent callbacks will + // register/unregister with iosource_mgr as needed. I tried just registering + // full time for both read and write but it leads to weird syncing issues + // within the hiredis code. This is safer in regards to the library, even if + // it results in waking up our IO loop more frequently. + // + // redisPollAttach sets these to functions internal to the poll attachment, + // but we override them for our own uses. See the callbacks for more info + // about why. + async_ctx->ev.addRead = redisAddRead; + async_ctx->ev.delRead = redisDelRead; + async_ctx->ev.addWrite = redisAddWrite; + async_ctx->ev.delWrite = redisDelWrite; + + if ( ! cb ) + // Polling here will eventually call OnConnect, which will set the flag + // that we're connected. + Poll(); return std::nullopt; } @@ -179,20 +215,17 @@ ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { ErrorResult Redis::DoClose(ErrorResultCallback* cb) { connected = false; - if ( async_mode ) { - // This will probably result in an error since hiredis should have - // already removed the file descriptor via the delRead and delWrite - // callbacks, but do it anyways just to be sure. - iosource_mgr->UnregisterFd(async_ctx->c.fd, this, IOSource::READ | IOSource::WRITE); - redisAsyncDisconnect(async_ctx); - redisAsyncFree(async_ctx); - async_ctx = nullptr; - } - else { - redisFree(ctx); - ctx = nullptr; + redisAsyncDisconnect(async_ctx); + ++active_ops; + + if ( ! cb && ! zeek::run_state::terminating ) { + Poll(); + // TODO: handle response } + redisAsyncFree(async_ctx); + async_ctx = nullptr; + return std::nullopt; } @@ -208,39 +241,43 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira if ( ! overwrite ) format.append(" NX"); - // Use built-in expiration if reading live data, since time will move - // forward consistently. If reading pcaps, we'll do something else. - if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) - format.append(" PXAT %d"); - auto json_key = key->ToJSON()->ToStdString(); auto json_value = value->ToJSON()->ToStdString(); - if ( async_mode ) { - int status; - if ( expiration_time > 0.0 ) - status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), - json_value.data(), static_cast(expiration_time * 1e6)); - else - status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), - json_value.data()); - - if ( connected && status == REDIS_ERR ) - return util::fmt("Failed to queue async put operation: %s", async_ctx->errstr); + int status; + // Use built-in expiration if reading live data, since time will move + // forward consistently. If reading pcaps, we'll do something else. + if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) { + format.append(" PXAT %d"); + status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), + json_value.data(), static_cast(expiration_time * 1e6)); } - else { - redisReply* reply; - if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) - reply = (redisReply*)redisCommand(ctx, format.c_str(), key_prefix.data(), json_key.data(), - json_value.data(), static_cast(expiration_time * 1e6)); - else - reply = - (redisReply*)redisCommand(ctx, format.c_str(), key_prefix.data(), json_key.data(), json_value.data()); + else + status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), + json_value.data()); - if ( ! reply ) - return util::fmt("Put operation failed: %s", ctx->errstr); + if ( connected && status == REDIS_ERR ) + return util::fmt("Failed to queue put operation: %s", async_ctx->errstr); + + ++active_ops; + + if ( ! cb ) { + Poll(); + + redisReply* reply = reply_queue.front(); + reply_queue.pop_front(); + + ErrorResult res; + if ( ! connected ) + res = util::fmt("Connection is not open"); + else if ( ! reply ) + res = util::fmt("Async put operation returned null reply"); + else if ( reply && reply->type == REDIS_REPLY_ERROR ) + res = util::fmt("Async put operation failed: %s", reply->str); freeReplyObject(reply); + if ( res.has_value() ) + return res; } // If reading pcaps insert into a secondary set that's ordered by expiration @@ -251,12 +288,18 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira format.append(" NX"); format += " %f %s"; - redisReply* reply = - (redisReply*)redisCommand(ctx, format.c_str(), key_prefix.data(), expiration_time, json_key.data()); - if ( ! reply ) - return util::fmt("ZADD operation failed: %s", ctx->errstr); + status = redisAsyncCommand(async_ctx, redisGeneric, NULL, format.c_str(), key_prefix.data(), expiration_time, + json_key.data()); + if ( connected && status == REDIS_ERR ) + return util::fmt("ZADD operation failed: %s", async_ctx->errstr); - freeReplyObject(reply); + ++active_ops; + } + + if ( ! cb ) { + // We don't care about the result from the ZADD, just that we wait + // for it to finish. + Poll(); } return std::nullopt; @@ -270,26 +313,27 @@ ValResult Redis::DoGet(ValPtr key, ValResultCallback* cb) { if ( ! connected && ! async_ctx ) return zeek::unexpected("Connection is not open"); - if ( async_mode ) { - int status = redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%s", key_prefix.data(), - key->ToJSON()->ToStdStringView().data()); + int status = redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%s", key_prefix.data(), + key->ToJSON()->ToStdStringView().data()); - if ( connected && status == REDIS_ERR ) - return zeek::unexpected( - util::fmt("Failed to queue async get operation: %s", async_ctx->errstr)); + if ( connected && status == REDIS_ERR ) + return zeek::unexpected(util::fmt("Failed to queue get operation: %s", async_ctx->errstr)); - // There isn't a result to return here. That happens in HandleGetResult. - return zeek::unexpected(""); + ++active_ops; + + if ( ! cb ) { + Poll(); + redisReply* reply = reply_queue.front(); + reply_queue.pop_front(); + + auto res = ParseGetReply(reply); + freeReplyObject(reply); + return res; } - else { - auto reply = - (redisReply*)redisCommand(ctx, "GET %s:%s", key_prefix.data(), key->ToJSON()->ToStdStringView().data()); - if ( ! reply ) - return zeek::unexpected(util::fmt("Get operation failed: %s", ctx->errstr)); - - return ParseGetReply(reply); - } + // There isn't a result to return here. That happens in HandleGetResult for + // async operations. + return zeek::unexpected(""); } /** @@ -300,20 +344,18 @@ ErrorResult Redis::DoErase(ValPtr key, ErrorResultCallback* cb) { if ( ! connected && ! async_ctx ) return "Connection is not open"; - if ( async_mode ) { - int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%s", key_prefix.data(), - key->ToJSON()->ToStdStringView().data()); + int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%s", key_prefix.data(), + key->ToJSON()->ToStdStringView().data()); - if ( connected && status == REDIS_ERR ) - return util::fmt("Failed to queue async erase operation failed: %s", async_ctx->errstr); - } - else { - redisReply* reply = - (redisReply*)redisCommand(ctx, "DEL %s:%s", key_prefix.data(), key->ToJSON()->ToStdStringView().data()); + if ( connected && status == REDIS_ERR ) + return util::fmt("Failed to queue erase operation failed: %s", async_ctx->errstr); - if ( ! reply ) - return util::fmt("Put operation failed: %s", ctx->errstr); + ++active_ops; + if ( ! cb ) { + Poll(); + redisReply* reply = reply_queue.front(); + reply_queue.pop_front(); freeReplyObject(reply); } @@ -321,19 +363,28 @@ ErrorResult Redis::DoErase(ValPtr key, ErrorResultCallback* cb) { } void Redis::Expire() { + // Expiration is handled natively by Redis if not reading traces. if ( ! connected || ! zeek::run_state::reading_traces ) return; - redisReply* reply = - (redisReply*)redisCommand(ctx, "ZRANGEBYSCORE %s_expire -inf %f", key_prefix.data(), run_state::network_time); + int status = redisAsyncCommand(async_ctx, redisZRANGEBYSCORE, NULL, "ZRANGEBYSCORE %s_expire -inf %f", + key_prefix.data(), run_state::network_time); - if ( ! reply ) { + if ( status == REDIS_ERR ) { // TODO: do something with the error? - printf("ZRANGEBYSCORE command failed: %s\n", ctx->errstr); + printf("ZRANGEBYSCORE command failed: %s\n", async_ctx->errstr); return; } - if ( reply->elements == 0 ) { + ++active_ops; + + // Expire always happens in a synchronous fashion. Block here until we've received + // a response. + Poll(); + redisReply* reply = reply_queue.front(); + reply_queue.pop_front(); + + if ( reply && reply->elements == 0 ) { freeReplyObject(reply); return; } @@ -343,69 +394,92 @@ void Redis::Expire() { // and passing the array as a block somehow. There's no guarantee it'd be faster // anyways. for ( size_t i = 0; i < reply->elements; i++ ) { - auto del_reply = (redisReply*)redisCommand(ctx, "DEL %s:%s", key_prefix.data(), reply->element[i]->str); - - // Don't bother checking the response here. The only error that would matter is if the key - // didn't exist, but that would mean it was already removed for some other reason. - - freeReplyObject(del_reply); + status = + redisAsyncCommand(async_ctx, redisGeneric, NULL, "DEL %s:%s", key_prefix.data(), reply->element[i]->str); + ++active_ops; + Poll(); } - freeReplyObject(reply); - reply = (redisReply*)redisCommand(ctx, "ZREMRANGEBYSCORE %s_expire -inf %f", key_prefix.data(), - run_state::network_time); + // Remove all of the elements from the range-set that match the time range. + redisAsyncCommand(async_ctx, redisGeneric, NULL, "ZREMRANGEBYSCORE %s_expire -inf %f", key_prefix.data(), + run_state::network_time); - if ( ! reply ) { - // TODO: do something with the error? - printf("ZREMRANGEBYSCORE command failed: %s\n", ctx->errstr); - return; - } + ++active_ops; + Poll(); - freeReplyObject(reply); + // This can't be freed until the other commands finish because the memory for + // the strings doesn't get copied when making the DEL commands. + // freeReplyObject(reply); } void Redis::HandlePutResult(redisReply* reply, ErrorResultCallback* callback) { - ErrorResult res; - if ( ! connected ) - res = util::fmt("Connection is not open"); - else if ( ! reply ) - res = util::fmt("Async put operation returned null reply"); - else if ( reply && reply->type == REDIS_REPLY_ERROR ) - res = util::fmt("Async put operation failed: %s", reply->str); + --active_ops; - freeReplyObject(reply); - callback->Complete(res); - delete callback; + if ( callback ) { + ErrorResult res; + if ( ! connected ) + res = util::fmt("Connection is not open"); + else if ( ! reply ) + res = util::fmt("Async put operation returned null reply"); + else if ( reply && reply->type == REDIS_REPLY_ERROR ) + res = util::fmt("Async put operation failed: %s", reply->str); + + freeReplyObject(reply); + callback->Complete(res); + delete callback; + } + else + reply_queue.push_back(reply); } void Redis::HandleGetResult(redisReply* reply, ValResultCallback* callback) { - ValResult res; - if ( ! connected ) - res = zeek::unexpected("Connection is not open"); - else - res = ParseGetReply(reply); + --active_ops; - callback->Complete(res); - delete callback; + if ( callback ) { + ValResult res; + if ( ! connected ) + res = zeek::unexpected("Connection is not open"); + else + res = ParseGetReply(reply); + + callback->Complete(res); + freeReplyObject(reply); + delete callback; + } + else { + reply_queue.push_back(reply); + } } void Redis::HandleEraseResult(redisReply* reply, ErrorResultCallback* callback) { - ErrorResult res; - if ( ! connected ) - res = "Connection is not open"; - else if ( ! reply ) - res = util::fmt("Async erase operation returned null reply"); - else if ( reply && reply->type == REDIS_REPLY_ERROR ) - res = util::fmt("Async erase operation failed: %s", reply->str); + --active_ops; - freeReplyObject(reply); + if ( callback ) { + ErrorResult res; + if ( ! connected ) + res = "Connection is not open"; + else if ( ! reply ) + res = util::fmt("Async erase operation returned null reply"); + else if ( reply && reply->type == REDIS_REPLY_ERROR ) + res = util::fmt("Async erase operation failed: %s", reply->str); - callback->Complete(res); - delete callback; + freeReplyObject(reply); + callback->Complete(res); + delete callback; + } + else + reply_queue.push_back(reply); +} + +void Redis::HandleZRANGEBYSCORE(redisReply* reply) { + --active_ops; + reply_queue.push_back(reply); } void Redis::OnConnect(int status) { DBG_LOG(DBG_STORAGE, "Redis backend: connection event"); + --active_ops; + if ( status == REDIS_OK ) { connected = true; return; @@ -416,6 +490,8 @@ void Redis::OnConnect(int status) { void Redis::OnDisconnect(int status) { DBG_LOG(DBG_STORAGE, "Redis backend: disconnection event"); + --active_ops; + if ( status == REDIS_OK ) { // TODO: this was an intentional disconnect, nothing to do? } @@ -426,31 +502,6 @@ void Redis::OnDisconnect(int status) { connected = false; } -void Redis::OnAddRead() { - if ( ! async_ctx ) - return; - - iosource_mgr->RegisterFd(async_ctx->c.fd, this, IOSource::READ); -} -void Redis::OnDelRead() { - if ( ! async_ctx ) - return; - - iosource_mgr->UnregisterFd(async_ctx->c.fd, this, IOSource::READ); -} -void Redis::OnAddWrite() { - if ( ! async_ctx ) - return; - - iosource_mgr->RegisterFd(async_ctx->c.fd, this, IOSource::WRITE); -} -void Redis::OnDelWrite() { - if ( ! async_ctx ) - return; - - iosource_mgr->UnregisterFd(async_ctx->c.fd, this, IOSource::WRITE); -} - void Redis::ProcessFd(int fd, int flags) { if ( (flags & IOSource::ProcessFlags::READ) != 0 ) redisAsyncHandleRead(async_ctx); @@ -473,8 +524,12 @@ ValResult Redis::ParseGetReply(redisReply* reply) const { res = zeek::unexpected(std::get(val)); } - freeReplyObject(reply); return res; } +void Redis::Poll() { + while ( active_ops > 0 ) + int status = redisPollTick(async_ctx, 0.5); +} + } // namespace zeek::storage::backend::redis diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index ba32df6a88..613aca13b2 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -6,12 +6,11 @@ #include "zeek/storage/Backend.h" // Forward declare some types from hiredis to avoid including the header -struct redisContext; struct redisAsyncContext; struct redisReply; +struct redisPollEvents; namespace zeek::storage::backend::redis { - class Redis : public Backend, public iosource::IOSource { public: Redis(std::string_view tag) : Backend(true, tag), IOSource(true) {} @@ -73,25 +72,30 @@ public: void OnConnect(int status); void OnDisconnect(int status); - void OnAddRead(); - void OnDelRead(); - void OnAddWrite(); - void OnDelWrite(); - void HandlePutResult(redisReply* reply, ErrorResultCallback* callback); void HandleGetResult(redisReply* reply, ValResultCallback* callback); void HandleEraseResult(redisReply* reply, ErrorResultCallback* callback); + void HandleZRANGEBYSCORE(redisReply* reply); + + // HandleGeneric exists so that async-running-as-sync operations can remove + // themselves from the list of active operations. + void HandleGeneric() { --active_ops; } private: ValResult ParseGetReply(redisReply* reply) const; + void Poll(); - redisContext* ctx = nullptr; redisAsyncContext* async_ctx = nullptr; - bool connected = true; + + // When running in sync mode, this is used to keep a queue of replies as + // responses come in from the remote calls until we run out of data to + // poll. + std::deque reply_queue; std::string server_addr; std::string key_prefix; - bool async_mode = false; + std::atomic connected = false; + int active_ops = 0; }; } // namespace zeek::storage::backend::redis diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek index dec60fce83..1e4d17ccbf 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -1,5 +1,6 @@ # @TEST-DOC: Tests that Redis storage backend defaults back to sync mode reading pcaps +# @TEST-KNOWN-FAILURE: Currently broken due to the redis async rework # @TEST-REQUIRES: have-redis # @TEST-PORT: REDIS_PORT @@ -19,30 +20,37 @@ # Create a typename here that can be passed down into open_backend() type str: string; -event zeek_init() { - local opts : Storage::BackendOptions; - opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = T]; +event zeek_init() + { + local opts: Storage::BackendOptions; + opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( + "REDIS_PORT")), $key_prefix="testing" ]; local key = "key1234"; local value = "value5678"; local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); - when [b, key, value] ( local res = Storage::Async::put(b, [$key=key, $value=value]) ) { + when [b, key, value] ( local res = Storage::Async::put(b, [ $key=key, + $value=value ]) ) + { print "put result", res; - when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) { + when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) + { print "get result", res2; if ( res2?$val ) - print "get result same as inserted", value == (res2$val as string); + print "get result same as inserted", value == ( res2$val as string ); Storage::Sync::close_backend(b); + } + timeout 5sec + { + print "get request timed out"; + } } - timeout 5 sec { - print "get requeest timed out"; - } - } - timeout 5 sec { + timeout 5sec + { print "put request timed out"; + } } -} diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek index 37947d349d..225da69719 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek @@ -21,34 +21,41 @@ redef exit_only_after_terminate = T; # Create a typename here that can be passed down into open_backend() type str: string; -event zeek_init() { - local opts : Storage::BackendOptions; - opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = T]; +event zeek_init() + { + local opts: Storage::BackendOptions; + opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( + "REDIS_PORT")), $key_prefix="testing" ]; local key = "key1234"; local value = "value5678"; local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); - when [b, key, value] ( local res = Storage::Async::put(b, [$key=key, $value=value]) ) { + when [b, key, value] ( local res = Storage::Async::put(b, [ $key=key, + $value=value ]) ) + { print "put result", res; - when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) { + when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) + { print "get result", res2; if ( res2?$val ) - print "get result same as inserted", value == (res2$val as string); + print "get result same as inserted", value == ( res2$val as string ); Storage::Sync::close_backend(b); terminate(); - } - timeout 5 sec { - print "get requeest timed out"; + } + timeout 5sec + { + print "get request timed out"; terminate(); + } } - } - timeout 5 sec { + timeout 5sec + { print "put request timed out"; terminate(); + } } -} diff --git a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek index 3336c31101..8d6ede1d90 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek @@ -39,46 +39,52 @@ global redis_data_written: event() &is_used; global backend: opaque of Storage::BackendHandle; type str: string; -event zeek_init() { - local opts : Storage::BackendOptions; - opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F]; +event zeek_init() + { + local opts: Storage::BackendOptions; + opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( + "REDIS_PORT")), $key_prefix="testing" ]; backend = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); -} + } -event redis_data_written() { +event redis_data_written() + { print "redis_data_written"; local res = Storage::Sync::get(backend, "1234"); print Cluster::node, res; Storage::Sync::close_backend(backend); terminate(); -} + } @else global node_count: count = 0; -event Cluster::node_down(name: string, id: string) { +event Cluster::node_down(name: string, id: string) + { ++node_count; if ( node_count == 2 ) terminate(); -} + } -event redis_data_written() { +event redis_data_written() + { local e = Cluster::make_event(redis_data_written); Cluster::publish(Cluster::worker_topic, e); -} + } @endif @if ( Cluster::node == "worker-1" ) -event Cluster::Experimental::cluster_started() { - local res = Storage::Sync::put(backend, [$key="1234", $value="5678"]); +event Cluster::Experimental::cluster_started() + { + local res = Storage::Sync::put(backend, [ $key="1234", $value="5678" ]); print Cluster::node, "put result", res; local e = Cluster::make_event(redis_data_written); Cluster::publish(Cluster::manager_topic, e); -} + } @endif diff --git a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek index 6c9b1398e8..fcc6d03439 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek @@ -15,7 +15,7 @@ @load base/frameworks/storage/sync @load policy/frameworks/storage/backend/redis -redef Storage::expire_interval = 2 secs; +redef Storage::expire_interval = 2secs; redef exit_only_after_terminate = T; # Create a typename here that can be passed down into open_backend() @@ -25,34 +25,38 @@ global b: opaque of Storage::BackendHandle; global key: string = "key1234"; global value: string = "value7890"; -event check_removed() { +event check_removed() + { local res2 = Storage::Sync::get(b, key); print "get result after expiration", res2; Storage::Sync::close_backend(b); terminate(); -} + } -event setup_test() { - local opts : Storage::BackendOptions; - opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F]; +event setup_test() + { + local opts: Storage::BackendOptions; + opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( + "REDIS_PORT")), $key_prefix="testing" ]; b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); - local res = Storage::Sync::put(b, [$key=key, $value=value, $expire_time=2 secs]); + local res = Storage::Sync::put(b, [ $key=key, $value=value, $expire_time=2secs ]); print "put result", res; local res2 = Storage::Sync::get(b, key); print "get result", res2; if ( res2?$val ) - print "get result same as inserted", value == (res2$val as string); + print "get result same as inserted", value == ( res2$val as string ); - schedule 5 secs { check_removed() }; -} + schedule 5secs { check_removed() }; + } -event zeek_init() { +event zeek_init() + { # We need network time to be set to something other than zero for the # expiration time to be set correctly. Schedule an event on a short # timer so packets start getting read and do the setup there. - schedule 100 msecs { setup_test() }; -} + schedule 100msecs { setup_test() }; + } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek index cd65443290..bf49a30230 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -18,31 +18,33 @@ # Create a typename here that can be passed down into open_backend() type str: string; -event zeek_init() { - local opts : Storage::BackendOptions; - opts$redis = [$server_host = "127.0.0.1", $server_port = to_port(getenv("REDIS_PORT")), $key_prefix = "testing", $async_mode = F]; +event zeek_init() + { + local opts: Storage::BackendOptions; + opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( + "REDIS_PORT")), $key_prefix="testing" ]; local key = "key1234"; local value = "value1234"; local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); - local res = Storage::Sync::put(b, [$key=key, $value=value]); + local res = Storage::Sync::put(b, [ $key=key, $value=value ]); print "put result", res; local res2 = Storage::Sync::get(b, key); print "get result", res2; if ( res2?$val ) - print "get result same as inserted", value == (res2$val as string); + print "get result same as inserted", value == ( res2$val as string ); local value2 = "value5678"; - res = Storage::Sync::put(b, [$key=key, $value=value2, $overwrite=T]); + res = Storage::Sync::put(b, [ $key=key, $value=value2, $overwrite=T ]); print "overwrite put result", res; res2 = Storage::Sync::get(b, key); print "get result", res2; if ( res2?$val ) - print "get result same as inserted", value2 == (res2$val as string); + print "get result same as inserted", value2 == ( res2$val as string ); Storage::Sync::close_backend(b); -} + } diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek index 76989fa1c5..8a933de3c6 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek @@ -12,10 +12,11 @@ redef exit_only_after_terminate = T; # Create a typename here that can be passed down into get(). type str: string; -event zeek_init() { +event zeek_init() + { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::BackendOptions; - opts$sqlite = [$database_path = "test.sqlite", $table_name = "testing"]; + local opts: Storage::BackendOptions; + opts$sqlite = [ $database_path="test.sqlite", $table_name="testing" ]; local key = "key1234"; local value = "value5678"; @@ -24,25 +25,30 @@ event zeek_init() { # the backend yet. local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); - when [b, key, value] ( local res = Storage::Async::put(b, [$key=key, $value=value]) ) { + when [b, key, value] ( local res = Storage::Async::put(b, [ $key=key, + $value=value ]) ) + { print "put result", res; - when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) { + when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) + { print "get result", res2; if ( res2?$val ) - print "get result same as inserted", value == (res2$val as string); + print "get result same as inserted", value == ( res2$val as string ); Storage::Sync::close_backend(b); terminate(); - } - timeout 5 sec { - print "get requeest timed out"; + } + timeout 5sec + { + print "get request timed out"; terminate(); + } } - } - timeout 5 sec { + timeout 5sec + { print "put request timed out"; terminate(); + } } -} diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index 63d262e55b..c815996f7c 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -11,47 +11,59 @@ redef exit_only_after_terminate = T; # Create a typename here that can be passed down into get(). type str: string; -event zeek_init() { +event zeek_init() + { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::BackendOptions; - opts$sqlite = [$database_path = "test.sqlite", $table_name="testing"]; + local opts: Storage::BackendOptions; + opts$sqlite = [ $database_path="test.sqlite", $table_name="testing" ]; local key = "key1234"; local value = "value5678"; # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. - when [opts, key, value] ( local b = Storage::Async::open_backend(Storage::SQLITE, opts, str, str) ) { + when [opts, key, value] ( local b = Storage::Async::open_backend( + Storage::SQLITE, opts, str, str) ) + { print "open successful"; - when [b, key, value] ( local put_res = Storage::Async::put(b, [$key=key, $value=value]) ) { + when [b, key, value] ( local put_res = Storage::Async::put(b, [ $key=key, + $value=value ]) ) + { print "put result", put_res; - when [b, key, value] ( local get_res = Storage::Async::get(b, key) ) { + when [b, key, value] ( local get_res = Storage::Async::get(b, key) ) + { print "get result", get_res; if ( get_res?$val ) - print "get result same as inserted", value == (get_res$val as string); + print "get result same as inserted", value == ( get_res$val as string ); - when [b] ( local close_res = Storage::Async::close_backend(b) ) { + when [b] ( local close_res = Storage::Async::close_backend(b) ) + { print "closed succesfully"; terminate(); - } timeout 5 sec { + } + timeout 5sec + { print "close request timed out"; terminate(); + } + } + timeout 5sec + { + print "get request timed out"; + terminate(); } } - timeout 5 sec { - print "get requeest timed out"; - terminate(); - } - } - timeout 5 sec { + timeout 5sec + { print "put request timed out"; terminate(); + } } - } - timeout 5 sec { + timeout 5sec + { print "open request timed out"; terminate(); + } } -} From e766af7322caec2d8803e7cd70dc3b2185415a46 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 19 Feb 2025 14:58:59 -0700 Subject: [PATCH 30/52] Split sync/async handling into the BIF methods --- src/storage/Backend.cc | 113 ++++++------ src/storage/Backend.h | 44 ++++- src/storage/backend/redis/Redis.cc | 109 +++--------- src/storage/backend/redis/Redis.h | 6 +- src/storage/backend/sqlite/SQLite.h | 2 +- src/storage/storage.bif | 164 +++++++++++++++--- .../plugins/storage-plugin/src/StorageDummy.h | 2 +- .../storage/redis-async-reading-pcap.zeek | 1 - 8 files changed, 260 insertions(+), 181 deletions(-) diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index cfedd8bd54..12bd1eb60b 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -3,10 +3,8 @@ #include "zeek/storage/Backend.h" #include "zeek/Desc.h" -#include "zeek/RunState.h" #include "zeek/Trigger.h" #include "zeek/broker/Data.h" -#include "zeek/storage/Manager.h" namespace zeek::storage { @@ -16,84 +14,84 @@ ResultCallback::ResultCallback(zeek::detail::trigger::TriggerPtr trigger, const ResultCallback::~ResultCallback() {} void ResultCallback::Timeout() { - auto v = make_intrusive("Timeout during request"); - trigger->Cache(assoc, v.get()); + if ( ! IsSyncCallback() ) { + auto v = make_intrusive("Timeout during request"); + trigger->Cache(assoc, v.get()); + } } void ResultCallback::ValComplete(Val* result) { - trigger->Cache(assoc, result); + if ( ! IsSyncCallback() ) { + trigger->Cache(assoc, result); + trigger->Release(); + } + Unref(result); - trigger->Release(); } ErrorResultCallback::ErrorResultCallback(IntrusivePtr trigger, const void* assoc) : ResultCallback(std::move(trigger), assoc) {} void ErrorResultCallback::Complete(const ErrorResult& res) { - zeek::Val* result; + if ( IsSyncCallback() ) + result = res; + + zeek::Val* val_result; if ( res ) - result = new StringVal(res.value()); + val_result = new StringVal(res.value()); else - result = val_mgr->Bool(true).get(); + val_result = val_mgr->Bool(true).get(); - ValComplete(result); + ValComplete(val_result); } ValResultCallback::ValResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc) : ResultCallback(std::move(trigger), assoc) {} void ValResultCallback::Complete(const ValResult& res) { + if ( IsSyncCallback() ) + result = res; + static auto val_result_type = zeek::id::find_type("val_result"); - auto* result = new zeek::RecordVal(val_result_type); + auto* val_result = new zeek::RecordVal(val_result_type); if ( res ) - result->Assign(0, res.value()); + val_result->Assign(0, res.value()); else - result->Assign(1, zeek::make_intrusive(res.error())); + val_result->Assign(1, zeek::make_intrusive(res.error())); - ValComplete(result); + ValComplete(val_result); } +OpenResultCallback::OpenResultCallback(detail::BackendHandleVal* backend) : ResultCallback(), backend(backend) {} + OpenResultCallback::OpenResultCallback(IntrusivePtr trigger, const void* assoc, detail::BackendHandleVal* backend) : ResultCallback(std::move(trigger), assoc), backend(backend) {} void OpenResultCallback::Complete(const ErrorResult& res) { - zeek::Val* result; + if ( IsSyncCallback() ) + result = res; + + zeek::Val* val_result; if ( res ) - result = new StringVal(res.value()); + val_result = new StringVal(res.value()); else - result = backend; + val_result = backend; - ValComplete(result); + ValComplete(val_result); } ErrorResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb) { key_type = std::move(kt); val_type = std::move(vt); - auto res = DoOpen(std::move(options)); - - if ( (! native_async || zeek::run_state::reading_traces) && cb ) { - cb->Complete(res); - delete cb; - } - - return res; + return DoOpen(std::move(options)); } -ErrorResult Backend::Close(ErrorResultCallback* cb) { - auto res = DoClose(cb); - - if ( (! native_async || zeek::run_state::reading_traces) && cb ) { - cb->Complete(res); - delete cb; - } - - return res; -} +ErrorResult Backend::Close(ErrorResultCallback* cb) { return DoClose(cb); } ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expiration_time, ErrorResultCallback* cb) { // The intention for this method is to do some other heavy lifting in regard @@ -107,14 +105,7 @@ ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expira return util::fmt("type of value passed (%s) does not match backend's value type (%s)", obj_desc_short(value->GetType().get()).c_str(), val_type->GetName().c_str()); - auto res = DoPut(std::move(key), std::move(value), overwrite, expiration_time, cb); - - if ( ! native_async && cb ) { - cb->Complete(res); - delete cb; - } - - return res; + return DoPut(std::move(key), std::move(value), overwrite, expiration_time, cb); } ValResult Backend::Get(ValPtr key, ValResultCallback* cb) { @@ -123,14 +114,7 @@ ValResult Backend::Get(ValPtr key, ValResultCallback* cb) { return zeek::unexpected(util::fmt("type of key passed (%s) does not match backend's key type (%s)", key->GetType()->GetName().c_str(), key_type->GetName().c_str())); - auto res = DoGet(std::move(key), cb); - - if ( ! native_async && cb ) { - cb->Complete(res); - delete cb; - } - - return res; + return DoGet(std::move(key), cb); } ErrorResult Backend::Erase(ValPtr key, ErrorResultCallback* cb) { @@ -139,16 +123,31 @@ ErrorResult Backend::Erase(ValPtr key, ErrorResultCallback* cb) { return util::fmt("type of key passed (%s) does not match backend's key type (%s)", key->GetType()->GetName().c_str(), key_type->GetName().c_str()); - auto res = DoErase(std::move(key), cb); + return DoErase(std::move(key), cb); +} - if ( ! native_async && cb ) { - cb->Complete(res); +void Backend::CompleteCallback(ValResultCallback* cb, const ValResult& data) const { + cb->Complete(data); + if ( ! cb->IsSyncCallback() ) { delete cb; } - - return res; } +void Backend::CompleteCallback(ErrorResultCallback* cb, const ErrorResult& data) const { + cb->Complete(data); + if ( ! cb->IsSyncCallback() ) { + delete cb; + } +} + +void Backend::CompleteCallback(OpenResultCallback* cb, const ErrorResult& data) const { + cb->Complete(data); + if ( ! cb->IsSyncCallback() ) { + delete cb; + } +} + + zeek::OpaqueTypePtr detail::backend_opaque; IMPLEMENT_OPAQUE_VALUE(detail::BackendHandleVal) diff --git a/src/storage/Backend.h b/src/storage/Backend.h index 071ce1f507..ab89fb619f 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -29,34 +29,48 @@ using ValResult = zeek::expected; // code reuse in the other callback methods. class ResultCallback { public: + ResultCallback() = default; ResultCallback(IntrusivePtr trigger, const void* assoc); virtual ~ResultCallback(); void Timeout(); + bool IsSyncCallback() const { return ! trigger; } protected: void ValComplete(Val* result); private: IntrusivePtr trigger; - const void* assoc; + const void* assoc = nullptr; }; // A callback result that returns an ErrorResult. class ErrorResultCallback : public ResultCallback { public: + ErrorResultCallback() = default; ErrorResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc); virtual void Complete(const ErrorResult& res); + ErrorResult Result() { return result; } + +private: + ErrorResult result; }; // A callback result that returns a ValResult. class ValResultCallback : public ResultCallback { public: + ValResultCallback() = default; ValResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc); void Complete(const ValResult& res); + ValResult Result() { return result; } + +private: + ValResult result; }; class OpenResultCallback; +enum SupportedModes : uint8_t { SYNC = 0x01, ASYNC = 0x02 }; + class Backend : public zeek::Obj { public: /** @@ -105,6 +119,15 @@ public: */ virtual bool IsOpen() = 0; + bool SupportsSync() const { return (modes & SupportedModes::SYNC) == SupportedModes::SYNC; } + bool SupportsAsync() const { return (modes & SupportedModes::ASYNC) == SupportedModes::ASYNC; } + + /** + * Optional method to allow a backend to poll for data. This can be used to + * mimic sync mode even if the backend only supports async. + */ + virtual void Poll() {} + protected: // Allow the manager to call Open/Close. friend class storage::Manager; @@ -112,14 +135,12 @@ protected: /** * Constructor * - * @param native_async Denotes whether this backend can handle async request - * natively. If set to false, the Put/Get/Erase methods will call the - * callback after their corresponding Do methods return. If set to true, the - * backend needs to call the callback itself. + * @param modes A combination of values from SupportedModes. These modes + # define whether a backend only supports sync or async or both. * @param tag A string representation of the tag for this backend. This * is passed from the Manager through the component factory. */ - Backend(bool native_async, std::string_view tag) : tag(tag), native_async(native_async) {} + Backend(uint8_t modes, std::string_view tag) : tag(tag), modes(modes) {} /** * Called by the manager system to open the backend. @@ -176,13 +197,17 @@ protected: */ virtual void Expire() {} + void CompleteCallback(ValResultCallback* cb, const ValResult& data) const; + void CompleteCallback(ErrorResultCallback* cb, const ErrorResult& data) const; + void CompleteCallback(OpenResultCallback* cb, const ErrorResult& data) const; + TypePtr key_type; TypePtr val_type; std::string tag; private: - bool native_async = false; + uint8_t modes; }; using BackendPtr = zeek::IntrusivePtr; @@ -210,12 +235,15 @@ protected: // A callback for the Backend::Open() method that returns an error or a backend handle. class OpenResultCallback : public ResultCallback { public: + OpenResultCallback(detail::BackendHandleVal* backend); OpenResultCallback(IntrusivePtr trigger, const void* assoc, detail::BackendHandleVal* backend); void Complete(const ErrorResult& res); + ErrorResult Result() { return result; } private: - detail::BackendHandleVal* backend; + ErrorResult result; + detail::BackendHandleVal* backend = nullptr; }; } // namespace zeek::storage diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 18e3b45a2e..d23f62a4e4 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -17,10 +17,8 @@ namespace { class Tracer { public: - Tracer(const std::string& where) : where(where) { // printf("%s\n", where.c_str()); - } - ~Tracer() { // printf("%s done\n", where.c_str()); - } + Tracer(const std::string& where) : where(where) {} // DBG_LOG(zeek::DBG_STORAGE, "%s", where.c_str()); } + ~Tracer() {} // DBG_LOG(zeek::DBG_STORAGE, "%s done", where.c_str()); } std::string where; }; @@ -58,7 +56,7 @@ void redisErase(redisAsyncContext* ctx, void* reply, void* privdata) { } void redisZRANGEBYSCORE(redisAsyncContext* ctx, void* reply, void* privdata) { - auto t = Tracer("erase"); + auto t = Tracer("zrangebyscore"); auto backend = static_cast(ctx->data); backend->HandleZRANGEBYSCORE(static_cast(reply)); } @@ -201,11 +199,6 @@ ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { async_ctx->ev.addWrite = redisAddWrite; async_ctx->ev.delWrite = redisDelWrite; - if ( ! cb ) - // Polling here will eventually call OnConnect, which will set the flag - // that we're connected. - Poll(); - return std::nullopt; } @@ -218,7 +211,7 @@ ErrorResult Redis::DoClose(ErrorResultCallback* cb) { redisAsyncDisconnect(async_ctx); ++active_ops; - if ( ! cb && ! zeek::run_state::terminating ) { + if ( cb->IsSyncCallback() && ! zeek::run_state::terminating ) { Poll(); // TODO: handle response } @@ -261,25 +254,6 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira ++active_ops; - if ( ! cb ) { - Poll(); - - redisReply* reply = reply_queue.front(); - reply_queue.pop_front(); - - ErrorResult res; - if ( ! connected ) - res = util::fmt("Connection is not open"); - else if ( ! reply ) - res = util::fmt("Async put operation returned null reply"); - else if ( reply && reply->type == REDIS_REPLY_ERROR ) - res = util::fmt("Async put operation failed: %s", reply->str); - - freeReplyObject(reply); - if ( res.has_value() ) - return res; - } - // If reading pcaps insert into a secondary set that's ordered by expiration // time that gets checked by Expire(). if ( expiration_time > 0.0 && zeek::run_state::reading_traces ) { @@ -296,12 +270,6 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira ++active_ops; } - if ( ! cb ) { - // We don't care about the result from the ZADD, just that we wait - // for it to finish. - Poll(); - } - return std::nullopt; } @@ -321,16 +289,6 @@ ValResult Redis::DoGet(ValPtr key, ValResultCallback* cb) { ++active_ops; - if ( ! cb ) { - Poll(); - redisReply* reply = reply_queue.front(); - reply_queue.pop_front(); - - auto res = ParseGetReply(reply); - freeReplyObject(reply); - return res; - } - // There isn't a result to return here. That happens in HandleGetResult for // async operations. return zeek::unexpected(""); @@ -352,13 +310,6 @@ ErrorResult Redis::DoErase(ValPtr key, ErrorResultCallback* cb) { ++active_ops; - if ( ! cb ) { - Poll(); - redisReply* reply = reply_queue.front(); - reply_queue.pop_front(); - freeReplyObject(reply); - } - return std::nullopt; } @@ -415,46 +366,37 @@ void Redis::Expire() { void Redis::HandlePutResult(redisReply* reply, ErrorResultCallback* callback) { --active_ops; - if ( callback ) { - ErrorResult res; - if ( ! connected ) - res = util::fmt("Connection is not open"); - else if ( ! reply ) - res = util::fmt("Async put operation returned null reply"); - else if ( reply && reply->type == REDIS_REPLY_ERROR ) - res = util::fmt("Async put operation failed: %s", reply->str); + ErrorResult res; + if ( ! connected ) + res = util::fmt("Connection is not open"); + else if ( ! reply ) + res = util::fmt("Async put operation returned null reply"); + else if ( reply && reply->type == REDIS_REPLY_ERROR ) + res = util::fmt("Async put operation failed: %s", reply->str); - freeReplyObject(reply); - callback->Complete(res); - delete callback; - } - else - reply_queue.push_back(reply); + freeReplyObject(reply); + CompleteCallback(callback, res); } void Redis::HandleGetResult(redisReply* reply, ValResultCallback* callback) { --active_ops; - if ( callback ) { - ValResult res; - if ( ! connected ) - res = zeek::unexpected("Connection is not open"); - else - res = ParseGetReply(reply); + ValResult res; + if ( ! connected ) + res = zeek::unexpected("Connection is not open"); + else + res = ParseGetReply(reply); - callback->Complete(res); - freeReplyObject(reply); - delete callback; - } - else { - reply_queue.push_back(reply); - } + freeReplyObject(reply); + CompleteCallback(callback, res); } void Redis::HandleEraseResult(redisReply* reply, ErrorResultCallback* callback) { --active_ops; - if ( callback ) { + if ( callback->IsSyncCallback() ) + reply_queue.push_back(reply); + else { ErrorResult res; if ( ! connected ) res = "Connection is not open"; @@ -464,11 +406,8 @@ void Redis::HandleEraseResult(redisReply* reply, ErrorResultCallback* callback) res = util::fmt("Async erase operation failed: %s", reply->str); freeReplyObject(reply); - callback->Complete(res); - delete callback; + CompleteCallback(callback, res); } - else - reply_queue.push_back(reply); } void Redis::HandleZRANGEBYSCORE(redisReply* reply) { diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index 613aca13b2..3a7dd21dcc 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -13,7 +13,7 @@ struct redisPollEvents; namespace zeek::storage::backend::redis { class Redis : public Backend, public iosource::IOSource { public: - Redis(std::string_view tag) : Backend(true, tag), IOSource(true) {} + Redis(std::string_view tag) : Backend(SupportedModes::ASYNC, tag), IOSource(true) {} ~Redis() override = default; static BackendPtr Instantiate(std::string_view tag); @@ -81,9 +81,11 @@ public: // themselves from the list of active operations. void HandleGeneric() { --active_ops; } +protected: + void Poll() override; + private: ValResult ParseGetReply(redisReply* reply) const; - void Poll(); redisAsyncContext* async_ctx = nullptr; diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index f3eff1536a..ad0869bad2 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -12,7 +12,7 @@ namespace zeek::storage::backend::sqlite { class SQLite : public Backend { public: - SQLite(std::string_view tag) : Backend(false, tag) {} + SQLite(std::string_view tag) : Backend(SupportedModes::SYNC, tag) {} ~SQLite() override = default; static BackendPtr Instantiate(std::string_view tag); diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 6c68e5f638..a29c5fd801 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -8,12 +8,12 @@ using namespace zeek; using namespace zeek::storage; static zeek::detail::trigger::TriggerPtr init_trigger(zeek::detail::Frame* frame) { - auto trigger = frame->GetTrigger(); + auto trigger = frame->GetTrigger(); - if ( ! trigger ) { - emit_builtin_error("Asynchronous storage operations must be called via a when-condition"); - return nullptr; - } + if ( ! trigger ) { + emit_builtin_error("Asynchronous storage operations must be called via a when-condition"); + return nullptr; + } if ( auto timeout = trigger->TimeoutValue(); timeout < 0 ) { emit_builtin_error("Async Storage operations must specify a timeout block"); @@ -56,7 +56,7 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, %{ auto trigger = init_trigger(frame); if ( ! trigger ) - return val_mgr->Bool(false); + return val_mgr->Bool(false); auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; Tag tag{btype_val}; @@ -74,7 +74,19 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; - storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); + auto open_res = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); + + if ( ! b.value()->SupportsAsync() ) { + // If the backend doesn't support async, we blocked in order to get here already. Handle the + // callback manually. + cb->Complete(open_res); + delete cb; + } + else if ( run_state::reading_traces ) { + // If the backend is truly async and we're reading traces, we need to fake being in sync mode + // because otherwise time doesn't move forward correctly. + b.value()->Poll(); + } return nullptr; %} @@ -94,7 +106,19 @@ function Storage::Async::__close_backend%(backend: opaque of Storage::BackendHan return val_mgr->Bool(true); auto cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); - storage_mgr->CloseBackend(b->backend, cb); + auto close_res = storage_mgr->CloseBackend(b->backend, cb); + + if ( ! b->backend->SupportsAsync() ) { + // If the backend doesn't support async, we blocked in order to get here already. Handle the + // callback manually. + cb->Complete(close_res); + delete cb; + } + else if ( run_state::reading_traces ) { + // If the backend is truly async and we're reading traces, we need to fake being in sync mode + // because otherwise time doesn't move forward correctly. + b->backend->Poll(); + } return nullptr; %} @@ -120,7 +144,19 @@ function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: auto cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; - b->backend->Put(key_v, val_v, overwrite, expire_time, cb); + auto put_res = b->backend->Put(key_v, val_v, overwrite, expire_time, cb); + + if ( ! b->backend->SupportsAsync() ) { + // If the backend doesn't support async, we blocked in order to get here already. Handle the + // callback manually. + cb->Complete(put_res); + delete cb; + } + else if ( run_state::reading_traces ) { + // If the backend is truly async and we're reading traces, we need to fake being in sync mode + // because otherwise time doesn't move forward correctly. + b->backend->Poll(); + } return nullptr; %} @@ -148,7 +184,19 @@ function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: auto cb = new ValResultCallback(trigger, frame->GetTriggerAssoc()); auto key_v = IntrusivePtr{NewRef{}, key}; - auto result = b->backend->Get(key_v, cb); + auto get_res = b->backend->Get(key_v, cb); + + if ( ! b->backend->SupportsAsync() ) { + // If the backend doesn't support async, we blocked in order to get here already. Handle the + // callback manually. + cb->Complete(get_res); + delete cb; + } + else if ( run_state::reading_traces ) { + // If the backend is truly async and we're reading traces, we need to fake being in sync mode + // because otherwise time doesn't move forward correctly. + b->backend->Poll(); + } return nullptr; %} @@ -169,7 +217,19 @@ function Storage::Async::__erase%(backend: opaque of Storage::BackendHandle, key auto cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); auto key_v = IntrusivePtr{NewRef{}, key}; - b->backend->Erase(key_v, cb); + auto erase_res = b->backend->Erase(key_v, cb); + + if ( ! b->backend->SupportsAsync() ) { + // If the backend doesn't support async, we blocked in order to get here already. Handle the + // callback manually. + cb->Complete(erase_res); + delete cb; + } + else if ( run_state::reading_traces ) { + // If the backend is truly async and we're reading traces, we need to fake being in sync mode + // because otherwise time doesn't move forward correctly. + b->backend->Poll(); + } return nullptr; %} @@ -188,17 +248,29 @@ function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, k return val_mgr->Bool(false); } + auto bh = make_intrusive(b.value()); + + auto cb = new OpenResultCallback(bh.get()); auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; - auto open_res = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, nullptr); + auto open_res = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); + + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! b.value()->SupportsSync() ) { + b.value()->Poll(); + open_res = cb->Result(); + } + + delete cb; if ( open_res.has_value() ) { emit_builtin_error(open_res.value().c_str()); return val_mgr->Bool(false); } - return make_intrusive(b.value()); + return bh; %} function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHandle%) : bool @@ -212,10 +284,20 @@ function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHand // Return true here since the backend is already closed return val_mgr->Bool(true); - auto result = storage_mgr->CloseBackend(b->backend, nullptr); + auto cb = new ErrorResultCallback(); + auto close_res = storage_mgr->CloseBackend(b->backend, cb); - if ( result.has_value() ) { - emit_builtin_error(result.value().c_str()); + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! b->backend->SupportsSync() ) { + b->backend->Poll(); + close_res = cb->Result(); + } + + delete cb; + + if ( close_res.has_value() ) { + emit_builtin_error(close_res.value().c_str()); return val_mgr->Bool(false); } @@ -233,12 +315,22 @@ function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: a else if ( ! b->backend->IsOpen() ) return val_mgr->Bool(false); + auto cb = new ErrorResultCallback(); auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; - auto result = b->backend->Put(key_v, val_v, overwrite, expire_time, nullptr); + auto put_res = b->backend->Put(key_v, val_v, overwrite, expire_time, cb); - if ( result.has_value() ) { - emit_builtin_error(util::fmt("Failed to store data: %s", result.value().c_str())); + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! b->backend->SupportsSync() ) { + b->backend->Poll(); + put_res = cb->Result(); + } + + delete cb; + + if ( put_res.has_value() ) { + emit_builtin_error(util::fmt("Failed to store data: %s", put_res.value().c_str())); return val_mgr->Bool(false); } @@ -261,15 +353,25 @@ function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: a } auto key_v = IntrusivePtr{NewRef{}, key}; - auto result = b->backend->Get(key_v, nullptr); + auto cb = new ValResultCallback(); + auto get_res = b->backend->Get(key_v, cb); - if ( ! result.has_value() ) { + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! b->backend->SupportsSync() ) { + b->backend->Poll(); + get_res = cb->Result(); + } + + delete cb; + + if ( ! get_res.has_value() ) { val_result->Assign(1, make_intrusive( - util::fmt("Failed to retrieve data: %s", result.error().c_str()))); + util::fmt("Failed to retrieve data: %s", get_res.error().c_str()))); return val_result; } - val_result->Assign(0, result.value()); + val_result->Assign(0, get_res.value()); return val_result; %} @@ -283,11 +385,21 @@ function Storage::Sync::__erase%(backend: opaque of Storage::BackendHandle, key: else if ( ! b->backend->IsOpen() ) return val_mgr->Bool(false); + auto cb = new ErrorResultCallback(); auto key_v = IntrusivePtr{NewRef{}, key}; - auto result = b->backend->Erase(key_v, nullptr); + auto erase_res = b->backend->Erase(key_v, cb); - if ( result.has_value() ) { - emit_builtin_error(util::fmt("Failed to erase data for key: %s", result.value().c_str())); + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! b->backend->SupportsSync() ) { + b->backend->Poll(); + erase_res = cb->Result(); + } + + delete cb; + + if ( erase_res.has_value() ) { + emit_builtin_error(util::fmt("Failed to erase data for key: %s", erase_res.value().c_str())); return val_mgr->Bool(false); } diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h index e45acf3335..4c8a2dfc29 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.h +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -13,7 +13,7 @@ namespace btest::storage::backend { */ class StorageDummy : public zeek::storage::Backend { public: - StorageDummy(std::string_view tag) : Backend(false, tag) {} + StorageDummy(std::string_view tag) : Backend(zeek::storage::SupportedModes::SYNC, tag) {} ~StorageDummy() override = default; static zeek::storage::BackendPtr Instantiate(std::string_view tag); diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek index 1e4d17ccbf..2fe19ddb37 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -1,6 +1,5 @@ # @TEST-DOC: Tests that Redis storage backend defaults back to sync mode reading pcaps -# @TEST-KNOWN-FAILURE: Currently broken due to the redis async rework # @TEST-REQUIRES: have-redis # @TEST-PORT: REDIS_PORT From 8ddda016ff315ada4e52eed1f123b8d6340d3439 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 20 Feb 2025 12:43:35 -0700 Subject: [PATCH 31/52] Update some btests due to timing changes --- .../core.tunnels.teredo-known-services/known_services.log | 2 +- testing/btest/Baseline/language.expire_func-copy/output | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log b/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log index d7565fafdf..665be284a4 100644 --- a/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log +++ b/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log @@ -8,6 +8,6 @@ #fields ts host port_num port_proto service #types time addr port enum set[string] XXXXXXXXXX.XXXXXX 192.168.2.1 53 udp DNS -XXXXXXXXXX.XXXXXX 192.168.2.16 1577 tcp (empty) XXXXXXXXXX.XXXXXX 192.168.2.16 1576 tcp (empty) +XXXXXXXXXX.XXXXXX 192.168.2.16 1577 tcp (empty) #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/language.expire_func-copy/output b/testing/btest/Baseline/language.expire_func-copy/output index 17b843d3f7..26d3fcb7b5 100644 --- a/testing/btest/Baseline/language.expire_func-copy/output +++ b/testing/btest/Baseline/language.expire_func-copy/output @@ -18,10 +18,10 @@ @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp, proto=17] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49658/tcp, resp_h=172.16.238.131, resp_p=80/tcp, proto=6] @XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=37975/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] -@XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49659/tcp, resp_h=172.16.238.131, resp_p=21/tcp, proto=6] -@XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=45126/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] @XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=45126/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] @XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.1, orig_p=49659/tcp, resp_h=172.16.238.131, resp_p=21/tcp, proto=6] +@XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.1, orig_p=49659/tcp, resp_h=172.16.238.131, resp_p=21/tcp, proto=6] +@XXXXXXXXXX.XXXXXX expired copy [orig_h=172.16.238.131, orig_p=45126/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] @XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=55368/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] @XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=33818/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] @XXXXXXXXXX.XXXXXX expired [orig_h=172.16.238.131, orig_p=50205/udp, resp_h=172.16.238.2, resp_p=53/udp, proto=17] From 9ed3e33f97748bfc773b56393de7a81fb2be3b68 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 24 Feb 2025 14:37:11 -0700 Subject: [PATCH 32/52] Completely rework return values from storage operations --- scripts/base/frameworks/storage/async.zeek | 50 ++-- scripts/base/frameworks/storage/main.zeek | 6 +- scripts/base/frameworks/storage/sync.zeek | 50 ++-- scripts/base/init-bare.zeek | 53 ++++- .../storage/backend/redis/main.zeek | 8 +- .../storage/backend/sqlite/main.zeek | 8 +- src/IntrusivePtr.h | 22 +- src/storage/Backend.cc | 185 ++++++++------- src/storage/Backend.h | 114 ++++----- src/storage/CMakeLists.txt | 1 + src/storage/Manager.cc | 33 ++- src/storage/Manager.h | 22 +- src/storage/ReturnCode.cc | 77 ++++++ src/storage/ReturnCode.h | 37 +++ src/storage/backend/redis/Redis.cc | 113 +++++---- src/storage/backend/redis/Redis.h | 22 +- src/storage/backend/sqlite/SQLite.cc | 88 +++---- src/storage/backend/sqlite/SQLite.h | 14 +- src/storage/storage.bif | 223 ++++++++++-------- testing/btest/Baseline/plugins.storage/output | 5 +- .../Baseline/plugins.storage/zeek-stderr | 3 - .../out | 7 +- .../scripts.base.frameworks.storage.erase/out | 5 +- .../out | 7 +- .../out | 5 +- .../out | 5 +- .../out | 6 +- .../worker-1..stdout | 4 +- .../worker-2..stdout | 2 +- .../out | 7 +- .../out | 9 +- .../out | 5 +- .../out | 6 +- .../.stderr | 2 - .../out | 5 +- .../storage-plugin/src/StorageDummy.cc | 48 ++-- .../plugins/storage-plugin/src/StorageDummy.h | 18 +- testing/btest/plugins/storage.zeek | 20 +- .../frameworks/storage/compound-types.zeek | 4 +- .../base/frameworks/storage/erase.zeek | 17 +- .../base/frameworks/storage/expiration.zeek | 12 +- .../base/frameworks/storage/overwriting.zeek | 8 +- .../storage/redis-async-reading-pcap.zeek | 8 +- .../base/frameworks/storage/redis-async.zeek | 43 +++- .../frameworks/storage/redis-cluster.zeek | 3 +- .../frameworks/storage/redis-expiration.zeek | 9 +- .../base/frameworks/storage/redis-sync.zeek | 13 +- .../storage/sqlite-basic-reading-pcap.zeek | 9 +- .../base/frameworks/storage/sqlite-basic.zeek | 9 +- .../storage/sqlite-error-handling.zeek | 15 +- 50 files changed, 859 insertions(+), 586 deletions(-) create mode 100644 src/storage/ReturnCode.cc create mode 100644 src/storage/ReturnCode.h diff --git a/scripts/base/frameworks/storage/async.zeek b/scripts/base/frameworks/storage/async.zeek index 3b2ba08dcd..8fbadb6e1b 100644 --- a/scripts/base/frameworks/storage/async.zeek +++ b/scripts/base/frameworks/storage/async.zeek @@ -22,17 +22,19 @@ export { ## well as for type conversions for return values from ## :zeek:see:`Storage::Async::get`. ## - ## Returns: A handle to the new backend connection, or ``F`` if the connection - ## failed. + ## Returns: A record containing the status of the operation, and either an error + ## string on failure or a value on success. The value returned here will + ## be an ``opaque of BackendHandle``. global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, - val_type: any): opaque of Storage::BackendHandle; + val_type: any): Storage::OperationResult; ## Closes an existing backend connection asynchronously. ## ## backend: A handle to a backend connection. ## - ## Returns: A boolean indicating success or failure of the operation. - global close_backend: function(backend: opaque of Storage::BackendHandle): bool; + ## Returns: A record containing the status of the operation and an optional error + ## string for failures. + global close_backend: function(backend: opaque of Storage::BackendHandle): Storage::OperationResult; ## Inserts a new entry into a backend asynchronously. ## @@ -41,11 +43,9 @@ export { ## args: A :zeek:see:`Storage::PutArgs` record containing the arguments for the ## operation. ## - ## Returns: A boolean indicating success or failure of the operation. Type - ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to - ## be returned. - global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool; + ## Returns: A record containing the status of the operation and an optional error + ## string for failures. + global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): Storage::OperationResult; ## Gets an entry from the backend asynchronously. ## @@ -53,13 +53,11 @@ export { ## ## key: The key to look up. ## - ## Returns: A boolean indicating success or failure of the operation. Type - ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to - ## be returned. The caller should check the validity of the value before - ## attempting to use it. If the value is unset, an error string may be - ## available to describe the failure. - global get: function(backend: opaque of Storage::BackendHandle, key: any): val_result; + ## Returns: A record containing the status of the operation, an optional error + ## string for failures, and an optional value for success. The value + ## returned here will be of the type passed into + ## :zeek:see:`Storage::open_backend`. + global get: function(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult; ## Erases an entry from the backend asynchronously. ## @@ -67,35 +65,33 @@ export { ## ## key: The key to erase. ## - ## Returns: A boolean indicating success or failure of the operation. Type - ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to - ## be returned. - global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; + ## Returns: A record containing the status of the operation and an optional error + ## string for failures. + global erase: function(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult; } function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, - val_type: any): opaque of Storage::BackendHandle + val_type: any): Storage::OperationResult { return Storage::Async::__open_backend(btype, options, key_type, val_type); } -function close_backend(backend: opaque of Storage::BackendHandle): bool +function close_backend(backend: opaque of Storage::BackendHandle): Storage::OperationResult { return Storage::Async::__close_backend(backend); } -function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool +function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): Storage::OperationResult { return Storage::Async::__put(backend, args$key, args$value, args$overwrite, args$expire_time); } -function get(backend: opaque of Storage::BackendHandle, key: any): val_result +function get(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult { return Storage::Async::__get(backend, key); } -function erase(backend: opaque of Storage::BackendHandle, key: any): bool +function erase(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult { return Storage::Async::__erase(backend, key); } diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index 66ab1c2744..ee928cbc28 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -5,9 +5,9 @@ module Storage; export { - ## Base record for backend options. Backend plugins can redef this record to add - ## relevant fields to it. - type BackendOptions: record {}; + ## Base record for backend options. Backend plugins can redef this record to add + ## relevant fields to it. + type BackendOptions: record { }; ## Record for passing arguments to :zeek:see:`Storage::Async::put` and ## :zeek:see:`Storage::Sync::put`. diff --git a/scripts/base/frameworks/storage/sync.zeek b/scripts/base/frameworks/storage/sync.zeek index 90aaa6a302..7eb5fa5a1d 100644 --- a/scripts/base/frameworks/storage/sync.zeek +++ b/scripts/base/frameworks/storage/sync.zeek @@ -20,17 +20,19 @@ export { ## as for type conversions for return values from ## :zeek:see:`Storage::Sync::get`. ## - ## Returns: A handle to the new backend connection, or ``F`` if the connection - ## failed. + ## Returns: A record containing the status of the operation, and either an error + ## string on failure or a value on success. The value returned here will + ## be an ``opaque of BackendHandle``. global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, - val_type: any): opaque of Storage::BackendHandle; + val_type: any): Storage::OperationResult; ## Closes an existing backend connection. ## ## backend: A handle to a backend connection. ## - ## Returns: A boolean indicating success or failure of the operation. - global close_backend: function(backend: opaque of Storage::BackendHandle): bool; + ## Returns: A record containing the status of the operation and an optional error + ## string for failures. + global close_backend: function(backend: opaque of Storage::BackendHandle): Storage::OperationResult; ## Inserts a new entry into a backend. ## @@ -39,11 +41,9 @@ export { ## args: A :zeek:see:`Storage::PutArgs` record containing the arguments for the ## operation. ## - ## Returns: A boolean indicating success or failure of the operation. Type - ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to - ## be returned. - global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool; + ## Returns: A record containing the status of the operation and an optional error + ## string for failures. + global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): Storage::OperationResult; ## Gets an entry from the backend. ## @@ -51,13 +51,11 @@ export { ## ## key: The key to look up. ## - ## Returns: A boolean indicating success or failure of the operation. Type - ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to - ## be returned. The caller should check the validity of the value before - ## attempting to use it. If the value is unset, an error string may be - ## available to describe the failure. - global get: function(backend: opaque of Storage::BackendHandle, key: any): val_result; + ## Returns: A record containing the status of the operation, an optional error + ## string for failures, and an optional value for success. The value + ## returned here will be of the type passed into + ## :zeek:see:`Storage::open_backend`. + global get: function(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult; ## Erases an entry from the backend. ## @@ -65,35 +63,33 @@ export { ## ## key: The key to erase. ## - ## Returns: A boolean indicating success or failure of the operation. Type - ## comparison failures against the types passed to - ## :zeek:see:`Storage::open_backend` for the backend will cause ``F`` to - ## be returned. - global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; + ## Returns: A record containing the status of the operation and an optional error + ## string for failures. + global erase: function(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult; } function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, - val_type: any): opaque of Storage::BackendHandle + val_type: any): Storage::OperationResult { return Storage::Sync::__open_backend(btype, options, key_type, val_type); } -function close_backend(backend: opaque of Storage::BackendHandle): bool +function close_backend(backend: opaque of Storage::BackendHandle): Storage::OperationResult { return Storage::Sync::__close_backend(backend); } -function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): bool +function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): Storage::OperationResult { return Storage::Sync::__put(backend, args$key, args$value, args$overwrite, args$expire_time); } -function get(backend: opaque of Storage::BackendHandle, key: any): val_result +function get(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult { return Storage::Sync::__get(backend, key); } -function erase(backend: opaque of Storage::BackendHandle, key: any): bool +function erase(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult { return Storage::Sync::__erase(backend, key); } diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 625415586f..a71963f1fd 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -356,14 +356,6 @@ type ftp_port: record { valid: bool; ##< True if format was right. Only then are *h* and *p* valid. }; -## A generic type for returning either a value or an error string from a -## function or a BIF method. This is sort of equivalent to std::expected -## in C++. -type val_result: record { - val: any &optional; - error: string &optional; -}; - ## Statistics about what a TCP endpoint sent. ## ## .. zeek:see:: conn_stats @@ -6224,7 +6216,50 @@ export { ## The interval used by the storage framework for automatic expiration ## of elements in all backends that don't support it natively, or if ## using expiration while reading pcap files. - const expire_interval = 15.0 secs &redef; + const expire_interval = 15.0secs &redef; + + ## Common set of statuses that can be returned by storage operations. Backend plugins + ## can add to this enum if custom values are needed. + type ReturnCode: enum { + ## Operation succeeded. + SUCCESS, + ## Type of value passed to operation does not match type of + ## value passed when opening backend. + VAL_TYPE_MISMATCH, + ## Type of key passed to operation does not match type of + ## key passed when opening backend. + KEY_TYPE_MISMATCH, + ## Backend is not connected. + NOT_CONNECTED, + ## Operation timed out. + TIMEOUT, + ## Connection to backed was lost. + CONNECTION_LOST, + ## Generic operation failed. + OPERATION_FAILED, + ## Key requested was not found in backend. + KEY_NOT_FOUND, + ## Key requested for overwrite already exists. + KEY_EXISTS, + ## Generic connection failure. + CONNECTION_FAILED, + ## Generic disconnection failure. + DISCONNECTION_FAILED, + ## Generic initialization failure. + INITIALIZATION_FAILED + } &redef; + + ## Returned as the result of the various storage operations. + type OperationResult: record { + ## One of a set of backend-redefinable return codes. + code: ReturnCode; + ## An optional error string. This should be set when the + ## ``code`` field is not set ``SUCCESS``. + error_str: string &optional; + ## An optional value returned by ``get`` operations when a match + ## was found the key requested. + value: any &optional; + }; } module GLOBAL; diff --git a/scripts/policy/frameworks/storage/backend/redis/main.zeek b/scripts/policy/frameworks/storage/backend/redis/main.zeek index f37fe0b3c8..a921fefc3a 100644 --- a/scripts/policy/frameworks/storage/backend/redis/main.zeek +++ b/scripts/policy/frameworks/storage/backend/redis/main.zeek @@ -23,8 +23,8 @@ export { # backend opened. key_prefix: string &default=""; }; - - redef record Storage::BackendOptions += { - redis: Options &optional; - }; } + +redef record Storage::BackendOptions += { + redis: Storage::Backend::Redis::Options &optional; +}; diff --git a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek index 9b4fd386f8..24e15fd857 100644 --- a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek +++ b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek @@ -28,8 +28,8 @@ export { ["temp_store"] = "memory" ); }; - - redef record Storage::BackendOptions += { - sqlite: Options &optional; - }; } + +redef record Storage::BackendOptions += { + sqlite: Storage::Backend::SQLite::Options &optional; +}; diff --git a/src/IntrusivePtr.h b/src/IntrusivePtr.h index b384b66e9a..6d44edb9df 100644 --- a/src/IntrusivePtr.h +++ b/src/IntrusivePtr.h @@ -144,10 +144,7 @@ public: } IntrusivePtr& operator=(std::nullptr_t) noexcept { - if ( ptr_ ) { - Unref(ptr_); - ptr_ = nullptr; - } + reset(); return *this; } @@ -161,6 +158,23 @@ public: explicit operator bool() const noexcept { return ptr_ != nullptr; } + void reset() noexcept { + if ( ptr_ ) { + Unref(ptr_); + ptr_ = nullptr; + } + } + + void reset(T* ptr) { + if ( ptr_ ) + Unref(ptr_); + + if ( ptr ) + Ref(ptr); + + ptr_ = ptr; + } + private: pointer ptr_ = nullptr; }; diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index 12bd1eb60b..b1ceb92a54 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -2,152 +2,157 @@ #include "zeek/storage/Backend.h" -#include "zeek/Desc.h" #include "zeek/Trigger.h" #include "zeek/broker/Data.h" +#include "zeek/storage/ReturnCode.h" namespace zeek::storage { +RecordValPtr OperationResult::BuildVal() { + static auto op_result_type = zeek::id::find_type("Storage::OperationResult"); + + auto rec = zeek::make_intrusive(op_result_type); + rec->Assign(0, code); + if ( ! err_str.empty() ) + rec->Assign(1, err_str); + if ( value ) + rec->Assign(2, value); + + return rec; +} + ResultCallback::ResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc) : trigger(std::move(trigger)), assoc(assoc) {} -ResultCallback::~ResultCallback() {} - void ResultCallback::Timeout() { + static const auto& op_result_type = zeek::id::find_type("Storage::OperationResult"); + if ( ! IsSyncCallback() ) { - auto v = make_intrusive("Timeout during request"); - trigger->Cache(assoc, v.get()); + auto op_result = make_intrusive(op_result_type); + op_result->Assign(0, ReturnCode::TIMEOUT); + + trigger->Cache(assoc, op_result.release()); } } -void ResultCallback::ValComplete(Val* result) { - if ( ! IsSyncCallback() ) { - trigger->Cache(assoc, result); - trigger->Release(); +OperationResultCallback::OperationResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc) + : ResultCallback(std::move(trigger), assoc) {} + +void OperationResultCallback::Complete(OperationResult res) { + // If this is a sync callback, there isn't a trigger to process. Store the result and bail. + if ( IsSyncCallback() ) { + result = std::move(res); + return; } - Unref(result); -} + static auto op_result_type = zeek::id::find_type("Storage::OperationResult"); + auto* op_result = new zeek::RecordVal(op_result_type); -ErrorResultCallback::ErrorResultCallback(IntrusivePtr trigger, const void* assoc) - : ResultCallback(std::move(trigger), assoc) {} - -void ErrorResultCallback::Complete(const ErrorResult& res) { - if ( IsSyncCallback() ) - result = res; - - zeek::Val* val_result; - - if ( res ) - val_result = new StringVal(res.value()); + op_result->Assign(0, res.code); + if ( res.code->Get() != 0 ) + op_result->Assign(1, res.err_str); else - val_result = val_mgr->Bool(true).get(); + op_result->Assign(2, res.value); - ValComplete(val_result); + trigger->Cache(assoc, op_result); + trigger->Release(); + + Unref(op_result); } -ValResultCallback::ValResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc) - : ResultCallback(std::move(trigger), assoc) {} +OpenResultCallback::OpenResultCallback(IntrusivePtr backend) + : ResultCallback(), backend(std::move(backend)) {} -void ValResultCallback::Complete(const ValResult& res) { - if ( IsSyncCallback() ) - result = res; +OpenResultCallback::OpenResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc, + IntrusivePtr backend) + : ResultCallback(std::move(trigger), assoc), backend(std::move(backend)) {} - static auto val_result_type = zeek::id::find_type("val_result"); - auto* val_result = new zeek::RecordVal(val_result_type); +void OpenResultCallback::Complete(OperationResult res) { + // If this is a sync callback, there isn't a trigger to process. Store the result and bail. Always + // set result's value to the backend pointer so that it comes across in the result. This ensures + // the handle is always available in the result even on failures. + if ( IsSyncCallback() ) { + result = std::move(res); + result.value = backend; + return; + } - if ( res ) - val_result->Assign(0, res.value()); - else - val_result->Assign(1, zeek::make_intrusive(res.error())); + static auto op_result_type = zeek::id::find_type("Storage::OperationResult"); + auto* op_result = new zeek::RecordVal(op_result_type); - ValComplete(val_result); + op_result->Assign(0, res.code); + if ( res.code != ReturnCode::SUCCESS ) + op_result->Assign(1, res.err_str); + op_result->Assign(2, backend); + + trigger->Cache(assoc, op_result); + trigger->Release(); + + Unref(op_result); } -OpenResultCallback::OpenResultCallback(detail::BackendHandleVal* backend) : ResultCallback(), backend(backend) {} - -OpenResultCallback::OpenResultCallback(IntrusivePtr trigger, const void* assoc, - detail::BackendHandleVal* backend) - : ResultCallback(std::move(trigger), assoc), backend(backend) {} - -void OpenResultCallback::Complete(const ErrorResult& res) { - if ( IsSyncCallback() ) - result = res; - - zeek::Val* val_result; - - if ( res ) - val_result = new StringVal(res.value()); - else - val_result = backend; - - ValComplete(val_result); -} - -ErrorResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb) { +OperationResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb) { key_type = std::move(kt); val_type = std::move(vt); - return DoOpen(std::move(options)); + auto ret = DoOpen(std::move(options), cb); + if ( ! ret.value ) + ret.value = cb->Backend(); + + return ret; } -ErrorResult Backend::Close(ErrorResultCallback* cb) { return DoClose(cb); } +OperationResult Backend::Close(OperationResultCallback* cb) { return DoClose(cb); } -ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expiration_time, ErrorResultCallback* cb) { +OperationResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expiration_time, + OperationResultCallback* cb) { // The intention for this method is to do some other heavy lifting in regard // to backends that need to pass data through the manager instead of directly // through the workers. For the first versions of the storage framework it // just calls the backend itself directly. - if ( ! same_type(key->GetType(), key_type) ) - return util::fmt("type of key passed (%s) does not match backend's key type (%s)", - obj_desc_short(key->GetType().get()).c_str(), key_type->GetName().c_str()); - if ( ! same_type(value->GetType(), val_type) ) - return util::fmt("type of value passed (%s) does not match backend's value type (%s)", - obj_desc_short(value->GetType().get()).c_str(), val_type->GetName().c_str()); + if ( ! same_type(key->GetType(), key_type) ) { + auto ret = OperationResult{ReturnCode::KEY_TYPE_MISMATCH}; + CompleteCallback(cb, ret); + return ret; + } + if ( ! same_type(value->GetType(), val_type) ) { + auto ret = OperationResult{ReturnCode::VAL_TYPE_MISMATCH}; + CompleteCallback(cb, ret); + return ret; + } return DoPut(std::move(key), std::move(value), overwrite, expiration_time, cb); } -ValResult Backend::Get(ValPtr key, ValResultCallback* cb) { +OperationResult Backend::Get(ValPtr key, OperationResultCallback* cb) { // See the note in Put(). - if ( ! same_type(key->GetType(), key_type) ) - return zeek::unexpected(util::fmt("type of key passed (%s) does not match backend's key type (%s)", - key->GetType()->GetName().c_str(), key_type->GetName().c_str())); + if ( ! same_type(key->GetType(), key_type) ) { + auto ret = OperationResult{ReturnCode::KEY_TYPE_MISMATCH}; + CompleteCallback(cb, ret); + return ret; + } return DoGet(std::move(key), cb); } -ErrorResult Backend::Erase(ValPtr key, ErrorResultCallback* cb) { +OperationResult Backend::Erase(ValPtr key, OperationResultCallback* cb) { // See the note in Put(). - if ( ! same_type(key->GetType(), key_type) ) - return util::fmt("type of key passed (%s) does not match backend's key type (%s)", - key->GetType()->GetName().c_str(), key_type->GetName().c_str()); + if ( ! same_type(key->GetType(), key_type) ) { + auto ret = OperationResult{ReturnCode::KEY_TYPE_MISMATCH}; + CompleteCallback(cb, ret); + return ret; + } return DoErase(std::move(key), cb); } -void Backend::CompleteCallback(ValResultCallback* cb, const ValResult& data) const { +void Backend::CompleteCallback(ResultCallback* cb, const OperationResult& data) const { cb->Complete(data); if ( ! cb->IsSyncCallback() ) { delete cb; } } -void Backend::CompleteCallback(ErrorResultCallback* cb, const ErrorResult& data) const { - cb->Complete(data); - if ( ! cb->IsSyncCallback() ) { - delete cb; - } -} - -void Backend::CompleteCallback(OpenResultCallback* cb, const ErrorResult& data) const { - cb->Complete(data); - if ( ! cb->IsSyncCallback() ) { - delete cb; - } -} - - zeek::OpaqueTypePtr detail::backend_opaque; IMPLEMENT_OPAQUE_VALUE(detail::BackendHandleVal) diff --git a/src/storage/Backend.h b/src/storage/Backend.h index ab89fb619f..139a1ab96c 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -4,7 +4,6 @@ #include "zeek/OpaqueVal.h" #include "zeek/Val.h" -#include "zeek/util.h" namespace zeek::detail::trigger { class Trigger; @@ -15,14 +14,13 @@ namespace zeek::storage { class Manager; -// Result from storage operations that may return an error message. If the -// optional value is unset, the operation succeeded. -using ErrorResult = std::optional; +struct OperationResult { + EnumValPtr code; + std::string err_str; + ValPtr value; -// Result from storage operations that return Vals. The ValPtr is an -// IntrusivePtr to some result, and can be null if the operation failed. The -// string value will store an error message if the result is null. -using ValResult = zeek::expected; + RecordValPtr BuildVal(); +}; // Base callback object for async operations. This is just here to allow some @@ -30,41 +28,29 @@ using ValResult = zeek::expected; class ResultCallback { public: ResultCallback() = default; - ResultCallback(IntrusivePtr trigger, const void* assoc); - virtual ~ResultCallback(); + ResultCallback(detail::trigger::TriggerPtr trigger, const void* assoc); + virtual ~ResultCallback() = default; void Timeout(); bool IsSyncCallback() const { return ! trigger; } -protected: - void ValComplete(Val* result); + virtual void Complete(OperationResult res) = 0; + +protected: + void CompleteWithVal(Val* result); -private: IntrusivePtr trigger; const void* assoc = nullptr; }; -// A callback result that returns an ErrorResult. -class ErrorResultCallback : public ResultCallback { +class OperationResultCallback : public ResultCallback { public: - ErrorResultCallback() = default; - ErrorResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc); - virtual void Complete(const ErrorResult& res); - ErrorResult Result() { return result; } + OperationResultCallback() = default; + OperationResultCallback(detail::trigger::TriggerPtr trigger, const void* assoc); + void Complete(OperationResult res) override; + OperationResult Result() { return result; } private: - ErrorResult result; -}; - -// A callback result that returns a ValResult. -class ValResultCallback : public ResultCallback { -public: - ValResultCallback() = default; - ValResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc); - void Complete(const ValResult& res); - ValResult Result() { return result; } - -private: - ValResult result; + OperationResult result; }; class OpenResultCallback; @@ -88,31 +74,31 @@ public: * removed. Set to zero to disable expiration. This time is based on the current network * time. * @param cb An optional callback object if being called via an async context. - * @return An optional value potentially containing an error string if - * needed Will be unset if the operation succeeded. + * @return A struct describing the result of the operation, containing a code, an + * optional error string, and a ValPtr for operations that return values. */ - ErrorResult Put(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, - ErrorResultCallback* cb = nullptr); + OperationResult Put(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + OperationResultCallback* cb = nullptr); /** * Retrieve a value from the backend for a provided key. * * @param key the key to lookup in the backend. * @param cb An optional callback object if being called via an async context. - * @return A std::expected containing either a valid ValPtr with the result - * of the operation or a string containing an error message for failure. + * @return A struct describing the result of the operation, containing a code, an + * optional error string, and a ValPtr for operations that return values. */ - ValResult Get(ValPtr key, ValResultCallback* cb = nullptr); + OperationResult Get(ValPtr key, OperationResultCallback* cb = nullptr); /** * Erases the value for a key from the backend. * * @param key the key to erase * @param cb An optional callback object if being called via an async context. - * @return An optional value potentially containing an error string if - * needed. Will be unset if the operation succeeded. + * @return A struct describing the result of the operation, containing a code, an + * optional error string, and a ValPtr for operations that return values. */ - ErrorResult Erase(ValPtr key, ErrorResultCallback* cb = nullptr); + OperationResult Erase(ValPtr key, OperationResultCallback* cb = nullptr); /** * Returns whether the backend is opened. @@ -151,45 +137,45 @@ protected: * @param vt The script-side type of the values stored in the backend. Used for * validation of types and conversion during retrieval. * @param cb An optional callback object if being called via an async context. - * @return An optional value potentially containing an error string if - * needed. Will be unset if the operation succeeded. + * @return A struct describing the result of the operation, containing a code, an + * optional error string, and a ValPtr for operations that return values. */ - ErrorResult Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb = nullptr); + OperationResult Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb = nullptr); /** * Finalizes the backend when it's being closed. * * @param cb An optional callback object if being called via an async context. - * @return An optional value potentially containing an error string if - * needed. Will be unset if the operation succeeded. + * @return A struct describing the result of the operation, containing a code, an + * optional error string, and a ValPtr for operations that return values. */ - ErrorResult Close(ErrorResultCallback* cb = nullptr); + OperationResult Close(OperationResultCallback* cb = nullptr); /** * The workhorse method for Open(). */ - virtual ErrorResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) = 0; + virtual OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) = 0; /** * The workhorse method for Close(). */ - virtual ErrorResult DoClose(ErrorResultCallback* cb = nullptr) = 0; + virtual OperationResult DoClose(OperationResultCallback* cb = nullptr) = 0; /** * The workhorse method for Put(). */ - virtual ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, - ErrorResultCallback* cb = nullptr) = 0; + virtual OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + OperationResultCallback* cb = nullptr) = 0; /** * The workhorse method for Get(). */ - virtual ValResult DoGet(ValPtr key, ValResultCallback* cb = nullptr) = 0; + virtual OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) = 0; /** * The workhorse method for Erase(). */ - virtual ErrorResult DoErase(ValPtr key, ErrorResultCallback* cb = nullptr) = 0; + virtual OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) = 0; /** * Removes any entries in the backend that have expired. Can be overridden by @@ -197,9 +183,7 @@ protected: */ virtual void Expire() {} - void CompleteCallback(ValResultCallback* cb, const ValResult& data) const; - void CompleteCallback(ErrorResultCallback* cb, const ErrorResult& data) const; - void CompleteCallback(OpenResultCallback* cb, const ErrorResult& data) const; + void CompleteCallback(ResultCallback* cb, const OperationResult& data) const; TypePtr key_type; TypePtr val_type; @@ -235,15 +219,17 @@ protected: // A callback for the Backend::Open() method that returns an error or a backend handle. class OpenResultCallback : public ResultCallback { public: - OpenResultCallback(detail::BackendHandleVal* backend); - OpenResultCallback(IntrusivePtr trigger, const void* assoc, - detail::BackendHandleVal* backend); - void Complete(const ErrorResult& res); - ErrorResult Result() { return result; } + OpenResultCallback(IntrusivePtr backend); + OpenResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc, + IntrusivePtr backend); + void Complete(OperationResult res) override; + + OperationResult Result() const { return result; } + IntrusivePtr Backend() const { return backend; } private: - ErrorResult result; - detail::BackendHandleVal* backend = nullptr; + OperationResult result{}; + IntrusivePtr backend; }; } // namespace zeek::storage diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 86071e81bd..d1401a3ee1 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -4,6 +4,7 @@ zeek_add_subdir_library( Manager.cc Backend.cc Component.cc + ReturnCode.cc BIFS storage.bif) diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index 18f3c9cb35..6f991af5c2 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -6,6 +6,7 @@ #include "zeek/Desc.h" #include "zeek/RunState.h" +#include "zeek/storage/ReturnCode.h" #include "const.bif.netvar_h" @@ -32,7 +33,17 @@ void detail::ExpirationTimer::Dispatch(double t, bool is_expire) { Manager::Manager() : plugin::ComponentManager("Storage", "Backend") {} +Manager::~Manager() { + // TODO: should we shut down any existing backends? force-poll until all of their existing + // operations finish and close them? + + // Don't leave all of these static objects to leak. + ReturnCode::Cleanup(); +} + void Manager::InitPostScript() { + ReturnCode::Initialize(); + detail::backend_opaque = make_intrusive("Storage::Backend"); StartExpirationTimer(); } @@ -62,20 +73,22 @@ zeek::expected Manager::Instantiate(const Tag& type) { return bp; } -ErrorResult Manager::OpenBackend(BackendPtr backend, RecordValPtr options, TypePtr key_type, TypePtr val_type, - OpenResultCallback* cb) { - if ( auto res = backend->Open(std::move(options), std::move(key_type), std::move(val_type), cb); res.has_value() ) { - return util::fmt("Failed to open backend %s: %s", backend->Tag(), res.value().c_str()); +OperationResult Manager::OpenBackend(BackendPtr backend, RecordValPtr options, TypePtr key_type, TypePtr val_type, + OpenResultCallback* cb) { + auto res = backend->Open(std::move(options), std::move(key_type), std::move(val_type), cb); + if ( res.code != ReturnCode::SUCCESS ) { + res.err_str = util::fmt("Failed to open backend %s: %s", backend->Tag(), res.err_str.c_str()); + return res; } RegisterBackend(std::move(backend)); // TODO: post Storage::backend_opened event - return std::nullopt; + return res; } -ErrorResult Manager::CloseBackend(BackendPtr backend, ErrorResultCallback* cb) { +OperationResult Manager::CloseBackend(BackendPtr backend, OperationResultCallback* cb) { // Expiration runs on a separate thread and loops over the vector of backends. The mutex // here ensures exclusive access. This one happens in a block because we can remove the // backend from the vector before actually closing it. @@ -86,13 +99,11 @@ ErrorResult Manager::CloseBackend(BackendPtr backend, ErrorResultCallback* cb) { backends.erase(it); } - if ( auto res = backend->Close(cb); res.has_value() ) { - return util::fmt("Failed to close backend %s: %s", backend->Tag(), res.value().c_str()); - } - - return std::nullopt; + auto res = backend->Close(cb); // TODO: post Storage::backend_lost event + + return res; } void Manager::Expire() { diff --git a/src/storage/Manager.h b/src/storage/Manager.h index 2f240e0486..badd2c6c5f 100644 --- a/src/storage/Manager.h +++ b/src/storage/Manager.h @@ -27,7 +27,7 @@ public: class Manager final : public plugin::ComponentManager { public: Manager(); - ~Manager() = default; + ~Manager(); /** * Initialization of the manager. This is called late during Zeek's @@ -36,8 +36,8 @@ public: void InitPostScript(); /** - * Instantiates a new backend object. The backend will be in a closed state, and OpenBackend() - * will need to be called to fully initialize it. + * Instantiates a new backend object. The backend will be in a closed state, + * and OpenBackend() will need to be called to fully initialize it. * * @param type The tag for the type of backend being opened. * @return A std::expected containing either a valid BackendPtr with the @@ -56,16 +56,22 @@ public: * validation of types. * @param val_type The script-side type of the values stored in the backend. Used for * validation of types and conversion during retrieval. - * @return An optional value potentially containing an error string if needed. Will be - * unset if the operation succeeded. + * @param cb An optional callback object if being called via an async context. + * @return A struct describing the result of the operation, containing a code, an + * optional error string, and a ValPtr for operations that return values. */ - ErrorResult OpenBackend(BackendPtr backend, RecordValPtr options, TypePtr key_type, TypePtr val_type, - OpenResultCallback* cb = nullptr); + OperationResult OpenBackend(BackendPtr backend, RecordValPtr options, TypePtr key_type, TypePtr val_type, + OpenResultCallback* cb = nullptr); /** * Closes a storage backend. + * + * @param backend A pointer to the backend being closed. + * @param cb An optional callback object if being called via an async context. + * @return A struct describing the result of the operation, containing a code, an + * optional error string, and a ValPtr for operations that return values. */ - ErrorResult CloseBackend(BackendPtr backend, ErrorResultCallback* cb = nullptr); + OperationResult CloseBackend(BackendPtr backend, OperationResultCallback* cb = nullptr); void Expire(); diff --git a/src/storage/ReturnCode.cc b/src/storage/ReturnCode.cc new file mode 100644 index 0000000000..50d13554ce --- /dev/null +++ b/src/storage/ReturnCode.cc @@ -0,0 +1,77 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/ReturnCode.h" + +#include "zeek/Val.h" + +namespace zeek::storage { + +EnumValPtr ReturnCode::SUCCESS; +EnumValPtr ReturnCode::VAL_TYPE_MISMATCH; +EnumValPtr ReturnCode::KEY_TYPE_MISMATCH; +EnumValPtr ReturnCode::NOT_CONNECTED; +EnumValPtr ReturnCode::TIMEOUT; +EnumValPtr ReturnCode::CONNECTION_LOST; +EnumValPtr ReturnCode::OPERATION_FAILED; +EnumValPtr ReturnCode::KEY_NOT_FOUND; +EnumValPtr ReturnCode::KEY_EXISTS; +EnumValPtr ReturnCode::CONNECTION_FAILED; +EnumValPtr ReturnCode::DISCONNECTION_FAILED; +EnumValPtr ReturnCode::INITIALIZATION_FAILED; + +void ReturnCode::Initialize() { + static const auto& return_code_type = zeek::id::find_type("Storage::ReturnCode"); + + auto tmp = return_code_type->Lookup("Storage::SUCCESS"); + SUCCESS = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::VAL_TYPE_MISMATCH"); + VAL_TYPE_MISMATCH = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::KEY_TYPE_MISMATCH"); + KEY_TYPE_MISMATCH = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::NOT_CONNECTED"); + NOT_CONNECTED = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::TIMEOUT"); + TIMEOUT = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::CONNECTION_LOST"); + CONNECTION_LOST = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::OPERATION_FAILED"); + OPERATION_FAILED = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::KEY_NOT_FOUND"); + KEY_NOT_FOUND = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::KEY_EXISTS"); + KEY_EXISTS = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::CONNECTION_FAILED"); + CONNECTION_FAILED = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::DISCONNECTION_FAILED"); + DISCONNECTION_FAILED = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::INITIALIZATION_FAILED"); + INITIALIZATION_FAILED = return_code_type->GetEnumVal(tmp); +} + +void ReturnCode::Cleanup() { + SUCCESS.reset(); + VAL_TYPE_MISMATCH.reset(); + KEY_TYPE_MISMATCH.reset(); + NOT_CONNECTED.reset(); + TIMEOUT.reset(); + CONNECTION_LOST.reset(); + OPERATION_FAILED.reset(); + KEY_NOT_FOUND.reset(); + KEY_EXISTS.reset(); + CONNECTION_FAILED.reset(); + DISCONNECTION_FAILED.reset(); + INITIALIZATION_FAILED.reset(); +} + +} // namespace zeek::storage diff --git a/src/storage/ReturnCode.h b/src/storage/ReturnCode.h new file mode 100644 index 0000000000..826408c9df --- /dev/null +++ b/src/storage/ReturnCode.h @@ -0,0 +1,37 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/IntrusivePtr.h" + +namespace zeek { +class EnumVal; +using EnumValPtr = IntrusivePtr; + +namespace storage { + +/** + * A collection of EnumValPtrs for the default set of result codes in the storage framework. + * should be kept up-to-date with the Storage::ReturnCodes script-level enum. + */ +class ReturnCode { +public: + static void Initialize(); + static void Cleanup(); + + static EnumValPtr SUCCESS; + static EnumValPtr VAL_TYPE_MISMATCH; + static EnumValPtr KEY_TYPE_MISMATCH; + static EnumValPtr NOT_CONNECTED; + static EnumValPtr TIMEOUT; + static EnumValPtr CONNECTION_LOST; + static EnumValPtr OPERATION_FAILED; + static EnumValPtr KEY_NOT_FOUND; + static EnumValPtr KEY_EXISTS; + static EnumValPtr CONNECTION_FAILED; + static EnumValPtr DISCONNECTION_FAILED; + static EnumValPtr INITIALIZATION_FAILED; +}; + +} // namespace storage +} // namespace zeek diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index d23f62a4e4..b1fb29ed03 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -7,6 +7,7 @@ #include "zeek/RunState.h" #include "zeek/Val.h" #include "zeek/iosource/Manager.h" +#include "zeek/storage/ReturnCode.h" #include "hiredis/adapters/poll.h" #include "hiredis/async.h" @@ -37,21 +38,21 @@ void redisOnDisconnect(const redisAsyncContext* ctx, int status) { void redisPut(redisAsyncContext* ctx, void* reply, void* privdata) { auto t = Tracer("put"); auto backend = static_cast(ctx->data); - auto callback = static_cast(privdata); + auto callback = static_cast(privdata); backend->HandlePutResult(static_cast(reply), callback); } void redisGet(redisAsyncContext* ctx, void* reply, void* privdata) { auto t = Tracer("get"); auto backend = static_cast(ctx->data); - auto callback = static_cast(privdata); + auto callback = static_cast(privdata); backend->HandleGetResult(static_cast(reply), callback); } void redisErase(redisAsyncContext* ctx, void* reply, void* privdata) { auto t = Tracer("erase"); auto backend = static_cast(ctx->data); - auto callback = static_cast(privdata); + auto callback = static_cast(privdata); backend->HandleEraseResult(static_cast(reply), callback); } @@ -122,7 +123,7 @@ storage::BackendPtr Redis::Instantiate(std::string_view tag) { return make_intru /** * Called by the manager system to open the backend. */ -ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { +OperationResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { RecordValPtr backend_options = options->GetField("redis"); key_prefix = backend_options->GetField("key_prefix")->ToStdString(); @@ -137,9 +138,10 @@ ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { } else { StringValPtr unix_sock = backend_options->GetField("server_unix_socket"); - if ( ! unix_sock ) - return util::fmt( - "Either server_host/server_port or server_unix_socket must be set in Redis options record"); + if ( ! unix_sock ) { + return {ReturnCode::CONNECTION_FAILED, + "Either server_host/server_port or server_unix_socket must be set in Redis options record"}; + } server_addr = unix_sock->ToStdString(); REDIS_OPTIONS_SET_UNIX(&opt, server_addr.c_str()); @@ -164,11 +166,16 @@ ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { redisAsyncFree(async_ctx); async_ctx = nullptr; - return errmsg; + return {ReturnCode::CONNECTION_FAILED, errmsg}; } ++active_ops; + // There's no way to pass privdata down to the connect handler like there is for + // the other callbacks. Store the open callback so that it can be dealt with from + // OnConnect(). + open_cb = cb; + // TODO: Sort out how to pass the zeek callbacks for both open/done to the async // callbacks from hiredis so they can return errors. @@ -199,13 +206,13 @@ ErrorResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { async_ctx->ev.addWrite = redisAddWrite; async_ctx->ev.delWrite = redisDelWrite; - return std::nullopt; + return {ReturnCode::SUCCESS}; } /** * Finalizes the backend when it's being closed. */ -ErrorResult Redis::DoClose(ErrorResultCallback* cb) { +OperationResult Redis::DoClose(OperationResultCallback* cb) { connected = false; redisAsyncDisconnect(async_ctx); @@ -216,19 +223,22 @@ ErrorResult Redis::DoClose(ErrorResultCallback* cb) { // TODO: handle response } + CompleteCallback(cb, {ReturnCode::SUCCESS}); + redisAsyncFree(async_ctx); async_ctx = nullptr; - return std::nullopt; + return {ReturnCode::SUCCESS}; } /** * The workhorse method for Put(). This must be implemented by plugins. */ -ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, ErrorResultCallback* cb) { +OperationResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, + OperationResultCallback* cb) { // The async context will queue operations until it's connected fully. if ( ! connected && ! async_ctx ) - return "Connection is not open"; + return {ReturnCode::NOT_CONNECTED}; std::string format = "SET %s:%s %s"; if ( ! overwrite ) @@ -250,7 +260,7 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira json_value.data()); if ( connected && status == REDIS_ERR ) - return util::fmt("Failed to queue put operation: %s", async_ctx->errstr); + return {ReturnCode::OPERATION_FAILED, util::fmt("Failed to queue put operation: %s", async_ctx->errstr)}; ++active_ops; @@ -265,52 +275,52 @@ ErrorResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expira status = redisAsyncCommand(async_ctx, redisGeneric, NULL, format.c_str(), key_prefix.data(), expiration_time, json_key.data()); if ( connected && status == REDIS_ERR ) - return util::fmt("ZADD operation failed: %s", async_ctx->errstr); + return {ReturnCode::OPERATION_FAILED, util::fmt("ZADD operation failed: %s", async_ctx->errstr)}; ++active_ops; } - return std::nullopt; + return {ReturnCode::SUCCESS}; } /** * The workhorse method for Get(). This must be implemented for plugins. */ -ValResult Redis::DoGet(ValPtr key, ValResultCallback* cb) { +OperationResult Redis::DoGet(ValPtr key, OperationResultCallback* cb) { // The async context will queue operations until it's connected fully. if ( ! connected && ! async_ctx ) - return zeek::unexpected("Connection is not open"); + return {ReturnCode::NOT_CONNECTED}; int status = redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%s", key_prefix.data(), key->ToJSON()->ToStdStringView().data()); if ( connected && status == REDIS_ERR ) - return zeek::unexpected(util::fmt("Failed to queue get operation: %s", async_ctx->errstr)); + return {ReturnCode::OPERATION_FAILED, util::fmt("Failed to queue get operation: %s", async_ctx->errstr)}; ++active_ops; // There isn't a result to return here. That happens in HandleGetResult for // async operations. - return zeek::unexpected(""); + return {ReturnCode::SUCCESS}; } /** * The workhorse method for Erase(). This must be implemented for plugins. */ -ErrorResult Redis::DoErase(ValPtr key, ErrorResultCallback* cb) { +OperationResult Redis::DoErase(ValPtr key, OperationResultCallback* cb) { // The async context will queue operations until it's connected fully. if ( ! connected && ! async_ctx ) - return "Connection is not open"; + return {ReturnCode::NOT_CONNECTED}; int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%s", key_prefix.data(), key->ToJSON()->ToStdStringView().data()); if ( connected && status == REDIS_ERR ) - return util::fmt("Failed to queue erase operation failed: %s", async_ctx->errstr); + return {ReturnCode::OPERATION_FAILED, async_ctx->errstr}; ++active_ops; - return std::nullopt; + return {ReturnCode::SUCCESS}; } void Redis::Expire() { @@ -335,11 +345,15 @@ void Redis::Expire() { redisReply* reply = reply_queue.front(); reply_queue.pop_front(); - if ( reply && reply->elements == 0 ) { + if ( reply->elements == 0 ) { freeReplyObject(reply); return; } + // The data from the reply to ZRANGEBYSCORE gets deleted as part of the + // commands below so we don't need to free it manually. Doing so results in + // a double-free. + // TODO: it's possible to pass multiple keys to a DEL operation but it requires // building an array of the strings, building up the DEL command with entries, // and passing the array as a block somehow. There's no guarantee it'd be faster @@ -357,33 +371,29 @@ void Redis::Expire() { ++active_ops; Poll(); - - // This can't be freed until the other commands finish because the memory for - // the strings doesn't get copied when making the DEL commands. - // freeReplyObject(reply); } -void Redis::HandlePutResult(redisReply* reply, ErrorResultCallback* callback) { +void Redis::HandlePutResult(redisReply* reply, OperationResultCallback* callback) { --active_ops; - ErrorResult res; + OperationResult res{ReturnCode::SUCCESS}; if ( ! connected ) - res = util::fmt("Connection is not open"); + res = {ReturnCode::NOT_CONNECTED}; else if ( ! reply ) - res = util::fmt("Async put operation returned null reply"); + res = {ReturnCode::OPERATION_FAILED, "Async put operation returned null reply"}; else if ( reply && reply->type == REDIS_REPLY_ERROR ) - res = util::fmt("Async put operation failed: %s", reply->str); + res = {ReturnCode::OPERATION_FAILED, util::fmt("Async put operation failed: %s", reply->str)}; freeReplyObject(reply); CompleteCallback(callback, res); } -void Redis::HandleGetResult(redisReply* reply, ValResultCallback* callback) { +void Redis::HandleGetResult(redisReply* reply, OperationResultCallback* callback) { --active_ops; - ValResult res; + OperationResult res; if ( ! connected ) - res = zeek::unexpected("Connection is not open"); + res = {ReturnCode::NOT_CONNECTED}; else res = ParseGetReply(reply); @@ -391,19 +401,20 @@ void Redis::HandleGetResult(redisReply* reply, ValResultCallback* callback) { CompleteCallback(callback, res); } -void Redis::HandleEraseResult(redisReply* reply, ErrorResultCallback* callback) { +void Redis::HandleEraseResult(redisReply* reply, OperationResultCallback* callback) { --active_ops; if ( callback->IsSyncCallback() ) reply_queue.push_back(reply); else { - ErrorResult res; + OperationResult res{ReturnCode::SUCCESS}; + if ( ! connected ) - res = "Connection is not open"; + res = {ReturnCode::NOT_CONNECTED}; else if ( ! reply ) - res = util::fmt("Async erase operation returned null reply"); + res = {ReturnCode::OPERATION_FAILED, "Async erase operation returned null reply"}; else if ( reply && reply->type == REDIS_REPLY_ERROR ) - res = util::fmt("Async erase operation failed: %s", reply->str); + res = {ReturnCode::OPERATION_FAILED, util::fmt("Async erase operation failed: %s", reply->str)}; freeReplyObject(reply); CompleteCallback(callback, res); @@ -421,9 +432,14 @@ void Redis::OnConnect(int status) { if ( status == REDIS_OK ) { connected = true; + CompleteCallback(open_cb, {ReturnCode::SUCCESS}); + // TODO: post connect event return; } + connected = false; + CompleteCallback(open_cb, {ReturnCode::CONNECTION_FAILED}); + // TODO: we could attempt to reconnect here } @@ -436,6 +452,7 @@ void Redis::OnDisconnect(int status) { } else { // TODO: this was unintentional, should we reconnect? + // TODO: post disconnect event } connected = false; @@ -448,19 +465,19 @@ void Redis::ProcessFd(int fd, int flags) { redisAsyncHandleWrite(async_ctx); } -ValResult Redis::ParseGetReply(redisReply* reply) const { - ValResult res; +OperationResult Redis::ParseGetReply(redisReply* reply) const { + OperationResult res; if ( ! reply ) - res = zeek::unexpected("GET returned null reply"); + res = {ReturnCode::OPERATION_FAILED, "GET returned null reply"}; else if ( ! reply->str ) - res = zeek::unexpected("GET returned key didn't exist"); + res = {ReturnCode::KEY_NOT_FOUND}; else { auto val = zeek::detail::ValFromJSON(reply->str, val_type, Func::nil); if ( std::holds_alternative(val) ) - res = std::get(val); + res = {ReturnCode::SUCCESS, "", std::get(val)}; else - res = zeek::unexpected(std::get(val)); + res = {ReturnCode::OPERATION_FAILED, std::get(val)}; } return res; diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index 3a7dd21dcc..b5a65d12d2 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -29,12 +29,12 @@ public: /** * Called by the manager system to open the backend. */ - ErrorResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; + OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; /** * Finalizes the backend when it's being closed. */ - ErrorResult DoClose(ErrorResultCallback* cb = nullptr) override; + OperationResult DoClose(OperationResultCallback* cb = nullptr) override; /** * Returns whether the backend is opened. @@ -44,18 +44,18 @@ public: /** * The workhorse method for Retrieve(). */ - ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, - ErrorResultCallback* cb = nullptr) override; + OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + OperationResultCallback* cb = nullptr) override; /** * The workhorse method for Get(). */ - ValResult DoGet(ValPtr key, ValResultCallback* cb = nullptr) override; + OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) override; /** * The workhorse method for Erase(). */ - ErrorResult DoErase(ValPtr key, ErrorResultCallback* cb = nullptr) override; + OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) override; /** * Removes any entries in the backend that have expired. Can be overridden by @@ -72,9 +72,9 @@ public: void OnConnect(int status); void OnDisconnect(int status); - void HandlePutResult(redisReply* reply, ErrorResultCallback* callback); - void HandleGetResult(redisReply* reply, ValResultCallback* callback); - void HandleEraseResult(redisReply* reply, ErrorResultCallback* callback); + void HandlePutResult(redisReply* reply, OperationResultCallback* callback); + void HandleGetResult(redisReply* reply, OperationResultCallback* callback); + void HandleEraseResult(redisReply* reply, OperationResultCallback* callback); void HandleZRANGEBYSCORE(redisReply* reply); // HandleGeneric exists so that async-running-as-sync operations can remove @@ -85,7 +85,7 @@ protected: void Poll() override; private: - ValResult ParseGetReply(redisReply* reply) const; + OperationResult ParseGetReply(redisReply* reply) const; redisAsyncContext* async_ctx = nullptr; @@ -94,6 +94,8 @@ private: // poll. std::deque reply_queue; + OpenResultCallback* open_cb; + std::string server_addr; std::string key_prefix; std::atomic connected = false; diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index 34f1aebcfd..ddac358994 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -5,6 +5,7 @@ #include "zeek/3rdparty/sqlite3.h" #include "zeek/Func.h" #include "zeek/Val.h" +#include "zeek/storage/ReturnCode.h" namespace zeek::storage::backend::sqlite { @@ -13,13 +14,13 @@ storage::BackendPtr SQLite::Instantiate(std::string_view tag) { return make_intr /** * Called by the manager system to open the backend. */ -ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { +OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { if ( sqlite3_threadsafe() == 0 ) { std::string res = "SQLite reports that it is not threadsafe. Zeek needs a threadsafe version of " "SQLite. Aborting"; Error(res.c_str()); - return res; + return {ReturnCode::INITIALIZATION_FAILED, res}; } // Allow connections to same DB to use single data/schema cache. Also @@ -33,10 +34,10 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { full_path = zeek::filesystem::path(path->ToStdString()).string(); table_name = backend_options->GetField("table_name")->ToStdString(); - auto open_res = - checkError(sqlite3_open_v2(full_path.c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, NULL)); - if ( open_res.has_value() ) { + if ( auto open_res = + checkError(sqlite3_open_v2(full_path.c_str(), &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, NULL)); + open_res.code != ReturnCode::SUCCESS ) { sqlite3_close_v2(db); db = nullptr; return open_res; @@ -51,7 +52,7 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { Error(err.c_str()); sqlite3_free(errorMsg); Close(); - return err; + return {ReturnCode::INITIALIZATION_FAILED, err}; } if ( int res = sqlite3_exec(db, "pragma integrity_check", NULL, NULL, &errorMsg); res != SQLITE_OK ) { @@ -59,7 +60,7 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { Error(err.c_str()); sqlite3_free(errorMsg); Close(); - return err; + return {ReturnCode::INITIALIZATION_FAILED, err}; } auto tuning_params = backend_options->GetField("tuning_params")->ToMap(); @@ -73,7 +74,7 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { Error(err.c_str()); sqlite3_free(errorMsg); Close(); - return err; + return {ReturnCode::INITIALIZATION_FAILED, err}; } } @@ -91,7 +92,7 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { for ( const auto& [key, stmt] : statements ) { sqlite3_stmt* ps; if ( auto prep_res = checkError(sqlite3_prepare_v2(db, stmt.c_str(), stmt.size(), &ps, NULL)); - prep_res.has_value() ) { + prep_res.code != ReturnCode::SUCCESS ) { Close(); return prep_res; } @@ -99,14 +100,14 @@ ErrorResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { prepared_stmts.insert({key, ps}); } - return std::nullopt; + return {ReturnCode::SUCCESS}; } /** * Finalizes the backend when it's being closed. */ -ErrorResult SQLite::DoClose(ErrorResultCallback* cb) { - ErrorResult err_res; +OperationResult SQLite::DoClose(OperationResultCallback* cb) { + OperationResult op_res{ReturnCode::SUCCESS}; if ( db ) { for ( const auto& [k, stmt] : prepared_stmts ) { @@ -117,27 +118,29 @@ ErrorResult SQLite::DoClose(ErrorResultCallback* cb) { char* errmsg; if ( int res = sqlite3_exec(db, "pragma optimize", NULL, NULL, &errmsg); res != SQLITE_OK ) { - err_res = util::fmt("Sqlite failed to optimize at shutdown: %s", errmsg); + op_res = {ReturnCode::DISCONNECTION_FAILED, util::fmt("Sqlite failed to optimize at shutdown: %s", errmsg)}; sqlite3_free(&errmsg); + // TODO: we're shutting down. does this error matter other than being informational? } if ( int res = sqlite3_close_v2(db); res != SQLITE_OK ) { - if ( ! err_res.has_value() ) - err_res = "Sqlite could not close connection"; + if ( op_res.err_str.empty() ) + op_res.err_str = "Sqlite could not close connection"; } db = nullptr; } - return err_res; + return op_res; } /** * The workhorse method for Put(). This must be implemented by plugins. */ -ErrorResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, ErrorResultCallback* cb) { +OperationResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, + OperationResultCallback* cb) { if ( ! db ) - return "Database was not open"; + return {ReturnCode::NOT_CONNECTED}; auto json_key = key->ToJSON(); auto json_value = value->ToJSON(); @@ -150,56 +153,56 @@ ErrorResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expir auto key_str = json_key->ToStdStringView(); if ( auto res = checkError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); - res.has_value() ) { + res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } auto value_str = json_value->ToStdStringView(); if ( auto res = checkError(sqlite3_bind_text(stmt, 2, value_str.data(), value_str.size(), SQLITE_STATIC)); - res.has_value() ) { + res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } - if ( auto res = checkError(sqlite3_bind_double(stmt, 3, expiration_time)); res.has_value() ) { + if ( auto res = checkError(sqlite3_bind_double(stmt, 3, expiration_time)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } if ( overwrite ) { if ( auto res = checkError(sqlite3_bind_text(stmt, 4, value_str.data(), value_str.size(), SQLITE_STATIC)); - res.has_value() ) { + res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } } - if ( auto res = checkError(sqlite3_step(stmt)); res.has_value() ) { + if ( auto res = checkError(sqlite3_step(stmt)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } sqlite3_reset(stmt); - return std::nullopt; + return {ReturnCode::SUCCESS}; } /** * The workhorse method for Get(). This must be implemented for plugins. */ -ValResult SQLite::DoGet(ValPtr key, ValResultCallback* cb) { +OperationResult SQLite::DoGet(ValPtr key, OperationResultCallback* cb) { if ( ! db ) - return zeek::unexpected("Database was not open"); + return {ReturnCode::NOT_CONNECTED}; auto json_key = key->ToJSON(); auto stmt = prepared_stmts["get"]; auto key_str = json_key->ToStdStringView(); if ( auto res = checkError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); - res.has_value() ) { + res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); - return zeek::unexpected(res.value()); + return res; } int errorcode = sqlite3_step(stmt); @@ -210,38 +213,38 @@ ValResult SQLite::DoGet(ValPtr key, ValResultCallback* cb) { sqlite3_reset(stmt); if ( std::holds_alternative(val) ) { ValPtr val_v = std::get(val); - return val_v; + return {ReturnCode::SUCCESS, "", val_v}; } else { - return zeek::unexpected(std::get(val)); + return {ReturnCode::OPERATION_FAILED, std::get(val)}; } } - return zeek::unexpected(util::fmt("Failed to find row for key: %s", sqlite3_errstr(errorcode))); + return {ReturnCode::KEY_NOT_FOUND}; } /** * The workhorse method for Erase(). This must be implemented for plugins. */ -ErrorResult SQLite::DoErase(ValPtr key, ErrorResultCallback* cb) { +OperationResult SQLite::DoErase(ValPtr key, OperationResultCallback* cb) { if ( ! db ) - return "Database was not open"; + return {ReturnCode::NOT_CONNECTED}; auto json_key = key->ToJSON(); auto stmt = prepared_stmts["erase"]; auto key_str = json_key->ToStdStringView(); if ( auto res = checkError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); - res.has_value() ) { + res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } - if ( auto res = checkError(sqlite3_step(stmt)); res.has_value() ) { + if ( auto res = checkError(sqlite3_step(stmt)); res.code != ReturnCode::SUCCESS ) { return res; } - return std::nullopt; + return {ReturnCode::SUCCESS}; } /** @@ -251,23 +254,24 @@ ErrorResult SQLite::DoErase(ValPtr key, ErrorResultCallback* cb) { void SQLite::Expire() { auto stmt = prepared_stmts["expire"]; - if ( auto res = checkError(sqlite3_bind_double(stmt, 1, run_state::network_time)); res.has_value() ) { + if ( auto res = checkError(sqlite3_bind_double(stmt, 1, run_state::network_time)); + res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); // TODO: do something with the error here? } - if ( auto res = checkError(sqlite3_step(stmt)); res.has_value() ) { + if ( auto res = checkError(sqlite3_step(stmt)); res.code != ReturnCode::SUCCESS ) { // TODO: do something with the error here? } } // returns true in case of error -ErrorResult SQLite::checkError(int code) { +OperationResult SQLite::checkError(int code) { if ( code != SQLITE_OK && code != SQLITE_DONE ) { - return util::fmt("SQLite call failed: %s", sqlite3_errmsg(db)); + return {ReturnCode::OPERATION_FAILED, util::fmt("SQLite call failed: %s", sqlite3_errmsg(db)), nullptr}; } - return std::nullopt; + return {ReturnCode::SUCCESS}; } } // namespace zeek::storage::backend::sqlite diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index ad0869bad2..17e3e9716d 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -20,12 +20,12 @@ public: /** * Called by the manager system to open the backend. */ - ErrorResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; + OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; /** * Finalizes the backend when it's being closed. */ - ErrorResult DoClose(ErrorResultCallback* cb = nullptr) override; + OperationResult DoClose(OperationResultCallback* cb = nullptr) override; /** * Returns whether the backend is opened. @@ -35,18 +35,18 @@ public: /** * The workhorse method for Put(). */ - ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, - ErrorResultCallback* cb = nullptr) override; + OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + OperationResultCallback* cb = nullptr) override; /** * The workhorse method for Get(). */ - ValResult DoGet(ValPtr key, ValResultCallback* cb = nullptr) override; + OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) override; /** * The workhorse method for Erase(). */ - ErrorResult DoErase(ValPtr key, ErrorResultCallback* cb = nullptr) override; + OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) override; /** * Removes any entries in the backend that have expired. Can be overridden by @@ -55,7 +55,7 @@ public: void Expire() override; private: - ErrorResult checkError(int code); + OperationResult checkError(int code); sqlite3* db = nullptr; std::unordered_map prepared_stmts; diff --git a/src/storage/storage.bif b/src/storage/storage.bif index a29c5fd801..013b95e8be 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -1,8 +1,9 @@ %%{ -#include "zeek/storage/Backend.h" -#include "zeek/storage/Manager.h" #include "zeek/Trigger.h" #include "zeek/Frame.h" +#include "zeek/storage/Backend.h" +#include "zeek/storage/Manager.h" +#include "zeek/storage/ReturnCode.h" using namespace zeek; using namespace zeek::storage; @@ -52,11 +53,11 @@ event Storage::backend_lost%(%); module Storage::Async; -function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): opaque of Storage::BackendHandle +function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): Storage::OperationResult %{ auto trigger = init_trigger(frame); if ( ! trigger ) - return val_mgr->Bool(false); + return nullptr; auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; Tag tag{btype_val}; @@ -64,13 +65,13 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, auto b = storage_mgr->Instantiate(tag); if ( ! b.has_value() ) { - emit_builtin_error(b.error().c_str()); + emit_builtin_error(util::fmt("Failed to instantiate backend: %s", b.error().c_str())); return nullptr; } auto bh = make_intrusive(b.value()); - auto cb = new OpenResultCallback(trigger, frame->GetTriggerAssoc(), bh.release()); + auto cb = new OpenResultCallback(trigger, frame->GetTriggerAssoc(), bh); auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; @@ -91,21 +92,28 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, return nullptr; %} -function Storage::Async::__close_backend%(backend: opaque of Storage::BackendHandle%) : bool +function Storage::Async::__close_backend%(backend: opaque of Storage::BackendHandle%) : Storage::OperationResult %{ + static auto op_result_type = id::find_type("Storage::OperationResult"); + auto op_result = make_intrusive(op_result_type); + auto trigger = init_trigger(frame); if ( ! trigger ) - return val_mgr->Bool(false); + return nullptr; auto b = dynamic_cast(backend); if ( ! b ) { - emit_builtin_error("Invalid storage handle", backend); - return val_mgr->Bool(false); + op_result->Assign(0, ReturnCode::OPERATION_FAILED); + op_result->Assign(1, make_intrusive("Invalid storage handlle")); + return op_result; + } + else if ( ! b->backend->IsOpen() ) { + op_result->Assign(0, ReturnCode::NOT_CONNECTED); + op_result->Assign(1, make_intrusive("Backend is closed")); + return op_result; } - else if ( ! b->backend->IsOpen() ) - return val_mgr->Bool(true); - auto cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); + auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); auto close_res = storage_mgr->CloseBackend(b->backend, cb); if ( ! b->backend->SupportsAsync() ) { @@ -124,24 +132,31 @@ function Storage::Async::__close_backend%(backend: opaque of Storage::BackendHan %} function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, - overwrite: bool, expire_time: interval%): bool + overwrite: bool, expire_time: interval%): Storage::OperationResult %{ + static auto op_result_type = id::find_type("Storage::OperationResult"); + auto op_result = make_intrusive(op_result_type); + auto trigger = init_trigger(frame); if ( ! trigger ) - return val_mgr->Bool(false); + return nullptr; auto b = dynamic_cast(backend); if ( ! b ) { - emit_builtin_error("Invalid storage handle", backend); - return val_mgr->Bool(false); + op_result->Assign(0, ReturnCode::OPERATION_FAILED); + op_result->Assign(1, make_intrusive("Invalid storage handlle")); + return op_result; + } + else if ( ! b->backend->IsOpen() ) { + op_result->Assign(0, ReturnCode::NOT_CONNECTED); + op_result->Assign(1, make_intrusive("Backend is closed")); + return op_result; } - else if ( ! b->backend->IsOpen() ) - return val_mgr->Bool(false); if ( expire_time > 0.0 ) expire_time += run_state::network_time; - auto cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); + auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; auto put_res = b->backend->Put(key_v, val_v, overwrite, expire_time, cb); @@ -161,28 +176,28 @@ function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: return nullptr; %} -function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: any%): val_result +function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult %{ - static auto val_result_type = id::find_type("val_result"); - auto val_result = make_intrusive(val_result_type); + static auto op_result_type = id::find_type("Storage::OperationResult"); + auto op_result = make_intrusive(op_result_type); auto trigger = init_trigger(frame); - if ( ! trigger ) { - val_result->Assign(1, make_intrusive("Failed to create trigger")); - return val_result; - } + if ( ! trigger ) + return nullptr; auto b = dynamic_cast(backend); if ( ! b ) { - val_result->Assign(1, make_intrusive("Invalid storage handlle")); - return val_result; + op_result->Assign(0, ReturnCode::OPERATION_FAILED); + op_result->Assign(1, make_intrusive("Invalid storage handlle")); + return op_result; } else if ( ! b->backend->IsOpen() ) { - val_result->Assign(1, make_intrusive("Backend is closed")); - return val_result; + op_result->Assign(0, ReturnCode::NOT_CONNECTED); + op_result->Assign(1, make_intrusive("Backend is closed")); + return op_result; } - auto cb = new ValResultCallback(trigger, frame->GetTriggerAssoc()); + auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); auto key_v = IntrusivePtr{NewRef{}, key}; auto get_res = b->backend->Get(key_v, cb); @@ -201,21 +216,28 @@ function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: return nullptr; %} -function Storage::Async::__erase%(backend: opaque of Storage::BackendHandle, key: any%): bool +function Storage::Async::__erase%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult %{ + static auto op_result_type = id::find_type("Storage::OperationResult"); + auto op_result = make_intrusive(op_result_type); + auto trigger = init_trigger(frame); if ( ! trigger ) - return val_mgr->Bool(false); + return nullptr; auto b = dynamic_cast(backend); if ( ! b ) { - emit_builtin_error("Invalid storage handle", backend); - return val_mgr->Bool(false); + op_result->Assign(0, ReturnCode::OPERATION_FAILED); + op_result->Assign(1, make_intrusive("Invalid storage handlle")); + return op_result; + } + else if ( ! b->backend->IsOpen() ) { + op_result->Assign(0, ReturnCode::NOT_CONNECTED); + op_result->Assign(1, make_intrusive("Backend is closed")); + return op_result; } - else if ( ! b->backend->IsOpen() ) - return val_mgr->Bool(false); - auto cb = new ErrorResultCallback(trigger, frame->GetTriggerAssoc()); + auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); auto key_v = IntrusivePtr{NewRef{}, key}; auto erase_res = b->backend->Erase(key_v, cb); @@ -236,8 +258,10 @@ function Storage::Async::__erase%(backend: opaque of Storage::BackendHandle, key module Storage::Sync; -function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): opaque of Storage::BackendHandle +function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): Storage::OperationResult %{ + static auto op_result_type = id::find_type("Storage::OperationResult"); + auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; Tag tag{btype_val}; @@ -250,7 +274,7 @@ function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, k auto bh = make_intrusive(b.value()); - auto cb = new OpenResultCallback(bh.get()); + auto cb = new OpenResultCallback(bh); auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; @@ -265,26 +289,28 @@ function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, k delete cb; - if ( open_res.has_value() ) { - emit_builtin_error(open_res.value().c_str()); - return val_mgr->Bool(false); - } - - return bh; + return open_res.BuildVal(); %} -function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHandle%) : bool +function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHandle%) : Storage::OperationResult %{ + static auto op_result_type = id::find_type("Storage::OperationResult"); + auto b = dynamic_cast(backend); if ( ! b ) { - emit_builtin_error("Invalid storage handle", backend); - return val_mgr->Bool(false); + auto op_result = make_intrusive(op_result_type); + op_result->Assign(0, ReturnCode::OPERATION_FAILED); + op_result->Assign(1, make_intrusive("Invalid storage handlle")); + return op_result; + } + else if ( ! b->backend->IsOpen() ) { + auto op_result = make_intrusive(op_result_type); + op_result->Assign(0, ReturnCode::NOT_CONNECTED); + op_result->Assign(1, make_intrusive("Backend is closed")); + return op_result; } - else if ( ! b->backend->IsOpen() ) - // Return true here since the backend is already closed - return val_mgr->Bool(true); - auto cb = new ErrorResultCallback(); + auto cb = new OperationResultCallback(); auto close_res = storage_mgr->CloseBackend(b->backend, cb); // If the backend only supports async, block until it's ready and then pull the result out of @@ -296,26 +322,29 @@ function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHand delete cb; - if ( close_res.has_value() ) { - emit_builtin_error(close_res.value().c_str()); - return val_mgr->Bool(false); - } - - return val_mgr->Bool(true); + return close_res.BuildVal(); %} function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, - overwrite: bool, expire_time: interval%): bool + overwrite: bool, expire_time: interval%): Storage::OperationResult %{ + static auto op_result_type = id::find_type("Storage::OperationResult"); + auto b = dynamic_cast(backend); if ( ! b ) { - emit_builtin_error("Invalid storage handle", backend); - return val_mgr->Bool(false); + auto op_result = make_intrusive(op_result_type); + op_result->Assign(0, ReturnCode::OPERATION_FAILED); + op_result->Assign(1, make_intrusive("Invalid storage handlle")); + return op_result; + } + else if ( ! b->backend->IsOpen() ) { + auto op_result = make_intrusive(op_result_type); + op_result->Assign(0, ReturnCode::NOT_CONNECTED); + op_result->Assign(1, make_intrusive("Backend is closed")); + return op_result; } - else if ( ! b->backend->IsOpen() ) - return val_mgr->Bool(false); - auto cb = new ErrorResultCallback(); + auto cb = new OperationResultCallback(); auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; auto put_res = b->backend->Put(key_v, val_v, overwrite, expire_time, cb); @@ -329,31 +358,29 @@ function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: a delete cb; - if ( put_res.has_value() ) { - emit_builtin_error(util::fmt("Failed to store data: %s", put_res.value().c_str())); - return val_mgr->Bool(false); - } - - return val_mgr->Bool(true); + return put_res.BuildVal(); %} -function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: any%): val_result +function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult %{ - static auto val_result_type = id::find_type("val_result"); - auto val_result = make_intrusive(val_result_type); + static auto op_result_type = id::find_type("Storage::OperationResult"); auto b = dynamic_cast(backend); if ( ! b ) { - val_result->Assign(1, make_intrusive("Invalid storage handlle")); - return val_result; + auto op_result = make_intrusive(op_result_type); + op_result->Assign(0, ReturnCode::OPERATION_FAILED); + op_result->Assign(1, make_intrusive("Invalid storage handlle")); + return op_result; } else if ( ! b->backend->IsOpen() ) { - val_result->Assign(1, make_intrusive("Backend is closed")); - return val_result; + auto op_result = make_intrusive(op_result_type); + op_result->Assign(0, ReturnCode::NOT_CONNECTED); + op_result->Assign(1, make_intrusive("Backend is closed")); + return op_result; } auto key_v = IntrusivePtr{NewRef{}, key}; - auto cb = new ValResultCallback(); + auto cb = new OperationResultCallback(); auto get_res = b->backend->Get(key_v, cb); // If the backend only supports async, block until it's ready and then pull the result out of @@ -365,27 +392,28 @@ function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: a delete cb; - if ( ! get_res.has_value() ) { - val_result->Assign(1, make_intrusive( - util::fmt("Failed to retrieve data: %s", get_res.error().c_str()))); - return val_result; - } - - val_result->Assign(0, get_res.value()); - return val_result; + return get_res.BuildVal(); %} -function Storage::Sync::__erase%(backend: opaque of Storage::BackendHandle, key: any%): bool +function Storage::Sync::__erase%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult %{ + static auto op_result_type = id::find_type("Storage::OperationResult"); + auto b = dynamic_cast(backend); if ( ! b ) { - emit_builtin_error("Invalid storage handle", backend); - return val_mgr->Bool(false); + auto op_result = make_intrusive(op_result_type); + op_result->Assign(0, ReturnCode::OPERATION_FAILED); + op_result->Assign(1, make_intrusive("Invalid storage handlle")); + return op_result; + } + else if ( ! b->backend->IsOpen() ) { + auto op_result = make_intrusive(op_result_type); + op_result->Assign(0, ReturnCode::NOT_CONNECTED); + op_result->Assign(1, make_intrusive("Backend is closed")); + return op_result; } - else if ( ! b->backend->IsOpen() ) - return val_mgr->Bool(false); - auto cb = new ErrorResultCallback(); + auto cb = new OperationResultCallback(); auto key_v = IntrusivePtr{NewRef{}, key}; auto erase_res = b->backend->Erase(key_v, cb); @@ -398,10 +426,5 @@ function Storage::Sync::__erase%(backend: opaque of Storage::BackendHandle, key: delete cb; - if ( erase_res.has_value() ) { - emit_builtin_error(util::fmt("Failed to erase data for key: %s", erase_res.value().c_str())); - return val_mgr->Bool(false); - } - - return val_mgr->Bool(true); + return erase_res.BuildVal(); %} diff --git a/testing/btest/Baseline/plugins.storage/output b/testing/btest/Baseline/plugins.storage/output index 0812d557db..0e95ed4a97 100644 --- a/testing/btest/Baseline/plugins.storage/output +++ b/testing/btest/Baseline/plugins.storage/output @@ -1,2 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -results of trying to use closed handle: get: 0, put: 0, erase: 0 +open result, [code=Storage::SUCCESS, error_str=, value=] +results of trying to use closed handle: get: Storage::NOT_CONNECTED, put: Storage::NOT_CONNECTED, erase: Storage::NOT_CONNECTED +open result 2, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error, value=] +close result of closed handle, [code=Storage::NOT_CONNECTED, error_str=Backend is closed, value=] diff --git a/testing/btest/Baseline/plugins.storage/zeek-stderr b/testing/btest/Baseline/plugins.storage/zeek-stderr index 8dcb29da7c..49d861c74c 100644 --- a/testing/btest/Baseline/plugins.storage/zeek-stderr +++ b/testing/btest/Baseline/plugins.storage/zeek-stderr @@ -1,4 +1 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/storage.zeek, line 45: Failed to retrieve data: Failed to find key -error in <...>/sync.zeek, line 75: Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error (Storage::Sync::__open_backend(Storage::Sync::btype, to_any_coerce Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) -error in <...>/sync.zeek, line 80: Invalid storage handle (Storage::Sync::__close_backend(Storage::Sync::backend) and F) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out b/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out index 0b2661d2e3..96bc442e9c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.compound-types/out @@ -1,7 +1,8 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -put result, T -get result, [val={ +open result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value={ [2] = b, [1] = a, [3] = c -}, error=] +}] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out b/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out index 7ca5354b61..5649e321fd 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.erase/out @@ -1,3 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -erase result, T -get result, Failed to retrieve data: Failed to find row for key: no more rows available +open result, [code=Storage::SUCCESS, error_str=, value=] +erase result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::KEY_NOT_FOUND, error_str=, value=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out index 08758e1bf2..c5b4964ce2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out @@ -1,5 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -put result, T -get result, [val=value7890, error=] +open result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value7890] get result same as inserted, T -get result, Failed to retrieve data: Failed to find row for key: no more rows available +get result, [code=Storage::KEY_NOT_FOUND, error_str=, value=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out b/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out index d1338d7ba9..c65062e83a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.overwriting/out @@ -1,4 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -put result, T -get result, [val=value7890, error=] +open result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value7890] get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out index b32c104878..960761de7b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async-reading-pcap/out @@ -1,4 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -put result, T -get result, [val=value5678, error=] +open result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value5678] get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out index b32c104878..3c40bed4be 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-async/out @@ -1,4 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -put result, T -get result, [val=value5678, error=] +open result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value5678] get result same as inserted, T +close result, [code=Storage::SUCCESS, error_str=, value=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-1..stdout index 01c6005755..8fc986fdaf 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-1..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-1..stdout @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -worker-1, put result, T +worker-1, put result, [code=Storage::SUCCESS, error_str=, value=] redis_data_written -worker-1, [val=5678, error=] +worker-1, [code=Storage::SUCCESS, error_str=, value=5678] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-2..stdout index 76ee9cc9df..bd57bb7fe5 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-2..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-cluster/worker-2..stdout @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. redis_data_written -worker-2, [val=5678, error=] +worker-2, [code=Storage::SUCCESS, error_str=, value=5678] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out index 996cf2ebac..e2421b325f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out @@ -1,5 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -put result, T -get result, [val=value7890, error=] +open result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value7890] get result same as inserted, T -get result after expiration, [val=, error=Failed to retrieve data: GET returned key didn't exist] +get result after expiration, [code=Storage::KEY_NOT_FOUND, error_str=, value=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out index b77104dcd8..2f1d82d855 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out @@ -1,7 +1,8 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -put result, T -get result, [val=value1234, error=] +open_result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value1234] get result same as inserted, T -overwrite put result, T -get result, [val=value5678, error=] +overwrite put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value5678] get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out index b32c104878..960761de7b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-reading-pcap/out @@ -1,4 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -put result, T -get result, [val=value5678, error=] +open result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value5678] get result same as inserted, T diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out index 1eca70373d..c9184cdc98 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out @@ -1,6 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -open successful -put result, T -get result, [val=value5678, error=] +open result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value5678] get result same as inserted, T closed succesfully diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr index 1013a7f2f4..49d861c74c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/.stderr @@ -1,3 +1 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/sync.zeek, line 75: Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file (Storage::Sync::__open_backend(Storage::Sync::btype, to_any_coerce Storage::Sync::options, Storage::Sync::key_type, Storage::Sync::val_type)) -error in <...>/sync.zeek, line 85: Failed to store data: type of key passed (count) does not match backend's key type (str) (Storage::Sync::__put(Storage::Sync::backend, Storage::Sync::args$key, Storage::Sync::args$value, Storage::Sync::args$overwrite, Storage::Sync::args$expire_time)) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out index c3ff9662ac..0f0dfe41cb 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out @@ -1,2 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -Put result on closed handle: 0 +Open result, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file, value=] +Open result 2, [code=Storage::SUCCESS, error_str=, value=] +Put result with bad key type, [code=Storage::KEY_TYPE_MISMATCH, error_str=, value=] +Put result on closed handle, [code=Storage::NOT_CONNECTED, error_str=Backend is closed, value=] diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc index 8007131f3d..03e4360553 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -4,12 +4,14 @@ #include "zeek/Func.h" #include "zeek/Val.h" +#include "zeek/storage/ReturnCode.h" + +using namespace zeek; +using namespace zeek::storage; namespace btest::storage::backend { -zeek::storage::BackendPtr StorageDummy::Instantiate(std::string_view tag) { - return zeek::make_intrusive(tag); -} +BackendPtr StorageDummy::Instantiate(std::string_view tag) { return make_intrusive(tag); } /** * Called by the manager system to open the backend. @@ -18,65 +20,65 @@ zeek::storage::BackendPtr StorageDummy::Instantiate(std::string_view tag) { * implementation must call \a Opened(); if not, it must call Error() * with a corresponding message. */ -zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options, zeek::storage::OpenResultCallback* cb) { - zeek::RecordValPtr backend_options = options->GetField("dummy"); - bool open_fail = backend_options->GetField("open_fail")->Get(); +OperationResult StorageDummy::DoOpen(RecordValPtr options, OpenResultCallback* cb) { + RecordValPtr backend_options = options->GetField("dummy"); + bool open_fail = backend_options->GetField("open_fail")->Get(); if ( open_fail ) - return "open_fail was set to true, returning error"; + return {ReturnCode::OPERATION_FAILED, "open_fail was set to true, returning error"}; open = true; - return std::nullopt; + return {ReturnCode::SUCCESS}; } /** * Finalizes the backend when it's being closed. */ -zeek::storage::ErrorResult StorageDummy::DoClose(zeek::storage::ErrorResultCallback* cb) { +OperationResult StorageDummy::DoClose(OperationResultCallback* cb) { open = false; - return std::nullopt; + return {ReturnCode::SUCCESS}; } /** * The workhorse method for Put(). This must be implemented by plugins. */ -zeek::storage::ErrorResult StorageDummy::DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite, - double expiration_time, zeek::storage::ErrorResultCallback* cb) { +OperationResult StorageDummy::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, + OperationResultCallback* cb) { auto json_key = key->ToJSON()->ToStdString(); auto json_value = value->ToJSON()->ToStdString(); data[json_key] = json_value; - return std::nullopt; + return {ReturnCode::SUCCESS}; } /** * The workhorse method for Get(). This must be implemented for plugins. */ -zeek::storage::ValResult StorageDummy::DoGet(zeek::ValPtr key, zeek::storage::ValResultCallback* cb) { +OperationResult StorageDummy::DoGet(ValPtr key, OperationResultCallback* cb) { auto json_key = key->ToJSON(); auto it = data.find(json_key->ToStdString()); if ( it == data.end() ) - return zeek::unexpected("Failed to find key"); + return {ReturnCode::KEY_NOT_FOUND}; - auto val = zeek::detail::ValFromJSON(it->second.c_str(), val_type, zeek::Func::nil); - if ( std::holds_alternative(val) ) { - zeek::ValPtr val_v = std::get(val); - return val_v; + auto val = zeek::detail::ValFromJSON(it->second.c_str(), val_type, Func::nil); + if ( std::holds_alternative(val) ) { + ValPtr val_v = std::get(val); + return {ReturnCode::SUCCESS, "", val_v}; } - return zeek::unexpected(std::get(val)); + return {ReturnCode::OPERATION_FAILED, std::get(val)}; } /** * The workhorse method for Erase(). This must be implemented for plugins. */ -zeek::storage::ErrorResult StorageDummy::DoErase(zeek::ValPtr key, zeek::storage::ErrorResultCallback* cb) { +OperationResult StorageDummy::DoErase(ValPtr key, OperationResultCallback* cb) { auto json_key = key->ToJSON(); auto it = data.find(json_key->ToStdString()); if ( it == data.end() ) - return "Failed to find key"; + return {ReturnCode::KEY_NOT_FOUND}; data.erase(it); - return std::nullopt; + return {ReturnCode::SUCCESS}; } } // namespace btest::storage::backend diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h index 4c8a2dfc29..fbd0abc032 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.h +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -21,13 +21,13 @@ public: /** * Called by the manager system to open the backend. */ - zeek::storage::ErrorResult DoOpen(zeek::RecordValPtr options, - zeek::storage::OpenResultCallback* cb = nullptr) override; + zeek::storage::OperationResult DoOpen(zeek::RecordValPtr options, + zeek::storage::OpenResultCallback* cb = nullptr) override; /** * Finalizes the backend when it's being closed. */ - zeek::storage::ErrorResult DoClose(zeek::storage::ErrorResultCallback* cb = nullptr) override; + zeek::storage::OperationResult DoClose(zeek::storage::OperationResultCallback* cb = nullptr) override; /** * Returns whether the backend is opened. @@ -37,19 +37,21 @@ public: /** * The workhorse method for Put(). */ - zeek::storage::ErrorResult DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite = true, - double expiration_time = 0, - zeek::storage::ErrorResultCallback* cb = nullptr) override; + zeek::storage::OperationResult DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite = true, + double expiration_time = 0, + zeek::storage::OperationResultCallback* cb = nullptr) override; /** * The workhorse method for Get(). */ - zeek::storage::ValResult DoGet(zeek::ValPtr key, zeek::storage::ValResultCallback* cb = nullptr) override; + zeek::storage::OperationResult DoGet(zeek::ValPtr key, + zeek::storage::OperationResultCallback* cb = nullptr) override; /** * The workhorse method for Erase(). */ - zeek::storage::ErrorResult DoErase(zeek::ValPtr key, zeek::storage::ErrorResultCallback* cb = nullptr) override; + zeek::storage::OperationResult DoErase(zeek::ValPtr key, + zeek::storage::OperationResultCallback* cb = nullptr) override; private: std::map data; diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek index fe876ff31a..40948d5ba2 100644 --- a/testing/btest/plugins/storage.zeek +++ b/testing/btest/plugins/storage.zeek @@ -30,10 +30,12 @@ event zeek_init() { # Test basic operation. The second get() should return an error # as the key should have been erased. - local b = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, str, str); + local open_res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, str, str); + print "open result", open_res; + local b = open_res$value; local put_res = Storage::Sync::put(b, [$key=key, $value=value, $overwrite=F]); local get_res = Storage::Sync::get(b, key); - if ( get_res is bool ) { + if ( get_res$code != Storage::SUCCESS ) { print("Got an invalid value in response!"); } @@ -41,19 +43,21 @@ event zeek_init() { get_res = Storage::Sync::get(b, key); Storage::Sync::close_backend(b); - if ( get_res?$error ) - Reporter::error(get_res$error); + if ( get_res$code != Storage::SUCCESS && get_res?$error_str ) + Reporter::error(get_res$error_str); # Test attempting to use the closed handle. put_res = Storage::Sync::put(b, [$key="a", $value="b", $overwrite=F]); get_res = Storage::Sync::get(b, "a"); erase_res = Storage::Sync::erase(b, "a"); - print(fmt("results of trying to use closed handle: get: %d, put: %d, erase: %d", - get_res?$val, put_res, erase_res)); + print(fmt("results of trying to use closed handle: get: %s, put: %s, erase: %s", + get_res$code, put_res$code, erase_res$code)); # Test failing to open the handle and test closing an invalid handle. opts$dummy$open_fail = T; - local b2 = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, str, str); - Storage::Sync::close_backend(b2); + open_res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, str, str); + print "open result 2", open_res; + local close_res = Storage::Sync::close_backend(open_res$value); + print "close result of closed handle", close_res; } diff --git a/testing/btest/scripts/base/frameworks/storage/compound-types.zeek b/testing/btest/scripts/base/frameworks/storage/compound-types.zeek index ba68c4e006..419a1f3fc0 100644 --- a/testing/btest/scripts/base/frameworks/storage/compound-types.zeek +++ b/testing/btest/scripts/base/frameworks/storage/compound-types.zeek @@ -63,7 +63,9 @@ event zeek_init() { value[2] = "b"; value[3] = "c"; - local b = Storage::Sync::open_backend(Storage::SQLITE, opts, Rec, tbl); + local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, Rec, tbl); + print "open result", open_res; + local b = open_res$value; local res = Storage::Sync::put(b, [$key=key, $value=value]); print "put result", res; diff --git a/testing/btest/scripts/base/frameworks/storage/erase.zeek b/testing/btest/scripts/base/frameworks/storage/erase.zeek index 585da798b2..7380695b91 100644 --- a/testing/btest/scripts/base/frameworks/storage/erase.zeek +++ b/testing/btest/scripts/base/frameworks/storage/erase.zeek @@ -10,23 +10,26 @@ # Create a typename here that can be passed down into get(). type str: string; -event zeek_init() { +event zeek_init() + { # Create a database file in the .tmp directory with a 'testing' table - local opts : Storage::BackendOptions; - opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; + local opts: Storage::BackendOptions; + opts$sqlite = [ $database_path="storage-test.sqlite", $table_name="testing" ]; local key = "key1234"; # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. - local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + print "open result", open_res; + local b = open_res$value; local res = Storage::Sync::erase(b, key); print "erase result", res; local res2 = Storage::Sync::get(b, key); - if ( res2?$error ) - print "get result", res2$error; + if ( res2$code != Storage::SUCCESS ) + print "get result", res2; Storage::Sync::close_backend(b); -} + } diff --git a/testing/btest/scripts/base/frameworks/storage/expiration.zeek b/testing/btest/scripts/base/frameworks/storage/expiration.zeek index d53e23fa36..fc0efd6819 100644 --- a/testing/btest/scripts/base/frameworks/storage/expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/expiration.zeek @@ -20,8 +20,8 @@ event check_removed() { # This should return an error from the sqlite backend that there aren't any more # rows available. local res2 = Storage::Sync::get(backend, key); - if ( res2?$error ) - print "get result", res2$error; + if ( res2$code != Storage::SUCCESS ) + print "get result", res2; Storage::Sync::close_backend(backend); terminate(); @@ -31,15 +31,17 @@ event setup_test() { local opts : Storage::BackendOptions; opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; - backend = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + print "open result", open_res; + backend = open_res$value; local res = Storage::Sync::put(backend, [$key=key, $value=value, $expire_time=2 secs]); print "put result", res; local res2 = Storage::Sync::get(backend, key); print "get result", res2; - if ( res2?$val ) - print "get result same as inserted", value == (res2$val as string); + if ( res2$code == Storage::SUCCESS && res2?$value ) + print "get result same as inserted", value == (res2$value as string); schedule 5 secs { check_removed() }; } diff --git a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek index a31360802f..4a2f7be53b 100644 --- a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek +++ b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek @@ -17,15 +17,17 @@ event zeek_init() { local key = "key1234"; local value = "value7890"; - local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + print "open result", open_res; + local b = open_res$value; local res = Storage::Sync::put(b, [$key=key, $value=value]); print "put result", res; local res2 = Storage::Sync::get(b, key); print "get result", res2; - if ( res2?$val ) - print "get result same as inserted", value == (res2$val as string); + if ( res2$code == Storage::SUCCESS && res2?$value ) + print "get result same as inserted", value == (res2$value as string); Storage::Sync::close_backend(b); } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek index 2fe19ddb37..fba38d3e59 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -28,7 +28,9 @@ event zeek_init() local key = "key1234"; local value = "value5678"; - local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + print "open result", open_res; + local b = open_res$value; when [b, key, value] ( local res = Storage::Async::put(b, [ $key=key, $value=value ]) ) @@ -38,8 +40,8 @@ event zeek_init() when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) { print "get result", res2; - if ( res2?$val ) - print "get result same as inserted", value == ( res2$val as string ); + if ( res2$code == Storage::SUCCESS && res2?$value ) + print "get result same as inserted", value == ( res2$value as string ); Storage::Sync::close_backend(b); } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek index 225da69719..7923aa6f46 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek @@ -30,32 +30,49 @@ event zeek_init() local key = "key1234"; local value = "value5678"; - local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); - - when [b, key, value] ( local res = Storage::Async::put(b, [ $key=key, - $value=value ]) ) + when [opts, key, value] ( local open_res = Storage::Async::open_backend( + Storage::REDIS, opts, str, str) ) { - print "put result", res; + print "open result", open_res; + local b = open_res$value; - when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) + when [b, key, value] ( local put_res = Storage::Async::put(b, [ $key=key, + $value=value ]) ) { - print "get result", res2; - if ( res2?$val ) - print "get result same as inserted", value == ( res2$val as string ); + print "put result", put_res; - Storage::Sync::close_backend(b); + when [b, key, value] ( local get_res = Storage::Async::get(b, key) ) + { + print "get result", get_res; + if ( get_res$code == Storage::SUCCESS && get_res?$value ) + print "get result same as inserted", value == ( get_res$value as string ); - terminate(); + when [b] ( local close_res = Storage::Async::close_backend(b) ) + { + print "close result", close_res; + terminate(); + } + timeout 5sec + { + print "close request timed out"; + terminate(); + } + } + timeout 5sec + { + print "get request timed out"; + terminate(); + } } timeout 5sec { - print "get request timed out"; + print "put request timed out"; terminate(); } } timeout 5sec { - print "put request timed out"; + print "open request timed out"; terminate(); } } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek index 8d6ede1d90..7c4996f7f7 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek @@ -45,7 +45,8 @@ event zeek_init() opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( "REDIS_PORT")), $key_prefix="testing" ]; - backend = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + backend = open_res$value; } event redis_data_written() diff --git a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek index fcc6d03439..59ff5b540c 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek @@ -40,15 +40,18 @@ event setup_test() opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( "REDIS_PORT")), $key_prefix="testing" ]; - b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + print "open result", open_res; + + b = open_res$value; local res = Storage::Sync::put(b, [ $key=key, $value=value, $expire_time=2secs ]); print "put result", res; local res2 = Storage::Sync::get(b, key); print "get result", res2; - if ( res2?$val ) - print "get result same as inserted", value == ( res2$val as string ); + if ( res2$code == Storage::SUCCESS && res2?$value ) + print "get result same as inserted", value == ( res2$value as string ); schedule 5secs { check_removed() }; } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek index bf49a30230..42f2b8d0f5 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -27,15 +27,18 @@ event zeek_init() local key = "key1234"; local value = "value1234"; - local b = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + print "open_result", open_res; + + local b = open_res$value; local res = Storage::Sync::put(b, [ $key=key, $value=value ]); print "put result", res; local res2 = Storage::Sync::get(b, key); print "get result", res2; - if ( res2?$val ) - print "get result same as inserted", value == ( res2$val as string ); + if ( res2$code == Storage::SUCCESS && res2?$value ) + print "get result same as inserted", value == ( res2$value as string ); local value2 = "value5678"; res = Storage::Sync::put(b, [ $key=key, $value=value2, $overwrite=T ]); @@ -43,8 +46,8 @@ event zeek_init() res2 = Storage::Sync::get(b, key); print "get result", res2; - if ( res2?$val ) - print "get result same as inserted", value2 == ( res2$val as string ); + if ( res2$code == Storage::SUCCESS && res2?$value ) + print "get result same as inserted", value2 == ( res2$value as string ); Storage::Sync::close_backend(b); } diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek index 8a933de3c6..b5c0ec8adf 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek @@ -23,7 +23,10 @@ event zeek_init() # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. - local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + print "open result", open_res; + + local b = open_res$value; when [b, key, value] ( local res = Storage::Async::put(b, [ $key=key, $value=value ]) ) @@ -33,8 +36,8 @@ event zeek_init() when [b, key, value] ( local res2 = Storage::Async::get(b, key) ) { print "get result", res2; - if ( res2?$val ) - print "get result same as inserted", value == ( res2$val as string ); + if ( res2$code == Storage::SUCCESS && res2?$value ) + print "get result same as inserted", value == ( res2$value as string ); Storage::Sync::close_backend(b); diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index c815996f7c..92f2a34059 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -22,10 +22,11 @@ event zeek_init() # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. - when [opts, key, value] ( local b = Storage::Async::open_backend( + when [opts, key, value] ( local open_res = Storage::Async::open_backend( Storage::SQLITE, opts, str, str) ) { - print "open successful"; + print "open result", open_res; + local b = open_res$value; when [b, key, value] ( local put_res = Storage::Async::put(b, [ $key=key, $value=value ]) ) @@ -35,8 +36,8 @@ event zeek_init() when [b, key, value] ( local get_res = Storage::Async::get(b, key) ) { print "get result", get_res; - if ( get_res?$val ) - print "get result same as inserted", value == ( get_res$val as string ); + if ( get_res$code == Storage::SUCCESS && get_res?$value ) + print "get result same as inserted", value == ( get_res$value as string ); when [b] ( local close_res = Storage::Async::close_backend(b) ) { diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek index 899ba497aa..24c1ed40d0 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek @@ -17,18 +17,23 @@ event zeek_init() { $table_name = "testing"]; # This should report an error in .stderr and reporter.log - local b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + print "Open result", open_res; # Open a valid database file opts$sqlite$database_path = "test.sqlite"; - b = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + print "Open result 2", open_res; + + local b = open_res$value; local bad_key: count = 12345; local value = "abcde"; - Storage::Sync::put(b, [$key=bad_key, $value=value]); + local res = Storage::Sync::put(b, [$key=bad_key, $value=value]); + print "Put result with bad key type", res; # Close the backend and then attempt to use the closed handle Storage::Sync::close_backend(b); - local res = Storage::Sync::put(b, [$key="a", $value="b"]); - print fmt("Put result on closed handle: %d", res); + local res2 = Storage::Sync::put(b, [$key="a", $value="b"]); + print "Put result on closed handle", res2; } From b81e876ec8e324da52a87402cbc812ad2d3e2b26 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 27 Feb 2025 10:35:31 -0700 Subject: [PATCH 33/52] Change how redis-server is run during btests, removing redis.conf --- testing/btest/Files/redis.conf | 73 ------------------- .../storage/redis-async-reading-pcap.zeek | 5 +- .../base/frameworks/storage/redis-async.zeek | 5 +- .../frameworks/storage/redis-cluster.zeek | 6 +- .../frameworks/storage/redis-expiration.zeek | 5 +- .../base/frameworks/storage/redis-sync.zeek | 5 +- testing/scripts/run-redis-server | 21 ++++++ 7 files changed, 26 insertions(+), 94 deletions(-) delete mode 100644 testing/btest/Files/redis.conf create mode 100755 testing/scripts/run-redis-server diff --git a/testing/btest/Files/redis.conf b/testing/btest/Files/redis.conf deleted file mode 100644 index 875c46f9f7..0000000000 --- a/testing/btest/Files/redis.conf +++ /dev/null @@ -1,73 +0,0 @@ -bind 127.0.0.1 -port %REDIS_PORT% -pidfile %RUN_PATH%/redis.pid -loglevel verbose -logfile %RUN_PATH%/redis.log -dir %RUN_PATH% -daemonize no -databases 1 - -# All of the values from here down are the default values in the sample config file -# that comes with redis-server v7.2.7. -protected-mode yes -tcp-backlog 511 -timeout 0 -always-show-logo no -set-proc-title yes -proc-title-template "{title} {listen-addr} {server-mode}" -stop-writes-on-bgsave-error yes -rdbcompression yes -rdbchecksum yes -dbfilename dump.rdb -rdb-del-sync-files no -replica-serve-stale-data yes -replica-read-only yes -repl-diskless-sync yes -repl-diskless-sync-delay 5 -repl-diskless-sync-max-replicas 0 -repl-diskless-load disabled -repl-disable-tcp-nodelay no -replica-priority 100 -acllog-max-len 128 -lazyfree-lazy-eviction no -lazyfree-lazy-expire no -lazyfree-lazy-server-del no -replica-lazy-flush no -lazyfree-lazy-user-del no -lazyfree-lazy-user-flush no -oom-score-adj no -oom-score-adj-values 0 200 800 -disable-thp yes -appendonly no -appendfilename "appendonly.aof" -appenddirname "appendonlydir" -appendfsync everysec -no-appendfsync-on-rewrite no -auto-aof-rewrite-percentage 100 -auto-aof-rewrite-min-size 64mb -aof-load-truncated yes -aof-use-rdb-preamble yes -aof-timestamp-enabled no -slowlog-log-slower-than 10000 -slowlog-max-len 128 -latency-monitor-threshold 0 -notify-keyspace-events "" -hash-max-listpack-entries 512 -hash-max-listpack-value 64 -list-max-listpack-size -2 -list-compress-depth 0 -set-max-intset-entries 512 -zset-max-listpack-entries 128 -zset-max-listpack-value 64 -hll-sparse-max-bytes 3000 -stream-node-max-bytes 4096 -stream-node-max-entries 100 -activerehashing yes -client-output-buffer-limit normal 0 0 0 -client-output-buffer-limit replica 256mb 64mb 60 -client-output-buffer-limit pubsub 32mb 8mb 60 -hz 10 -dynamic-hz yes -aof-rewrite-incremental-fsync yes -rdb-save-incremental-fsync yes -jemalloc-bg-thread yes \ No newline at end of file diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek index fba38d3e59..c79e2383bf 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -3,10 +3,7 @@ # @TEST-REQUIRES: have-redis # @TEST-PORT: REDIS_PORT -# Generate a redis.conf file with the port defined above, but without the /tcp at the end of -# it. This also sets some paths in the conf to the testing directory. -# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf -# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: btest-bg-run redis-server run-redis-server ${REDIS_PORT%/tcp} # @TEST-EXEC: zeek -r $TRACES/http/get.trace -b %INPUT > out # @TEST-EXEC: btest-bg-wait -k 0 diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek index 7923aa6f46..c58596eff3 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek @@ -3,10 +3,7 @@ # @TEST-REQUIRES: have-redis # @TEST-PORT: REDIS_PORT -# Generate a redis.conf file with the port defined above, but without the /tcp at the end of -# it. This also sets some paths in the conf to the testing directory. -# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf -# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: btest-bg-run redis-server run-redis-server ${REDIS_PORT%/tcp} # @TEST-EXEC: zeek -b %INPUT > out # @TEST-EXEC: btest-bg-wait -k 0 diff --git a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek index 7c4996f7f7..ecc0476066 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek @@ -6,11 +6,7 @@ # @TEST-PORT: BROKER_PORT2 # @TEST-PORT: BROKER_PORT3 -# Generate a redis.conf file with the port defined above, but without the /tcp at the end of -# it. This also sets some paths in the conf to the testing directory. -# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf -# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf - +# @TEST-EXEC: btest-bg-run redis-server run-redis-server ${REDIS_PORT%/tcp} # @TEST-EXEC: btest-bg-run manager-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b %INPUT # @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT # @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b %INPUT diff --git a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek index 59ff5b540c..c7223c4277 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek @@ -3,10 +3,7 @@ # @TEST-REQUIRES: have-redis # @TEST-PORT: REDIS_PORT -# Generate a redis.conf file with the port defined above, but without the /tcp at the end of -# it. This also sets some paths in the conf to the testing directory. -# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf -# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: btest-bg-run redis-server run-redis-server ${REDIS_PORT%/tcp} # @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -B storage -b -Cr - %INPUT > out # @TEST-EXEC: btest-bg-wait -k 1 diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek index 42f2b8d0f5..f3d5d4b684 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -3,10 +3,7 @@ # @TEST-REQUIRES: have-redis # @TEST-PORT: REDIS_PORT -# Generate a redis.conf file with the port defined above, but without the /tcp at the end of -# it. This also sets some paths in the conf to the testing directory. -# @TEST-EXEC: cat $FILES/redis.conf | sed "s|%REDIS_PORT%|${REDIS_PORT%/tcp}|g" | sed "s|%RUN_PATH%|$(pwd)|g" > ./redis.conf -# @TEST-EXEC: btest-bg-run redis redis-server ../redis.conf +# @TEST-EXEC: btest-bg-run redis-server run-redis-server ${REDIS_PORT%/tcp} # @TEST-EXEC: zeek -b %INPUT > out # @TEST-EXEC: btest-bg-wait -k 0 diff --git a/testing/scripts/run-redis-server b/testing/scripts/run-redis-server new file mode 100755 index 0000000000..ae11c36b13 --- /dev/null +++ b/testing/scripts/run-redis-server @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -eux + +if ! redis-server --version; then + exit 1 +fi + +if [ $# -ne 1 ]; then + echo "Usage $0 " >2 + exit 1 +fi + +listen_port=$1 + +exec redis-server \ + --bind 127.0.0.1 \ + --port ${listen_port} \ + --loglevel verbose \ + --logfile redis.log \ + --pidfile redis.pid \ + --databases 1 From cca1d4f98886b0dc447b9d73ef6572d2ec54d7c0 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 27 Feb 2025 16:39:41 -0700 Subject: [PATCH 34/52] Redis: Fix thread-contention issues with Expire(), add more tests --- src/storage/backend/redis/Redis.cc | 94 ++++++++++++++----- src/storage/backend/redis/Redis.h | 9 +- src/storage/storage.bif | 3 + .../out | 10 +- .../out | 10 ++ .../frameworks/storage/redis-expiration.zeek | 36 +++++-- .../storage/redis-native-expiration.zeek | 78 +++++++++++++++ 7 files changed, 200 insertions(+), 40 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-native-expiration/out create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-native-expiration.zeek diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index b1fb29ed03..e5e5312987 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -16,6 +16,8 @@ // Anonymous callback handler methods for the hiredis async API. namespace { +bool during_expire = false; + class Tracer { public: Tracer(const std::string& where) : where(where) {} // DBG_LOG(zeek::DBG_STORAGE, "%s", where.c_str()); } @@ -56,17 +58,21 @@ void redisErase(redisAsyncContext* ctx, void* reply, void* privdata) { backend->HandleEraseResult(static_cast(reply), callback); } -void redisZRANGEBYSCORE(redisAsyncContext* ctx, void* reply, void* privdata) { - auto t = Tracer("zrangebyscore"); +void redisZADD(redisAsyncContext* ctx, void* reply, void* privdata) { + auto t = Tracer("generic"); auto backend = static_cast(ctx->data); - backend->HandleZRANGEBYSCORE(static_cast(reply)); + + // We don't care about the reply from the ZADD, m1ostly because blocking to poll + // for it adds a bunch of complication to DoPut() with having to handle the + // reply from SET first. + backend->HandleGeneric(nullptr); + freeReplyObject(reply); } void redisGeneric(redisAsyncContext* ctx, void* reply, void* privdata) { auto t = Tracer("generic"); auto backend = static_cast(ctx->data); - backend->HandleGeneric(); - freeReplyObject(reply); + backend->HandleGeneric(static_cast(reply)); } // Because we called redisPollAttach in DoOpen(), privdata here is a @@ -74,12 +80,17 @@ void redisGeneric(redisAsyncContext* ctx, void* reply, void* privdata) { // data, which contains the backend. Because we overrode these callbacks in // DoOpen, we still want to mimic their callbacks to redisPollTick functions // correctly. +// +// Additionally, if we're in the middle of running a manual Expire() because +// we're reading a pcap, don't add the file descriptor into iosource_mgr. Manual +// calls to Poll() during that will handle reading/writing any data, and we +// don't want the contention with the main loop. void redisAddRead(void* privdata) { auto t = Tracer("addread"); auto rpe = static_cast(privdata); auto backend = static_cast(rpe->context->data); - if ( rpe->reading == 0 ) + if ( rpe->reading == 0 && ! during_expire ) zeek::iosource_mgr->RegisterFd(rpe->fd, backend, zeek::iosource::IOSource::READ); rpe->reading = 1; } @@ -89,7 +100,7 @@ void redisDelRead(void* privdata) { auto rpe = static_cast(privdata); auto backend = static_cast(rpe->context->data); - if ( rpe->reading == 1 ) + if ( rpe->reading == 1 && ! during_expire ) zeek::iosource_mgr->UnregisterFd(rpe->fd, backend, zeek::iosource::IOSource::READ); rpe->reading = 0; } @@ -99,7 +110,7 @@ void redisAddWrite(void* privdata) { auto rpe = static_cast(privdata); auto backend = static_cast(rpe->context->data); - if ( rpe->writing == 0 ) + if ( rpe->writing == 0 && ! during_expire ) zeek::iosource_mgr->RegisterFd(rpe->fd, backend, zeek::iosource::IOSource::WRITE); rpe->writing = 1; } @@ -109,11 +120,21 @@ void redisDelWrite(void* privdata) { auto t = Tracer("delwrite"); auto backend = static_cast(rpe->context->data); - if ( rpe->writing == 1 ) + if ( rpe->writing == 1 && ! during_expire ) zeek::iosource_mgr->UnregisterFd(rpe->fd, backend, zeek::iosource::IOSource::WRITE); rpe->writing = 0; } +// Creates a unique_lock based on a condition against a mutex. This is used to +// conditionally lock the expire_mutex. We only need to do it while reading +// pcaps. The only thread contention happens during Expire(), which only happens +// when reading pcaps. It's not worth the cycles to lock the mutex otherwise, +// and hiredis will deal with other cross-command contention correctly as long +// as it's in a single thread. +std::unique_lock conditionally_lock(bool condition, std::mutex& mutex) { + return condition ? std::unique_lock(mutex) : std::unique_lock(); +} + } // namespace namespace zeek::storage::backend::redis { @@ -213,6 +234,8 @@ OperationResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { * Finalizes the backend when it's being closed. */ OperationResult Redis::DoClose(OperationResultCallback* cb) { + auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); + connected = false; redisAsyncDisconnect(async_ctx); @@ -240,6 +263,8 @@ OperationResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double ex if ( ! connected && ! async_ctx ) return {ReturnCode::NOT_CONNECTED}; + auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); + std::string format = "SET %s:%s %s"; if ( ! overwrite ) format.append(" NX"); @@ -251,9 +276,9 @@ OperationResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double ex // Use built-in expiration if reading live data, since time will move // forward consistently. If reading pcaps, we'll do something else. if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) { - format.append(" PXAT %d"); + format.append(" PXAT %" PRIu64); status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), - json_value.data(), static_cast(expiration_time * 1e6)); + json_value.data(), static_cast(expiration_time * 1e3)); } else status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), @@ -272,7 +297,7 @@ OperationResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double ex format.append(" NX"); format += " %f %s"; - status = redisAsyncCommand(async_ctx, redisGeneric, NULL, format.c_str(), key_prefix.data(), expiration_time, + status = redisAsyncCommand(async_ctx, redisZADD, NULL, format.c_str(), key_prefix.data(), expiration_time, json_key.data()); if ( connected && status == REDIS_ERR ) return {ReturnCode::OPERATION_FAILED, util::fmt("ZADD operation failed: %s", async_ctx->errstr)}; @@ -291,6 +316,8 @@ OperationResult Redis::DoGet(ValPtr key, OperationResultCallback* cb) { if ( ! connected && ! async_ctx ) return {ReturnCode::NOT_CONNECTED}; + auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); + int status = redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%s", key_prefix.data(), key->ToJSON()->ToStdStringView().data()); @@ -312,6 +339,8 @@ OperationResult Redis::DoErase(ValPtr key, OperationResultCallback* cb) { if ( ! connected && ! async_ctx ) return {ReturnCode::NOT_CONNECTED}; + auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); + int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%s", key_prefix.data(), key->ToJSON()->ToStdStringView().data()); @@ -328,12 +357,17 @@ void Redis::Expire() { if ( ! connected || ! zeek::run_state::reading_traces ) return; - int status = redisAsyncCommand(async_ctx, redisZRANGEBYSCORE, NULL, "ZRANGEBYSCORE %s_expire -inf %f", - key_prefix.data(), run_state::network_time); + auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); + + during_expire = true; + + int status = redisAsyncCommand(async_ctx, redisGeneric, NULL, "ZRANGEBYSCORE %s_expire -inf %f", key_prefix.data(), + run_state::network_time); if ( status == REDIS_ERR ) { // TODO: do something with the error? printf("ZRANGEBYSCORE command failed: %s\n", async_ctx->errstr); + during_expire = false; return; } @@ -347,22 +381,29 @@ void Redis::Expire() { if ( reply->elements == 0 ) { freeReplyObject(reply); + during_expire = false; return; } - // The data from the reply to ZRANGEBYSCORE gets deleted as part of the - // commands below so we don't need to free it manually. Doing so results in - // a double-free. + std::vector elements; + for ( size_t i = 0; i < reply->elements; i++ ) + elements.emplace_back(reply->element[i]->str); + + freeReplyObject(reply); // TODO: it's possible to pass multiple keys to a DEL operation but it requires // building an array of the strings, building up the DEL command with entries, // and passing the array as a block somehow. There's no guarantee it'd be faster // anyways. - for ( size_t i = 0; i < reply->elements; i++ ) { - status = - redisAsyncCommand(async_ctx, redisGeneric, NULL, "DEL %s:%s", key_prefix.data(), reply->element[i]->str); + for ( const auto& e : elements ) { + status = redisAsyncCommand(async_ctx, redisGeneric, NULL, "DEL %s:%s", key_prefix.data(), e.c_str()); ++active_ops; Poll(); + + redisReply* reply = reply_queue.front(); + reply_queue.pop_front(); + freeReplyObject(reply); + // TODO: do we care if this failed? } // Remove all of the elements from the range-set that match the time range. @@ -371,6 +412,11 @@ void Redis::Expire() { ++active_ops; Poll(); + + reply = reply_queue.front(); + reply_queue.pop_front(); + freeReplyObject(reply); + // TODO: do we care if this failed? } void Redis::HandlePutResult(redisReply* reply, OperationResultCallback* callback) { @@ -421,9 +467,11 @@ void Redis::HandleEraseResult(redisReply* reply, OperationResultCallback* callba } } -void Redis::HandleZRANGEBYSCORE(redisReply* reply) { +void Redis::HandleGeneric(redisReply* reply) { --active_ops; - reply_queue.push_back(reply); + + if ( reply ) + reply_queue.push_back(reply); } void Redis::OnConnect(int status) { @@ -459,6 +507,8 @@ void Redis::OnDisconnect(int status) { } void Redis::ProcessFd(int fd, int flags) { + auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); + if ( (flags & IOSource::ProcessFlags::READ) != 0 ) redisAsyncHandleRead(async_ctx); if ( (flags & IOSource::ProcessFlags::WRITE) != 0 ) diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index b5a65d12d2..7bdfce9285 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -2,6 +2,8 @@ #pragma once +#include + #include "zeek/iosource/IOSource.h" #include "zeek/storage/Backend.h" @@ -75,11 +77,7 @@ public: void HandlePutResult(redisReply* reply, OperationResultCallback* callback); void HandleGetResult(redisReply* reply, OperationResultCallback* callback); void HandleEraseResult(redisReply* reply, OperationResultCallback* callback); - void HandleZRANGEBYSCORE(redisReply* reply); - - // HandleGeneric exists so that async-running-as-sync operations can remove - // themselves from the list of active operations. - void HandleGeneric() { --active_ops; } + void HandleGeneric(redisReply* reply); protected: void Poll() override; @@ -95,6 +93,7 @@ private: std::deque reply_queue; OpenResultCallback* open_cb; + std::mutex expire_mutex; std::string server_addr; std::string key_prefix; diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 013b95e8be..c7c55ef2e4 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -344,6 +344,9 @@ function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: a return op_result; } + if ( expire_time > 0.0 ) + expire_time += run_state::network_time; + auto cb = new OperationResultCallback(); auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out index e2421b325f..428580958a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-expiration/out @@ -1,6 +1,10 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. open result, [code=Storage::SUCCESS, error_str=, value=] -put result, [code=Storage::SUCCESS, error_str=, value=] -get result, [code=Storage::SUCCESS, error_str=, value=value7890] +put result 1, [code=Storage::SUCCESS, error_str=, value=] +put result 2, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value1234] get result same as inserted, T -get result after expiration, [code=Storage::KEY_NOT_FOUND, error_str=, value=] +get result 2, [code=Storage::SUCCESS, error_str=, value=value2345] +get result 2 same as inserted, T +get result 1 after expiration, [code=Storage::KEY_NOT_FOUND, error_str=, value=] +get result 2 after expiration, [code=Storage::SUCCESS, error_str=, value=value2345] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-native-expiration/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-native-expiration/out new file mode 100644 index 0000000000..428580958a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-native-expiration/out @@ -0,0 +1,10 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +open result, [code=Storage::SUCCESS, error_str=, value=] +put result 1, [code=Storage::SUCCESS, error_str=, value=] +put result 2, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value1234] +get result same as inserted, T +get result 2, [code=Storage::SUCCESS, error_str=, value=value2345] +get result 2 same as inserted, T +get result 1 after expiration, [code=Storage::KEY_NOT_FOUND, error_str=, value=] +get result 2 after expiration, [code=Storage::SUCCESS, error_str=, value=value2345] diff --git a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek index c7223c4277..3d0ae7042c 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek @@ -19,13 +19,19 @@ redef exit_only_after_terminate = T; type str: string; global b: opaque of Storage::BackendHandle; -global key: string = "key1234"; -global value: string = "value7890"; +global key1: string = "key1234"; +global value1: string = "value1234"; + +global key2: string = "key2345"; +global value2: string = "value2345"; event check_removed() { - local res2 = Storage::Sync::get(b, key); - print "get result after expiration", res2; + local res = Storage::Sync::get(b, key1); + print "get result 1 after expiration", res; + + res = Storage::Sync::get(b, key2); + print "get result 2 after expiration", res; Storage::Sync::close_backend(b); terminate(); @@ -42,13 +48,23 @@ event setup_test() b = open_res$value; - local res = Storage::Sync::put(b, [ $key=key, $value=value, $expire_time=2secs ]); - print "put result", res; + # Insert a key that will expire in the time allotted + local res = Storage::Sync::put(b, [ $key=key1, $value=value1, $expire_time=2secs ]); + print "put result 1", res; - local res2 = Storage::Sync::get(b, key); - print "get result", res2; - if ( res2$code == Storage::SUCCESS && res2?$value ) - print "get result same as inserted", value == ( res2$value as string ); + # Insert a key that won't expire + res = Storage::Sync::put(b, [ $key=key2, $value=value2, $expire_time=20secs ]); + print "put result 2", res; + + res = Storage::Sync::get(b, key1); + print "get result", res; + if ( res$code == Storage::SUCCESS && res?$value ) + print "get result same as inserted", value1 == ( res$value as string ); + + res = Storage::Sync::get(b, key2); + print "get result 2", res; + if ( res$code == Storage::SUCCESS && res?$value ) + print "get result 2 same as inserted", value2 == ( res$value as string ); schedule 5secs { check_removed() }; } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-native-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-native-expiration.zeek new file mode 100644 index 0000000000..bbcf58a13d --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-native-expiration.zeek @@ -0,0 +1,78 @@ +# @TEST-DOC: Tests expiration of data from Redis when reading a pcap + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT + +# @TEST-EXEC: btest-bg-run redis-server run-redis-server ${REDIS_PORT%/tcp} +# @TEST-EXEC: zeek -B storage -b %INPUT > out +# @TEST-EXEC: btest-bg-wait -k 1 + +# @TEST-EXEC: btest-diff out + +@load base/frameworks/storage/sync +@load policy/frameworks/storage/backend/redis + +redef Storage::expire_interval = 2secs; +redef exit_only_after_terminate = T; + +# Create a typename here that can be passed down into open_backend() +type str: string; + +global b: opaque of Storage::BackendHandle; +global key1: string = "key1234"; +global value1: string = "value1234"; + +global key2: string = "key2345"; +global value2: string = "value2345"; + +event check_removed() + { + local res = Storage::Sync::get(b, key1); + print "get result 1 after expiration", res; + + res = Storage::Sync::get(b, key2); + print "get result 2 after expiration", res; + + Storage::Sync::close_backend(b); + terminate(); + } + +event setup_test() + { + local opts: Storage::BackendOptions; + opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( + "REDIS_PORT")), $key_prefix="testing" ]; + + local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + print "open result", open_res; + + b = open_res$value; + + # Insert a key that will expire in the time allotted + local res = Storage::Sync::put(b, [ $key=key1, $value=value1, $expire_time=2secs ]); + print "put result 1", res; + + # Insert a key that won't expire + res = Storage::Sync::put(b, [ $key=key2, $value=value2, $expire_time=20secs ]); + print "put result 2", res; + + res = Storage::Sync::get(b, key1); + print "get result", res; + if ( res$code == Storage::SUCCESS && res?$value ) + print "get result same as inserted", value1 == ( res$value as string ); + + res = Storage::Sync::get(b, key2); + print "get result 2", res; + if ( res$code == Storage::SUCCESS && res?$value ) + print "get result 2 same as inserted", value2 == ( res$value as string ); + + schedule 5secs { check_removed() }; + } + +event zeek_init() + { + # We need network time to be set to something other than zero for the + # expiration time to be set correctly. Schedule an event on a short + # timer so packets start getting read and do the setup there. + schedule 100msecs { setup_test() }; + } From ac4aef2d94b752d87ff6f49a76036bb0ea48945f Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 28 Feb 2025 13:40:03 -0700 Subject: [PATCH 35/52] SQLite: Handle other return values from sqlite3_step --- src/storage/backend/sqlite/SQLite.cc | 97 ++++++++++++++++------------ src/storage/backend/sqlite/SQLite.h | 3 +- 2 files changed, 57 insertions(+), 43 deletions(-) diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index ddac358994..50bfe52573 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -35,7 +35,7 @@ OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { table_name = backend_options->GetField("table_name")->ToStdString(); if ( auto open_res = - checkError(sqlite3_open_v2(full_path.c_str(), &db, + CheckError(sqlite3_open_v2(full_path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, NULL)); open_res.code != ReturnCode::SUCCESS ) { sqlite3_close_v2(db); @@ -91,7 +91,7 @@ OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { for ( const auto& [key, stmt] : statements ) { sqlite3_stmt* ps; - if ( auto prep_res = checkError(sqlite3_prepare_v2(db, stmt.c_str(), stmt.size(), &ps, NULL)); + if ( auto prep_res = CheckError(sqlite3_prepare_v2(db, stmt.c_str(), stmt.size(), &ps, NULL)); prep_res.code != ReturnCode::SUCCESS ) { Close(); return prep_res; @@ -100,6 +100,8 @@ OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { prepared_stmts.insert({key, ps}); } + sqlite3_busy_timeout(db, 5000); + return {ReturnCode::SUCCESS}; } @@ -152,40 +154,33 @@ OperationResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double e stmt = prepared_stmts["put_update"]; auto key_str = json_key->ToStdStringView(); - if ( auto res = checkError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); + if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } auto value_str = json_value->ToStdStringView(); - if ( auto res = checkError(sqlite3_bind_text(stmt, 2, value_str.data(), value_str.size(), SQLITE_STATIC)); + if ( auto res = CheckError(sqlite3_bind_text(stmt, 2, value_str.data(), value_str.size(), SQLITE_STATIC)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } - if ( auto res = checkError(sqlite3_bind_double(stmt, 3, expiration_time)); res.code != ReturnCode::SUCCESS ) { + if ( auto res = CheckError(sqlite3_bind_double(stmt, 3, expiration_time)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } if ( overwrite ) { - if ( auto res = checkError(sqlite3_bind_text(stmt, 4, value_str.data(), value_str.size(), SQLITE_STATIC)); + if ( auto res = CheckError(sqlite3_bind_text(stmt, 4, value_str.data(), value_str.size(), SQLITE_STATIC)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } } - if ( auto res = checkError(sqlite3_step(stmt)); res.code != ReturnCode::SUCCESS ) { - sqlite3_reset(stmt); - return res; - } - - sqlite3_reset(stmt); - - return {ReturnCode::SUCCESS}; + return Step(stmt, false); } /** @@ -199,28 +194,13 @@ OperationResult SQLite::DoGet(ValPtr key, OperationResultCallback* cb) { auto stmt = prepared_stmts["get"]; auto key_str = json_key->ToStdStringView(); - if ( auto res = checkError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); + if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } - int errorcode = sqlite3_step(stmt); - if ( errorcode == SQLITE_ROW ) { - // Column 1 is the value - const char* text = (const char*)sqlite3_column_text(stmt, 0); - auto val = zeek::detail::ValFromJSON(text, val_type, Func::nil); - sqlite3_reset(stmt); - if ( std::holds_alternative(val) ) { - ValPtr val_v = std::get(val); - return {ReturnCode::SUCCESS, "", val_v}; - } - else { - return {ReturnCode::OPERATION_FAILED, std::get(val)}; - } - } - - return {ReturnCode::KEY_NOT_FOUND}; + return Step(stmt, true); } /** @@ -234,17 +214,13 @@ OperationResult SQLite::DoErase(ValPtr key, OperationResultCallback* cb) { auto stmt = prepared_stmts["erase"]; auto key_str = json_key->ToStdStringView(); - if ( auto res = checkError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); + if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } - if ( auto res = checkError(sqlite3_step(stmt)); res.code != ReturnCode::SUCCESS ) { - return res; - } - - return {ReturnCode::SUCCESS}; + return Step(stmt, false); } /** @@ -254,19 +230,17 @@ OperationResult SQLite::DoErase(ValPtr key, OperationResultCallback* cb) { void SQLite::Expire() { auto stmt = prepared_stmts["expire"]; - if ( auto res = checkError(sqlite3_bind_double(stmt, 1, run_state::network_time)); + if ( auto res = CheckError(sqlite3_bind_double(stmt, 1, run_state::network_time)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); // TODO: do something with the error here? } - if ( auto res = checkError(sqlite3_step(stmt)); res.code != ReturnCode::SUCCESS ) { - // TODO: do something with the error here? - } + Step(stmt, false); } // returns true in case of error -OperationResult SQLite::checkError(int code) { +OperationResult SQLite::CheckError(int code) { if ( code != SQLITE_OK && code != SQLITE_DONE ) { return {ReturnCode::OPERATION_FAILED, util::fmt("SQLite call failed: %s", sqlite3_errmsg(db)), nullptr}; } @@ -274,4 +248,43 @@ OperationResult SQLite::checkError(int code) { return {ReturnCode::SUCCESS}; } +OperationResult SQLite::Step(sqlite3_stmt* stmt, bool parse_value) { + OperationResult ret; + + int step_status = sqlite3_step(stmt); + if ( step_status == SQLITE_ROW ) { + if ( parse_value ) { + // Column 1 is the value + const char* text = (const char*)sqlite3_column_text(stmt, 0); + auto val = zeek::detail::ValFromJSON(text, val_type, Func::nil); + sqlite3_reset(stmt); + if ( std::holds_alternative(val) ) { + ValPtr val_v = std::get(val); + ret = {ReturnCode::SUCCESS, "", val_v}; + } + else { + ret = {ReturnCode::OPERATION_FAILED, std::get(val)}; + } + } + else { + ret = {ReturnCode::OPERATION_FAILED, "sqlite3_step should not have returned a value"}; + } + } + else if ( step_status == SQLITE_DONE ) { + if ( parse_value ) + ret = {ReturnCode::KEY_NOT_FOUND}; + else + ret = {ReturnCode::SUCCESS}; + } + else if ( step_status == SQLITE_BUSY ) + // TODO: this could retry a number of times instead of just failing + ret = {ReturnCode::TIMEOUT}; + else + ret = {ReturnCode::OPERATION_FAILED}; + + sqlite3_reset(stmt); + + return ret; +} + } // namespace zeek::storage::backend::sqlite diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index 17e3e9716d..9f0e613de9 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -55,7 +55,8 @@ public: void Expire() override; private: - OperationResult checkError(int code); + OperationResult CheckError(int code); + OperationResult Step(sqlite3_stmt* stmt, bool parse_value = false); sqlite3* db = nullptr; std::unordered_map prepared_stmts; From a99a13dc4c4c9947dc733b733c64281b2ea30665 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 28 Feb 2025 15:14:26 -0700 Subject: [PATCH 36/52] SQLite: expand expiration test --- .../out | 10 ++- .../base/frameworks/storage/expiration.zeek | 65 ++++++++++++------- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out index c5b4964ce2..428580958a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.expiration/out @@ -1,6 +1,10 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. open result, [code=Storage::SUCCESS, error_str=, value=] -put result, [code=Storage::SUCCESS, error_str=, value=] -get result, [code=Storage::SUCCESS, error_str=, value=value7890] +put result 1, [code=Storage::SUCCESS, error_str=, value=] +put result 2, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value1234] get result same as inserted, T -get result, [code=Storage::KEY_NOT_FOUND, error_str=, value=] +get result 2, [code=Storage::SUCCESS, error_str=, value=value2345] +get result 2 same as inserted, T +get result 1 after expiration, [code=Storage::KEY_NOT_FOUND, error_str=, value=] +get result 2 after expiration, [code=Storage::SUCCESS, error_str=, value=value2345] diff --git a/testing/btest/scripts/base/frameworks/storage/expiration.zeek b/testing/btest/scripts/base/frameworks/storage/expiration.zeek index fc0efd6819..4a2d4d01fd 100644 --- a/testing/btest/scripts/base/frameworks/storage/expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/expiration.zeek @@ -12,43 +12,60 @@ redef exit_only_after_terminate = T; # Create a typename here that can be passed down into get(). type str: string; -global backend: opaque of Storage::BackendHandle; -global key: string = "key1234"; -global value: string = "value7890"; +global b: opaque of Storage::BackendHandle; +global key1: string = "key1234"; +global value1: string = "value1234"; -event check_removed() { - # This should return an error from the sqlite backend that there aren't any more - # rows available. - local res2 = Storage::Sync::get(backend, key); - if ( res2$code != Storage::SUCCESS ) - print "get result", res2; +global key2: string = "key2345"; +global value2: string = "value2345"; - Storage::Sync::close_backend(backend); +event check_removed() + { + local res = Storage::Sync::get(b, key1); + print "get result 1 after expiration", res; + + res = Storage::Sync::get(b, key2); + print "get result 2 after expiration", res; + + Storage::Sync::close_backend(b); terminate(); -} + } -event setup_test() { +event setup_test() + { local opts : Storage::BackendOptions; opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); print "open result", open_res; - backend = open_res$value; - local res = Storage::Sync::put(backend, [$key=key, $value=value, $expire_time=2 secs]); - print "put result", res; + b = open_res$value; - local res2 = Storage::Sync::get(backend, key); - print "get result", res2; - if ( res2$code == Storage::SUCCESS && res2?$value ) - print "get result same as inserted", value == (res2$value as string); + # Insert a key that will expire in the time allotted + local res = Storage::Sync::put(b, [ $key=key1, $value=value1, $expire_time=2secs ]); + print "put result 1", res; - schedule 5 secs { check_removed() }; -} + # Insert a key that won't expire + res = Storage::Sync::put(b, [ $key=key2, $value=value2, $expire_time=20secs ]); + print "put result 2", res; -event zeek_init() { + res = Storage::Sync::get(b, key1); + print "get result", res; + if ( res$code == Storage::SUCCESS && res?$value ) + print "get result same as inserted", value1 == ( res$value as string ); + + res = Storage::Sync::get(b, key2); + print "get result 2", res; + if ( res$code == Storage::SUCCESS && res?$value ) + print "get result 2 same as inserted", value2 == ( res$value as string ); + + schedule 5secs { check_removed() }; + } + +event zeek_init() + { # We need network time to be set to something other than zero for the # expiration time to be set correctly. Schedule an event on a short # timer so packets start getting read and do the setup there. - schedule 100 msecs { setup_test() }; -} + schedule 100msecs { setup_test() }; + } From cc7b2dc8902d1430e7de0ba02f7ccd2b0f4bd25d Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 3 Mar 2025 18:19:52 -0700 Subject: [PATCH 37/52] Implement Storage::backend_opened and Storage::backend_lost events --- src/storage/Backend.cc | 15 +++++++ src/storage/Backend.h | 19 ++++++++ src/storage/backend/redis/Redis.cc | 18 ++++---- src/storage/storage.bif | 10 +++-- .../out | 4 ++ .../out | 2 + .../out | 5 +++ .../frameworks/storage/redis-disconnect.zeek | 44 +++++++++++++++++++ .../base/frameworks/storage/redis-sync.zeek | 11 ++++- .../base/frameworks/storage/sqlite-basic.zeek | 4 ++ 10 files changed, 117 insertions(+), 15 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index b1ceb92a54..605cb900da 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -5,6 +5,7 @@ #include "zeek/Trigger.h" #include "zeek/broker/Data.h" #include "zeek/storage/ReturnCode.h" +#include "zeek/storage/storage.bif.h" namespace zeek::storage { @@ -68,6 +69,10 @@ OpenResultCallback::OpenResultCallback(zeek::detail::trigger::TriggerPtr trigger : ResultCallback(std::move(trigger), assoc), backend(std::move(backend)) {} void OpenResultCallback::Complete(OperationResult res) { + if ( res.code == ReturnCode::SUCCESS ) { + backend->backend->EnqueueBackendOpened(); + } + // If this is a sync callback, there isn't a trigger to process. Store the result and bail. Always // set result's value to the backend pointer so that it comes across in the result. This ensures // the handle is always available in the result even on failures. @@ -94,6 +99,7 @@ void OpenResultCallback::Complete(OperationResult res) { OperationResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb) { key_type = std::move(kt); val_type = std::move(vt); + backend_options = options; auto ret = DoOpen(std::move(options), cb); if ( ! ret.value ) @@ -153,6 +159,15 @@ void Backend::CompleteCallback(ResultCallback* cb, const OperationResult& data) } } +void Backend::EnqueueBackendOpened() { + event_mgr.Enqueue(Storage::backend_opened, make_intrusive(Tag()), backend_options); +} + +void Backend::EnqueueBackendLost(std::string_view reason) { + event_mgr.Enqueue(Storage::backend_lost, make_intrusive(Tag()), backend_options, + make_intrusive(reason)); +} + zeek::OpaqueTypePtr detail::backend_opaque; IMPLEMENT_OPAQUE_VALUE(detail::BackendHandleVal) diff --git a/src/storage/Backend.h b/src/storage/Backend.h index 139a1ab96c..828349526a 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -114,10 +114,15 @@ public: */ virtual void Poll() {} + const RecordValPtr& Options() const { return backend_options; } + protected: // Allow the manager to call Open/Close. friend class storage::Manager; + // Allow OpenResultCallback to call EnqueueConnectionEstablished. + friend class storage::OpenResultCallback; + /** * Constructor * @@ -183,10 +188,24 @@ protected: */ virtual void Expire() {} + /** + * Enqueues the Storage::backend_opened event. This is called automatically + * when an OpenResultCallback is completed successfully. + */ + void EnqueueBackendOpened(); + + /** + * Enqueues the Storage::backend_lost event with an optional reason + * string. This should be called by the backends whenever they lose their + * connection. + */ + void EnqueueBackendLost(std::string_view reason); + void CompleteCallback(ResultCallback* cb, const OperationResult& data) const; TypePtr key_type; TypePtr val_type; + RecordValPtr backend_options; std::string tag; diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index e5e5312987..bda140ef0a 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -8,6 +8,7 @@ #include "zeek/Val.h" #include "zeek/iosource/Manager.h" #include "zeek/storage/ReturnCode.h" +#include "zeek/storage/storage.bif.h" #include "hiredis/adapters/poll.h" #include "hiredis/async.h" @@ -481,7 +482,7 @@ void Redis::OnConnect(int status) { if ( status == REDIS_OK ) { connected = true; CompleteCallback(open_cb, {ReturnCode::SUCCESS}); - // TODO: post connect event + // The connection_established event is sent via the open callback handler. return; } @@ -493,17 +494,14 @@ void Redis::OnConnect(int status) { void Redis::OnDisconnect(int status) { DBG_LOG(DBG_STORAGE, "Redis backend: disconnection event"); + --active_ops; - - if ( status == REDIS_OK ) { - // TODO: this was an intentional disconnect, nothing to do? - } - else { - // TODO: this was unintentional, should we reconnect? - // TODO: post disconnect event - } - connected = false; + + if ( status == REDIS_ERR ) + EnqueueBackendLost(async_ctx->errstr); + else + EnqueueBackendLost("Client disconnected"); } void Redis::ProcessFd(int fd, int flags) { diff --git a/src/storage/storage.bif b/src/storage/storage.bif index c7c55ef2e4..7213618c56 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -45,11 +45,13 @@ static IntrusivePtr make_backend_handle(Val* module Storage; -# Generated when a new backend connection is opened -event Storage::backend_opened%(%); +## Generated automatically when a new backend connection is opened successfully. +event Storage::backend_opened%(tag: string, options: any%); -# Generated when a backend connection is lost -event Storage::backend_lost%(%); +## May be generated when a backend connection is lost, both normally and +## unexpectedly. This event depends on the backends implementing handling for +## it, and is not generated automatically by the storage framework. +event Storage::backend_lost%(tag: string, options: any, reason: string%); module Storage::Async; diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out new file mode 100644 index 0000000000..3308aef14e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +open_result, [code=Storage::SUCCESS, error_str=, value=] +Storage::backend_opened, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]] +Storage::backend_lost, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]], Server closed the connection diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out index 2f1d82d855..a8fa18538e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out @@ -6,3 +6,5 @@ get result same as inserted, T overwrite put result, [code=Storage::SUCCESS, error_str=, value=] get result, [code=Storage::SUCCESS, error_str=, value=value5678] get result same as inserted, T +Storage::backend_opened, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]] +Storage::backend_lost, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]], Client disconnected diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out index c9184cdc98..781e32a333 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out @@ -1,4 +1,9 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Storage::backend_opened, Storage::SQLITE, [sqlite=[database_path=test.sqlite, table_name=testing, tuning_params={ +[synchronous] = normal, +[temp_store] = memory, +[journal_mode] = WAL +}]] open result, [code=Storage::SUCCESS, error_str=, value=] put result, [code=Storage::SUCCESS, error_str=, value=] get result, [code=Storage::SUCCESS, error_str=, value=value5678] diff --git a/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek b/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek new file mode 100644 index 0000000000..c43dcbc99a --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek @@ -0,0 +1,44 @@ +# @TEST-DOC: Tests basic Redis storage backend functions in sync mode, including overwriting + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT + +# @TEST-EXEC: btest-bg-run redis-server run-redis-server ${REDIS_PORT%/tcp} +# @TEST-EXEC: zeek -b %INPUT | sed 's|=[0-9]*/tcp|=xxxx/tcp|g' > out +# @TEST-EXEC: btest-bg-wait -k 0 + +# @TEST-EXEC: btest-diff out + +@load base/frameworks/storage/sync +@load policy/frameworks/storage/backend/redis + +redef exit_only_after_terminate = T; + +# Create a typename here that can be passed down into open_backend() +type str: string; + +event Storage::backend_opened(tag: string, config: any) { + print "Storage::backend_opened", tag, config; +} + +event Storage::backend_lost(tag: string, config: any, reason: string) { + print "Storage::backend_lost", tag, config, reason; + terminate(); +} + +event zeek_init() + { + local opts: Storage::BackendOptions; + opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( + "REDIS_PORT")), $key_prefix="testing" ]; + + local key = "key1234"; + local value = "value1234"; + + local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + print "open_result", open_res; + + # Kill the redis server so the backend will disconnect and fire the backend_lost event. + system("cat redis-server/redis.pid"); + system("kill $(cat redis-server/redis.pid)"); + } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek index f3d5d4b684..ee2dc88031 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -4,7 +4,7 @@ # @TEST-PORT: REDIS_PORT # @TEST-EXEC: btest-bg-run redis-server run-redis-server ${REDIS_PORT%/tcp} -# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: zeek -b %INPUT | sed 's|=[0-9]*/tcp|=xxxx/tcp|g' > out # @TEST-EXEC: btest-bg-wait -k 0 # @TEST-EXEC: btest-diff out @@ -15,6 +15,15 @@ # Create a typename here that can be passed down into open_backend() type str: string; +event Storage::backend_opened(tag: string, config: any) { + print "Storage::backend_opened", tag, config; +} + +event Storage::backend_lost(tag: string, config: any, reason: string) { + print "Storage::backend_lost", tag, config, reason; + terminate(); +} + event zeek_init() { local opts: Storage::BackendOptions; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index 92f2a34059..3f37dd352b 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -11,6 +11,10 @@ redef exit_only_after_terminate = T; # Create a typename here that can be passed down into get(). type str: string; +event Storage::backend_opened(tag: string, config: any) { + print "Storage::backend_opened", tag, config; +} + event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table From 99160f8fcddd4a50f45e42ce20fdd266b0588253 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 7 Mar 2025 15:18:42 -0700 Subject: [PATCH 38/52] Rearrange visibility of Backend methods, add DoPoll/DoExpire, add return comments --- src/storage/Backend.h | 39 ++++++--------------- src/storage/backend/redis/Redis.cc | 4 +-- src/storage/backend/redis/Redis.h | 52 +++++++--------------------- src/storage/backend/sqlite/SQLite.cc | 2 +- src/storage/backend/sqlite/SQLite.h | 39 +++++++-------------- 5 files changed, 40 insertions(+), 96 deletions(-) diff --git a/src/storage/Backend.h b/src/storage/Backend.h index 828349526a..e9e42992f1 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -112,7 +112,7 @@ public: * Optional method to allow a backend to poll for data. This can be used to * mimic sync mode even if the backend only supports async. */ - virtual void Poll() {} + void Poll() { DoPoll(); } const RecordValPtr& Options() const { return backend_options; } @@ -156,37 +156,11 @@ protected: */ OperationResult Close(OperationResultCallback* cb = nullptr); - /** - * The workhorse method for Open(). - */ - virtual OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) = 0; - - /** - * The workhorse method for Close(). - */ - virtual OperationResult DoClose(OperationResultCallback* cb = nullptr) = 0; - - /** - * The workhorse method for Put(). - */ - virtual OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, - OperationResultCallback* cb = nullptr) = 0; - - /** - * The workhorse method for Get(). - */ - virtual OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) = 0; - - /** - * The workhorse method for Erase(). - */ - virtual OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) = 0; - /** * Removes any entries in the backend that have expired. Can be overridden by * derived classes. */ - virtual void Expire() {} + void Expire() { DoExpire(); } /** * Enqueues the Storage::backend_opened event. This is called automatically @@ -210,6 +184,15 @@ protected: std::string tag; private: + virtual OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) = 0; + virtual OperationResult DoClose(OperationResultCallback* cb = nullptr) = 0; + virtual OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + OperationResultCallback* cb = nullptr) = 0; + virtual OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) = 0; + virtual OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) = 0; + virtual void DoPoll() {} + virtual void DoExpire() {} + uint8_t modes; }; diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index bda140ef0a..7d1eb0a854 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -353,7 +353,7 @@ OperationResult Redis::DoErase(ValPtr key, OperationResultCallback* cb) { return {ReturnCode::SUCCESS}; } -void Redis::Expire() { +void Redis::DoExpire() { // Expiration is handled natively by Redis if not reading traces. if ( ! connected || ! zeek::run_state::reading_traces ) return; @@ -531,7 +531,7 @@ OperationResult Redis::ParseGetReply(redisReply* reply) const { return res; } -void Redis::Poll() { +void Redis::DoPoll() { while ( active_ops > 0 ) int status = redisPollTick(async_ctx, 0.5); } diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index 7bdfce9285..ae7238308d 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -28,43 +28,6 @@ public: */ const char* Tag() override { return tag.c_str(); } - /** - * Called by the manager system to open the backend. - */ - OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; - - /** - * Finalizes the backend when it's being closed. - */ - OperationResult DoClose(OperationResultCallback* cb = nullptr) override; - - /** - * Returns whether the backend is opened. - */ - bool IsOpen() override { return connected; } - - /** - * The workhorse method for Retrieve(). - */ - OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, - OperationResultCallback* cb = nullptr) override; - - /** - * The workhorse method for Get(). - */ - OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) override; - - /** - * The workhorse method for Erase(). - */ - OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) override; - - /** - * Removes any entries in the backend that have expired. Can be overridden by - * derived classes. - */ - void Expire() override; - // IOSource interface double GetNextTimeout() override { return -1; } void Process() override {} @@ -79,10 +42,21 @@ public: void HandleEraseResult(redisReply* reply, OperationResultCallback* callback); void HandleGeneric(redisReply* reply); -protected: - void Poll() override; + /** + * Returns whether the backend is opened. + */ + bool IsOpen() override { return connected; } private: + OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; + OperationResult DoClose(OperationResultCallback* cb = nullptr) override; + OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, + OperationResultCallback* cb = nullptr) override; + OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) override; + OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) override; + void DoExpire() override; + void DoPoll() override; + OperationResult ParseGetReply(redisReply* reply) const; redisAsyncContext* async_ctx = nullptr; diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index 50bfe52573..e1d5929d17 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -227,7 +227,7 @@ OperationResult SQLite::DoErase(ValPtr key, OperationResultCallback* cb) { * Removes any entries in the backend that have expired. Can be overridden by * derived classes. */ -void SQLite::Expire() { +void SQLite::DoExpire() { auto stmt = prepared_stmts["expire"]; if ( auto res = CheckError(sqlite3_bind_double(stmt, 1, run_state::network_time)); diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index 9f0e613de9..d4929bfa3e 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -17,45 +17,32 @@ public: static BackendPtr Instantiate(std::string_view tag); - /** - * Called by the manager system to open the backend. - */ - OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; - - /** - * Finalizes the backend when it's being closed. - */ - OperationResult DoClose(OperationResultCallback* cb = nullptr) override; - /** * Returns whether the backend is opened. */ bool IsOpen() override { return db != nullptr; } - /** - * The workhorse method for Put(). - */ +private: + OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; + OperationResult DoClose(OperationResultCallback* cb = nullptr) override; OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, OperationResultCallback* cb = nullptr) override; - - /** - * The workhorse method for Get(). - */ OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) override; - - /** - * The workhorse method for Erase(). - */ OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) override; + void DoExpire() override; /** - * Removes any entries in the backend that have expired. Can be overridden by - * derived classes. + * Checks whether a status code returned by an sqlite call is a success. + * + * @return A result structure containing a result code and an optional error + * string based on the status code. */ - void Expire() override; - -private: OperationResult CheckError(int code); + + /** + * Abstracts calls to sqlite3_step to properly create an OperationResult + * structure based on the result. + */ OperationResult Step(sqlite3_stmt* stmt, bool parse_value = false); sqlite3* db = nullptr; From ad224d9a3be5336e899baed8dfa170555f99c8c9 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 7 Mar 2025 16:52:35 -0700 Subject: [PATCH 39/52] Add OperationResult::MakeVal, use it to reduce some code duplication --- src/storage/Backend.cc | 54 +++++---------- src/storage/Backend.h | 3 +- src/storage/storage.bif | 142 +++++++++------------------------------- 3 files changed, 51 insertions(+), 148 deletions(-) diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index 605cb900da..daf74b8c8f 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -9,15 +9,17 @@ namespace zeek::storage { -RecordValPtr OperationResult::BuildVal() { +RecordValPtr OperationResult::BuildVal() { return MakeVal(code, err_str, value); } + +RecordValPtr OperationResult::MakeVal(EnumValPtr code, std::string_view err_str, ValPtr value) { static auto op_result_type = zeek::id::find_type("Storage::OperationResult"); auto rec = zeek::make_intrusive(op_result_type); - rec->Assign(0, code); + rec->Assign(0, std::move(code)); if ( ! err_str.empty() ) - rec->Assign(1, err_str); + rec->Assign(1, std::string{err_str}); if ( value ) - rec->Assign(2, value); + rec->Assign(2, std::move(value)); return rec; } @@ -28,12 +30,8 @@ ResultCallback::ResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void ResultCallback::Timeout() { static const auto& op_result_type = zeek::id::find_type("Storage::OperationResult"); - if ( ! IsSyncCallback() ) { - auto op_result = make_intrusive(op_result_type); - op_result->Assign(0, ReturnCode::TIMEOUT); - - trigger->Cache(assoc, op_result.release()); - } + if ( ! IsSyncCallback() ) + trigger->Cache(assoc, OperationResult::MakeVal(ReturnCode::TIMEOUT).release()); } OperationResultCallback::OperationResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc) @@ -46,19 +44,9 @@ void OperationResultCallback::Complete(OperationResult res) { return; } - static auto op_result_type = zeek::id::find_type("Storage::OperationResult"); - auto* op_result = new zeek::RecordVal(op_result_type); - - op_result->Assign(0, res.code); - if ( res.code->Get() != 0 ) - op_result->Assign(1, res.err_str); - else - op_result->Assign(2, res.value); - - trigger->Cache(assoc, op_result); + auto res_val = res.BuildVal(); + trigger->Cache(assoc, res_val.get()); trigger->Release(); - - Unref(op_result); } OpenResultCallback::OpenResultCallback(IntrusivePtr backend) @@ -73,27 +61,19 @@ void OpenResultCallback::Complete(OperationResult res) { backend->backend->EnqueueBackendOpened(); } - // If this is a sync callback, there isn't a trigger to process. Store the result and bail. Always - // set result's value to the backend pointer so that it comes across in the result. This ensures - // the handle is always available in the result even on failures. + // Set the result's value to the backend so that it ends up in the result getting either + // passed back to the trigger or the one stored for sync backends. + res.value = backend; + + // If this is a sync callback, there isn't a trigger to process. Store the result and bail. if ( IsSyncCallback() ) { result = std::move(res); - result.value = backend; return; } - static auto op_result_type = zeek::id::find_type("Storage::OperationResult"); - auto* op_result = new zeek::RecordVal(op_result_type); - - op_result->Assign(0, res.code); - if ( res.code != ReturnCode::SUCCESS ) - op_result->Assign(1, res.err_str); - op_result->Assign(2, backend); - - trigger->Cache(assoc, op_result); + auto res_val = res.BuildVal(); + trigger->Cache(assoc, res_val.get()); trigger->Release(); - - Unref(op_result); } OperationResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb) { diff --git a/src/storage/Backend.h b/src/storage/Backend.h index e9e42992f1..a038d04b95 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -20,6 +20,7 @@ struct OperationResult { ValPtr value; RecordValPtr BuildVal(); + static RecordValPtr MakeVal(EnumValPtr code, std::string_view err_str = "", ValPtr value = nullptr); }; @@ -38,7 +39,7 @@ public: protected: void CompleteWithVal(Val* result); - IntrusivePtr trigger; + zeek::detail::trigger::TriggerPtr trigger; const void* assoc = nullptr; }; diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 7213618c56..9239f59ed4 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -96,24 +96,15 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, function Storage::Async::__close_backend%(backend: opaque of Storage::BackendHandle%) : Storage::OperationResult %{ - static auto op_result_type = id::find_type("Storage::OperationResult"); - auto op_result = make_intrusive(op_result_type); - auto trigger = init_trigger(frame); if ( ! trigger ) return nullptr; auto b = dynamic_cast(backend); - if ( ! b ) { - op_result->Assign(0, ReturnCode::OPERATION_FAILED); - op_result->Assign(1, make_intrusive("Invalid storage handlle")); - return op_result; - } - else if ( ! b->backend->IsOpen() ) { - op_result->Assign(0, ReturnCode::NOT_CONNECTED); - op_result->Assign(1, make_intrusive("Backend is closed")); - return op_result; - } + if ( ! b ) + return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); + else if ( ! b->backend->IsOpen() ) + return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); auto close_res = storage_mgr->CloseBackend(b->backend, cb); @@ -136,24 +127,15 @@ function Storage::Async::__close_backend%(backend: opaque of Storage::BackendHan function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool, expire_time: interval%): Storage::OperationResult %{ - static auto op_result_type = id::find_type("Storage::OperationResult"); - auto op_result = make_intrusive(op_result_type); - auto trigger = init_trigger(frame); if ( ! trigger ) return nullptr; auto b = dynamic_cast(backend); - if ( ! b ) { - op_result->Assign(0, ReturnCode::OPERATION_FAILED); - op_result->Assign(1, make_intrusive("Invalid storage handlle")); - return op_result; - } - else if ( ! b->backend->IsOpen() ) { - op_result->Assign(0, ReturnCode::NOT_CONNECTED); - op_result->Assign(1, make_intrusive("Backend is closed")); - return op_result; - } + if ( ! b ) + return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); + else if ( ! b->backend->IsOpen() ) + return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); if ( expire_time > 0.0 ) expire_time += run_state::network_time; @@ -180,24 +162,15 @@ function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult %{ - static auto op_result_type = id::find_type("Storage::OperationResult"); - auto op_result = make_intrusive(op_result_type); - auto trigger = init_trigger(frame); if ( ! trigger ) return nullptr; auto b = dynamic_cast(backend); - if ( ! b ) { - op_result->Assign(0, ReturnCode::OPERATION_FAILED); - op_result->Assign(1, make_intrusive("Invalid storage handlle")); - return op_result; - } - else if ( ! b->backend->IsOpen() ) { - op_result->Assign(0, ReturnCode::NOT_CONNECTED); - op_result->Assign(1, make_intrusive("Backend is closed")); - return op_result; - } + if ( ! b ) + return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); + else if ( ! b->backend->IsOpen() ) + return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); auto key_v = IntrusivePtr{NewRef{}, key}; @@ -220,24 +193,15 @@ function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: function Storage::Async::__erase%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult %{ - static auto op_result_type = id::find_type("Storage::OperationResult"); - auto op_result = make_intrusive(op_result_type); - auto trigger = init_trigger(frame); if ( ! trigger ) return nullptr; auto b = dynamic_cast(backend); - if ( ! b ) { - op_result->Assign(0, ReturnCode::OPERATION_FAILED); - op_result->Assign(1, make_intrusive("Invalid storage handlle")); - return op_result; - } - else if ( ! b->backend->IsOpen() ) { - op_result->Assign(0, ReturnCode::NOT_CONNECTED); - op_result->Assign(1, make_intrusive("Backend is closed")); - return op_result; - } + if ( ! b ) + return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); + else if ( ! b->backend->IsOpen() ) + return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); auto key_v = IntrusivePtr{NewRef{}, key}; @@ -262,8 +226,6 @@ module Storage::Sync; function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): Storage::OperationResult %{ - static auto op_result_type = id::find_type("Storage::OperationResult"); - auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; Tag tag{btype_val}; @@ -296,21 +258,11 @@ function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, k function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHandle%) : Storage::OperationResult %{ - static auto op_result_type = id::find_type("Storage::OperationResult"); - auto b = dynamic_cast(backend); - if ( ! b ) { - auto op_result = make_intrusive(op_result_type); - op_result->Assign(0, ReturnCode::OPERATION_FAILED); - op_result->Assign(1, make_intrusive("Invalid storage handlle")); - return op_result; - } - else if ( ! b->backend->IsOpen() ) { - auto op_result = make_intrusive(op_result_type); - op_result->Assign(0, ReturnCode::NOT_CONNECTED); - op_result->Assign(1, make_intrusive("Backend is closed")); - return op_result; - } + if ( ! b ) + return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); + else if ( ! b->backend->IsOpen() ) + return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); auto cb = new OperationResultCallback(); auto close_res = storage_mgr->CloseBackend(b->backend, cb); @@ -330,21 +282,11 @@ function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHand function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool, expire_time: interval%): Storage::OperationResult %{ - static auto op_result_type = id::find_type("Storage::OperationResult"); - auto b = dynamic_cast(backend); - if ( ! b ) { - auto op_result = make_intrusive(op_result_type); - op_result->Assign(0, ReturnCode::OPERATION_FAILED); - op_result->Assign(1, make_intrusive("Invalid storage handlle")); - return op_result; - } - else if ( ! b->backend->IsOpen() ) { - auto op_result = make_intrusive(op_result_type); - op_result->Assign(0, ReturnCode::NOT_CONNECTED); - op_result->Assign(1, make_intrusive("Backend is closed")); - return op_result; - } + if ( ! b ) + return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); + else if ( ! b->backend->IsOpen() ) + return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); if ( expire_time > 0.0 ) expire_time += run_state::network_time; @@ -368,21 +310,11 @@ function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: a function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult %{ - static auto op_result_type = id::find_type("Storage::OperationResult"); - auto b = dynamic_cast(backend); - if ( ! b ) { - auto op_result = make_intrusive(op_result_type); - op_result->Assign(0, ReturnCode::OPERATION_FAILED); - op_result->Assign(1, make_intrusive("Invalid storage handlle")); - return op_result; - } - else if ( ! b->backend->IsOpen() ) { - auto op_result = make_intrusive(op_result_type); - op_result->Assign(0, ReturnCode::NOT_CONNECTED); - op_result->Assign(1, make_intrusive("Backend is closed")); - return op_result; - } + if ( ! b ) + return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); + else if ( ! b->backend->IsOpen() ) + return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); auto key_v = IntrusivePtr{NewRef{}, key}; auto cb = new OperationResultCallback(); @@ -402,21 +334,11 @@ function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: a function Storage::Sync::__erase%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult %{ - static auto op_result_type = id::find_type("Storage::OperationResult"); - auto b = dynamic_cast(backend); - if ( ! b ) { - auto op_result = make_intrusive(op_result_type); - op_result->Assign(0, ReturnCode::OPERATION_FAILED); - op_result->Assign(1, make_intrusive("Invalid storage handlle")); - return op_result; - } - else if ( ! b->backend->IsOpen() ) { - auto op_result = make_intrusive(op_result_type); - op_result->Assign(0, ReturnCode::NOT_CONNECTED); - op_result->Assign(1, make_intrusive("Backend is closed")); - return op_result; - } + if ( ! b ) + return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); + else if ( ! b->backend->IsOpen() ) + return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); auto cb = new OperationResultCallback(); auto key_v = IntrusivePtr{NewRef{}, key}; From 44c6e32ae775d6a604b4f67646858077e8fb8c12 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sat, 8 Mar 2025 09:43:07 -0700 Subject: [PATCH 40/52] Reduce code duplication in storage.bif --- src/storage/storage.bif | 283 +++++++++++++++++++--------------------- 1 file changed, 133 insertions(+), 150 deletions(-) diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 9239f59ed4..393a21fe0c 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -41,6 +41,32 @@ static IntrusivePtr make_backend_handle(Val* return make_intrusive(b.value()); } +static zeek::expected cast_handle(Val* handle) { + auto b = static_cast(handle); + + if ( ! b ) + return zeek::unexpected(OperationResult{ReturnCode::OPERATION_FAILED, "Invalid storage handlle"}); + else if ( ! b->backend->IsOpen() ) + return zeek::unexpected(OperationResult{ReturnCode::NOT_CONNECTED, "Backend is closed"}); + + return b; +} + +static void handle_async_result(const IntrusivePtr& backend, ResultCallback* cb, + const OperationResult& op_result) { + if ( ! backend->SupportsAsync() ) { + // If the backend doesn't support async, we blocked in order to get here already. Handle the + // callback manually. + cb->Complete(op_result); + delete cb; + } + else if ( run_state::reading_traces ) { + // If the backend is truly async and we're reading traces, we need to fake being in sync mode + // because otherwise time doesn't move forward correctly. + backend->Poll(); + } +} + %%} module Storage; @@ -67,7 +93,10 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, auto b = storage_mgr->Instantiate(tag); if ( ! b.has_value() ) { - emit_builtin_error(util::fmt("Failed to instantiate backend: %s", b.error().c_str())); + trigger->Cache( + frame->GetTriggerAssoc(), + new StringVal(util::fmt("Failed to instantiate backend: %s", b.error().c_str()))); + trigger->Release(); return nullptr; } @@ -77,19 +106,9 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; - auto open_res = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); + auto op_result = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); - if ( ! b.value()->SupportsAsync() ) { - // If the backend doesn't support async, we blocked in order to get here already. Handle the - // callback manually. - cb->Complete(open_res); - delete cb; - } - else if ( run_state::reading_traces ) { - // If the backend is truly async and we're reading traces, we need to fake being in sync mode - // because otherwise time doesn't move forward correctly. - b.value()->Poll(); - } + handle_async_result(b.value(), cb, op_result); return nullptr; %} @@ -100,26 +119,16 @@ function Storage::Async::__close_backend%(backend: opaque of Storage::BackendHan if ( ! trigger ) return nullptr; - auto b = dynamic_cast(backend); - if ( ! b ) - return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); - else if ( ! b->backend->IsOpen() ) - return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); - auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); - auto close_res = storage_mgr->CloseBackend(b->backend, cb); - - if ( ! b->backend->SupportsAsync() ) { - // If the backend doesn't support async, we blocked in order to get here already. Handle the - // callback manually. - cb->Complete(close_res); + auto b = cast_handle(backend); + if ( ! b ) { + cb->Complete(b.error()); delete cb; + return nullptr; } - else if ( run_state::reading_traces ) { - // If the backend is truly async and we're reading traces, we need to fake being in sync mode - // because otherwise time doesn't move forward correctly. - b->backend->Poll(); - } + + auto op_result = storage_mgr->CloseBackend((*b)->backend, cb); + handle_async_result((*b)->backend, cb, op_result); return nullptr; %} @@ -131,31 +140,21 @@ function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: if ( ! trigger ) return nullptr; - auto b = dynamic_cast(backend); - if ( ! b ) - return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); - else if ( ! b->backend->IsOpen() ) - return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); + auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); + auto b = cast_handle(backend); + if ( ! b ) { + cb->Complete(b.error()); + delete cb; + return nullptr; + } if ( expire_time > 0.0 ) expire_time += run_state::network_time; - auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; - auto put_res = b->backend->Put(key_v, val_v, overwrite, expire_time, cb); - - if ( ! b->backend->SupportsAsync() ) { - // If the backend doesn't support async, we blocked in order to get here already. Handle the - // callback manually. - cb->Complete(put_res); - delete cb; - } - else if ( run_state::reading_traces ) { - // If the backend is truly async and we're reading traces, we need to fake being in sync mode - // because otherwise time doesn't move forward correctly. - b->backend->Poll(); - } + auto op_result = (*b)->backend->Put(key_v, val_v, overwrite, expire_time, cb); + handle_async_result((*b)->backend, cb, op_result); return nullptr; %} @@ -166,27 +165,17 @@ function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: if ( ! trigger ) return nullptr; - auto b = dynamic_cast(backend); - if ( ! b ) - return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); - else if ( ! b->backend->IsOpen() ) - return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); - auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); - auto key_v = IntrusivePtr{NewRef{}, key}; - auto get_res = b->backend->Get(key_v, cb); - - if ( ! b->backend->SupportsAsync() ) { - // If the backend doesn't support async, we blocked in order to get here already. Handle the - // callback manually. - cb->Complete(get_res); + auto b = cast_handle(backend); + if ( ! b ) { + cb->Complete(b.error()); delete cb; + return nullptr; } - else if ( run_state::reading_traces ) { - // If the backend is truly async and we're reading traces, we need to fake being in sync mode - // because otherwise time doesn't move forward correctly. - b->backend->Poll(); - } + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto op_result = (*b)->backend->Get(key_v, cb); + handle_async_result((*b)->backend, cb, op_result); return nullptr; %} @@ -197,27 +186,17 @@ function Storage::Async::__erase%(backend: opaque of Storage::BackendHandle, key if ( ! trigger ) return nullptr; - auto b = dynamic_cast(backend); - if ( ! b ) - return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); - else if ( ! b->backend->IsOpen() ) - return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); - auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); - auto key_v = IntrusivePtr{NewRef{}, key}; - auto erase_res = b->backend->Erase(key_v, cb); - - if ( ! b->backend->SupportsAsync() ) { - // If the backend doesn't support async, we blocked in order to get here already. Handle the - // callback manually. - cb->Complete(erase_res); + auto b = cast_handle(backend); + if ( ! b ) { + cb->Complete(b.error()); delete cb; + return nullptr; } - else if ( run_state::reading_traces ) { - // If the backend is truly async and we're reading traces, we need to fake being in sync mode - // because otherwise time doesn't move forward correctly. - b->backend->Poll(); - } + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto op_result = (*b)->backend->Erase(key_v, cb); + handle_async_result((*b)->backend, cb, op_result); return nullptr; %} @@ -242,116 +221,120 @@ function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, k auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; - auto open_res = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); + auto op_result = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); // If the backend only supports async, block until it's ready and then pull the result out of // the callback. if ( ! b.value()->SupportsSync() ) { b.value()->Poll(); - open_res = cb->Result(); + op_result = cb->Result(); } delete cb; - return open_res.BuildVal(); + return op_result.BuildVal(); %} function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHandle%) : Storage::OperationResult %{ - auto b = dynamic_cast(backend); + OperationResult op_result; + + auto b = cast_handle(backend); if ( ! b ) - return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); - else if ( ! b->backend->IsOpen() ) - return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); + op_result = b.error(); + else { + auto cb = new OperationResultCallback(); + op_result = storage_mgr->CloseBackend((*b)->backend, cb); - auto cb = new OperationResultCallback(); - auto close_res = storage_mgr->CloseBackend(b->backend, cb); + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! (*b)->backend->SupportsSync() ) { + (*b)->backend->Poll(); + op_result = cb->Result(); + } - // If the backend only supports async, block until it's ready and then pull the result out of - // the callback. - if ( ! b->backend->SupportsSync() ) { - b->backend->Poll(); - close_res = cb->Result(); + delete cb; } - delete cb; - - return close_res.BuildVal(); + return op_result.BuildVal(); %} function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool, expire_time: interval%): Storage::OperationResult %{ - auto b = dynamic_cast(backend); + OperationResult op_result; + + auto b = cast_handle(backend); if ( ! b ) - return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); - else if ( ! b->backend->IsOpen() ) - return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); + op_result = b.error(); + else { + if ( expire_time > 0.0 ) + expire_time += run_state::network_time; - if ( expire_time > 0.0 ) - expire_time += run_state::network_time; + auto cb = new OperationResultCallback(); + auto key_v = IntrusivePtr{NewRef{}, key}; + auto val_v = IntrusivePtr{NewRef{}, value}; + op_result = (*b)->backend->Put(key_v, val_v, overwrite, expire_time, cb); - auto cb = new OperationResultCallback(); - auto key_v = IntrusivePtr{NewRef{}, key}; - auto val_v = IntrusivePtr{NewRef{}, value}; - auto put_res = b->backend->Put(key_v, val_v, overwrite, expire_time, cb); + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! (*b)->backend->SupportsSync() ) { + (*b)->backend->Poll(); + op_result = cb->Result(); + } - // If the backend only supports async, block until it's ready and then pull the result out of - // the callback. - if ( ! b->backend->SupportsSync() ) { - b->backend->Poll(); - put_res = cb->Result(); + delete cb; } - delete cb; - - return put_res.BuildVal(); + return op_result.BuildVal(); %} function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult %{ - auto b = dynamic_cast(backend); + OperationResult op_result; + + auto b = cast_handle(backend); if ( ! b ) - return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); - else if ( ! b->backend->IsOpen() ) - return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); + op_result = b.error(); + else { + auto cb = new OperationResultCallback(); + auto key_v = IntrusivePtr{NewRef{}, key}; + op_result = (*b)->backend->Get(key_v, cb); - auto key_v = IntrusivePtr{NewRef{}, key}; - auto cb = new OperationResultCallback(); - auto get_res = b->backend->Get(key_v, cb); + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! (*b)->backend->SupportsSync() ) { + (*b)->backend->Poll(); + op_result = cb->Result(); + } - // If the backend only supports async, block until it's ready and then pull the result out of - // the callback. - if ( ! b->backend->SupportsSync() ) { - b->backend->Poll(); - get_res = cb->Result(); + delete cb; } - delete cb; - - return get_res.BuildVal(); + return op_result.BuildVal(); %} function Storage::Sync::__erase%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult %{ - auto b = dynamic_cast(backend); + OperationResult op_result; + + auto b = cast_handle(backend); if ( ! b ) - return OperationResult::MakeVal(ReturnCode::OPERATION_FAILED, "Invalid storage handlle"); - else if ( ! b->backend->IsOpen() ) - return OperationResult::MakeVal(ReturnCode::NOT_CONNECTED, "Backend is closed"); + op_result = b.error(); + else { + auto cb = new OperationResultCallback(); + auto key_v = IntrusivePtr{NewRef{}, key}; + op_result = (*b)->backend->Erase(key_v, cb); - auto cb = new OperationResultCallback(); - auto key_v = IntrusivePtr{NewRef{}, key}; - auto erase_res = b->backend->Erase(key_v, cb); + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! (*b)->backend->SupportsSync() ) { + (*b)->backend->Poll(); + op_result = cb->Result(); + } - // If the backend only supports async, block until it's ready and then pull the result out of - // the callback. - if ( ! b->backend->SupportsSync() ) { - b->backend->Poll(); - erase_res = cb->Result(); + delete cb; } - delete cb; - - return erase_res.BuildVal(); + return op_result.BuildVal(); %} From 60aa987e06e87ba4de6f9ea62b55e09f3d38d9e8 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sat, 8 Mar 2025 14:25:37 -0700 Subject: [PATCH 41/52] Store sqlite3_stmts directly instead of looking up from a map --- src/storage/backend/sqlite/SQLite.cc | 51 ++++++++++++++++------------ src/storage/backend/sqlite/SQLite.h | 9 ++++- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index e1d5929d17..18f96870ab 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -78,18 +78,19 @@ OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { } } - static std::map statements = - {{"put", util::fmt("insert into %s (key_str, value_str, expire_time) values(?, ?, ?)", table_name.c_str())}, - {"put_update", - util::fmt("insert into %s (key_str, value_str, expire_time) values(?, ?, ?) ON CONFLICT(key_str) " - "DO UPDATE SET value_str=?", - table_name.c_str())}, - {"get", util::fmt("select value_str from %s where key_str=?", table_name.c_str())}, - {"erase", util::fmt("delete from %s where key_str=?", table_name.c_str())}, - {"expire", util::fmt("delete from %s where expire_time > 0 and expire_time != 0 and expire_time <= ?", - table_name.c_str())}}; + static std::array statements = + {util::fmt("insert into %s (key_str, value_str, expire_time) values(?, ?, ?)", table_name.c_str()), + util::fmt("insert into %s (key_str, value_str, expire_time) values(?, ?, ?) ON CONFLICT(key_str) " + "DO UPDATE SET value_str=?", + table_name.c_str()), + util::fmt("select value_str from %s where key_str=?", table_name.c_str()), + util::fmt("delete from %s where key_str=?", table_name.c_str()), + util::fmt("delete from %s where expire_time > 0 and expire_time != 0 and expire_time <= ?", + table_name.c_str())}; - for ( const auto& [key, stmt] : statements ) { + std::array stmt_ptrs; + int i = 0; + for ( const auto& stmt : statements ) { sqlite3_stmt* ps; if ( auto prep_res = CheckError(sqlite3_prepare_v2(db, stmt.c_str(), stmt.size(), &ps, NULL)); prep_res.code != ReturnCode::SUCCESS ) { @@ -97,9 +98,15 @@ OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { return prep_res; } - prepared_stmts.insert({key, ps}); + stmt_ptrs[i++] = unique_stmt_ptr(ps, [](sqlite3_stmt* stmt) { sqlite3_finalize(stmt); }); } + put_stmt = std::move(stmt_ptrs[0]); + put_update_stmt = std::move(stmt_ptrs[1]); + get_stmt = std::move(stmt_ptrs[2]); + erase_stmt = std::move(stmt_ptrs[3]); + expire_stmt = std::move(stmt_ptrs[4]); + sqlite3_busy_timeout(db, 5000); return {ReturnCode::SUCCESS}; @@ -112,11 +119,11 @@ OperationResult SQLite::DoClose(OperationResultCallback* cb) { OperationResult op_res{ReturnCode::SUCCESS}; if ( db ) { - for ( const auto& [k, stmt] : prepared_stmts ) { - sqlite3_finalize(stmt); - } - - prepared_stmts.clear(); + put_stmt.reset(); + put_update_stmt.reset(); + get_stmt.reset(); + erase_stmt.reset(); + expire_stmt.reset(); char* errmsg; if ( int res = sqlite3_exec(db, "pragma optimize", NULL, NULL, &errmsg); res != SQLITE_OK ) { @@ -149,9 +156,9 @@ OperationResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double e sqlite3_stmt* stmt; if ( ! overwrite ) - stmt = prepared_stmts["put"]; + stmt = put_stmt.get(); else - stmt = prepared_stmts["put_update"]; + stmt = put_update_stmt.get(); auto key_str = json_key->ToStdStringView(); if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); @@ -191,7 +198,7 @@ OperationResult SQLite::DoGet(ValPtr key, OperationResultCallback* cb) { return {ReturnCode::NOT_CONNECTED}; auto json_key = key->ToJSON(); - auto stmt = prepared_stmts["get"]; + auto stmt = get_stmt.get(); auto key_str = json_key->ToStdStringView(); if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); @@ -211,7 +218,7 @@ OperationResult SQLite::DoErase(ValPtr key, OperationResultCallback* cb) { return {ReturnCode::NOT_CONNECTED}; auto json_key = key->ToJSON(); - auto stmt = prepared_stmts["erase"]; + auto stmt = erase_stmt.get(); auto key_str = json_key->ToStdStringView(); if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); @@ -228,7 +235,7 @@ OperationResult SQLite::DoErase(ValPtr key, OperationResultCallback* cb) { * derived classes. */ void SQLite::DoExpire() { - auto stmt = prepared_stmts["expire"]; + auto stmt = expire_stmt.get(); if ( auto res = CheckError(sqlite3_bind_double(stmt, 1, run_state::network_time)); res.code != ReturnCode::SUCCESS ) { diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index d4929bfa3e..bbef996215 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -46,7 +46,14 @@ private: OperationResult Step(sqlite3_stmt* stmt, bool parse_value = false); sqlite3* db = nullptr; - std::unordered_map prepared_stmts; + + using stmt_deleter = std::function; + using unique_stmt_ptr = std::unique_ptr; + unique_stmt_ptr put_stmt; + unique_stmt_ptr put_update_stmt; + unique_stmt_ptr get_stmt; + unique_stmt_ptr erase_stmt; + unique_stmt_ptr expire_stmt; std::string full_path; std::string table_name; From c7503654e80c597aee3d6096b71177cbec5585f1 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sun, 9 Mar 2025 20:48:26 -0700 Subject: [PATCH 42/52] Add IN_PROGRESS return code, handle for async backends --- scripts/base/init-bare.zeek | 5 ++++- src/storage/Manager.cc | 2 +- src/storage/ReturnCode.cc | 5 +++++ src/storage/ReturnCode.h | 1 + src/storage/backend/redis/Redis.cc | 10 ++++------ src/storage/storage.bif | 9 ++++++--- 6 files changed, 21 insertions(+), 11 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index a71963f1fd..a45c5904f1 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -6246,7 +6246,10 @@ export { ## Generic disconnection failure. DISCONNECTION_FAILED, ## Generic initialization failure. - INITIALIZATION_FAILED + INITIALIZATION_FAILED, + ## Returned from async operations when the backend is waiting + ## for a result. + IN_PROGRESS, } &redef; ## Returned as the result of the various storage operations. diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index 6f991af5c2..0e1bf86ca4 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -76,7 +76,7 @@ zeek::expected Manager::Instantiate(const Tag& type) { OperationResult Manager::OpenBackend(BackendPtr backend, RecordValPtr options, TypePtr key_type, TypePtr val_type, OpenResultCallback* cb) { auto res = backend->Open(std::move(options), std::move(key_type), std::move(val_type), cb); - if ( res.code != ReturnCode::SUCCESS ) { + if ( res.code != ReturnCode::SUCCESS && res.code != ReturnCode::IN_PROGRESS ) { res.err_str = util::fmt("Failed to open backend %s: %s", backend->Tag(), res.err_str.c_str()); return res; } diff --git a/src/storage/ReturnCode.cc b/src/storage/ReturnCode.cc index 50d13554ce..22a3e8a2e6 100644 --- a/src/storage/ReturnCode.cc +++ b/src/storage/ReturnCode.cc @@ -18,6 +18,7 @@ EnumValPtr ReturnCode::KEY_EXISTS; EnumValPtr ReturnCode::CONNECTION_FAILED; EnumValPtr ReturnCode::DISCONNECTION_FAILED; EnumValPtr ReturnCode::INITIALIZATION_FAILED; +EnumValPtr ReturnCode::IN_PROGRESS; void ReturnCode::Initialize() { static const auto& return_code_type = zeek::id::find_type("Storage::ReturnCode"); @@ -57,6 +58,9 @@ void ReturnCode::Initialize() { tmp = return_code_type->Lookup("Storage::INITIALIZATION_FAILED"); INITIALIZATION_FAILED = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::IN_PROGRESS"); + IN_PROGRESS = return_code_type->GetEnumVal(tmp); } void ReturnCode::Cleanup() { @@ -72,6 +76,7 @@ void ReturnCode::Cleanup() { CONNECTION_FAILED.reset(); DISCONNECTION_FAILED.reset(); INITIALIZATION_FAILED.reset(); + IN_PROGRESS.reset(); } } // namespace zeek::storage diff --git a/src/storage/ReturnCode.h b/src/storage/ReturnCode.h index 826408c9df..8b110641f0 100644 --- a/src/storage/ReturnCode.h +++ b/src/storage/ReturnCode.h @@ -31,6 +31,7 @@ public: static EnumValPtr CONNECTION_FAILED; static EnumValPtr DISCONNECTION_FAILED; static EnumValPtr INITIALIZATION_FAILED; + static EnumValPtr IN_PROGRESS; }; } // namespace storage diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 7d1eb0a854..5ec512b3f7 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -228,7 +228,7 @@ OperationResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { async_ctx->ev.addWrite = redisAddWrite; async_ctx->ev.delWrite = redisDelWrite; - return {ReturnCode::SUCCESS}; + return {ReturnCode::IN_PROGRESS}; } /** @@ -247,8 +247,6 @@ OperationResult Redis::DoClose(OperationResultCallback* cb) { // TODO: handle response } - CompleteCallback(cb, {ReturnCode::SUCCESS}); - redisAsyncFree(async_ctx); async_ctx = nullptr; @@ -306,7 +304,7 @@ OperationResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double ex ++active_ops; } - return {ReturnCode::SUCCESS}; + return {ReturnCode::IN_PROGRESS}; } /** @@ -329,7 +327,7 @@ OperationResult Redis::DoGet(ValPtr key, OperationResultCallback* cb) { // There isn't a result to return here. That happens in HandleGetResult for // async operations. - return {ReturnCode::SUCCESS}; + return {ReturnCode::IN_PROGRESS}; } /** @@ -350,7 +348,7 @@ OperationResult Redis::DoErase(ValPtr key, OperationResultCallback* cb) { ++active_ops; - return {ReturnCode::SUCCESS}; + return {ReturnCode::IN_PROGRESS}; } void Redis::DoExpire() { diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 393a21fe0c..82d85e6afb 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -54,9 +54,12 @@ static zeek::expected cast_ static void handle_async_result(const IntrusivePtr& backend, ResultCallback* cb, const OperationResult& op_result) { - if ( ! backend->SupportsAsync() ) { - // If the backend doesn't support async, we blocked in order to get here already. Handle the - // callback manually. + if ( op_result.code != ReturnCode::IN_PROGRESS || ! backend->SupportsAsync() ) { + // We need to complete the callback early if: + // 1. The operation didn't start up successfully. For async operations, this means + // it didn't report back IN_PROGRESS. + // 2. The backend doesn't support async. This means we already blocked in order + // to get here already. cb->Complete(op_result); delete cb; } From ebefb21c53bc5596faf3dee10de8ca9da853c0fa Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sun, 9 Mar 2025 21:04:21 -0700 Subject: [PATCH 43/52] Pass network time down to Expire() --- src/storage/Backend.h | 7 +++++-- src/storage/Manager.cc | 3 ++- src/storage/backend/redis/Redis.cc | 6 +++--- src/storage/backend/redis/Redis.h | 2 +- src/storage/backend/sqlite/SQLite.cc | 5 ++--- src/storage/backend/sqlite/SQLite.h | 2 +- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/storage/Backend.h b/src/storage/Backend.h index a038d04b95..80810cd97f 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -160,8 +160,11 @@ protected: /** * Removes any entries in the backend that have expired. Can be overridden by * derived classes. + * + * @param current_network_time The network time as of the start of the + * expiration operation. */ - void Expire() { DoExpire(); } + void Expire(double current_network_time) { DoExpire(current_network_time); } /** * Enqueues the Storage::backend_opened event. This is called automatically @@ -192,7 +195,7 @@ private: virtual OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) = 0; virtual OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) = 0; virtual void DoPoll() {} - virtual void DoExpire() {} + virtual void DoExpire(double current_network_time) {} uint8_t modes; }; diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index 0e1bf86ca4..a9cf0bf7ff 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -113,9 +113,10 @@ void Manager::Expire() { DBG_LOG(DBG_STORAGE, "Expiration running, have %zu backends to check", backends.size()); + double current_network_time = run_state::network_time; for ( auto it = backends.begin(); it != backends.end() && ! run_state::terminating; ++it ) { if ( (*it)->IsOpen() ) - (*it)->Expire(); + (*it)->Expire(current_network_time); } expire_running.clear(); diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 5ec512b3f7..4990502e63 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -351,7 +351,7 @@ OperationResult Redis::DoErase(ValPtr key, OperationResultCallback* cb) { return {ReturnCode::IN_PROGRESS}; } -void Redis::DoExpire() { +void Redis::DoExpire(double current_network_time) { // Expiration is handled natively by Redis if not reading traces. if ( ! connected || ! zeek::run_state::reading_traces ) return; @@ -361,7 +361,7 @@ void Redis::DoExpire() { during_expire = true; int status = redisAsyncCommand(async_ctx, redisGeneric, NULL, "ZRANGEBYSCORE %s_expire -inf %f", key_prefix.data(), - run_state::network_time); + current_network_time); if ( status == REDIS_ERR ) { // TODO: do something with the error? @@ -407,7 +407,7 @@ void Redis::DoExpire() { // Remove all of the elements from the range-set that match the time range. redisAsyncCommand(async_ctx, redisGeneric, NULL, "ZREMRANGEBYSCORE %s_expire -inf %f", key_prefix.data(), - run_state::network_time); + current_network_time); ++active_ops; Poll(); diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index ae7238308d..1be442b82d 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -54,7 +54,7 @@ private: OperationResultCallback* cb = nullptr) override; OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) override; OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) override; - void DoExpire() override; + void DoExpire(double current_network_time) override; void DoPoll() override; OperationResult ParseGetReply(redisReply* reply) const; diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index 18f96870ab..53678f68ab 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -234,11 +234,10 @@ OperationResult SQLite::DoErase(ValPtr key, OperationResultCallback* cb) { * Removes any entries in the backend that have expired. Can be overridden by * derived classes. */ -void SQLite::DoExpire() { +void SQLite::DoExpire(double current_network_time) { auto stmt = expire_stmt.get(); - if ( auto res = CheckError(sqlite3_bind_double(stmt, 1, run_state::network_time)); - res.code != ReturnCode::SUCCESS ) { + if ( auto res = CheckError(sqlite3_bind_double(stmt, 1, current_network_time)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); // TODO: do something with the error here? } diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index bbef996215..3fc7bcfcf7 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -29,7 +29,7 @@ private: OperationResultCallback* cb = nullptr) override; OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) override; OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) override; - void DoExpire() override; + void DoExpire(double current_network_time) override; /** * Checks whether a status code returned by an sqlite call is a success. From 605973497f0dddab9f3f741e5e175e663dbcc079 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 10 Mar 2025 05:51:59 -0700 Subject: [PATCH 44/52] Remove file-local expire_running variable --- src/storage/backend/redis/Redis.cc | 16 +++++++--------- src/storage/backend/redis/Redis.h | 6 +++++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 4990502e63..57fe3c5727 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -17,8 +17,6 @@ // Anonymous callback handler methods for the hiredis async API. namespace { -bool during_expire = false; - class Tracer { public: Tracer(const std::string& where) : where(where) {} // DBG_LOG(zeek::DBG_STORAGE, "%s", where.c_str()); } @@ -91,7 +89,7 @@ void redisAddRead(void* privdata) { auto rpe = static_cast(privdata); auto backend = static_cast(rpe->context->data); - if ( rpe->reading == 0 && ! during_expire ) + if ( rpe->reading == 0 && ! backend->ExpireRunning() ) zeek::iosource_mgr->RegisterFd(rpe->fd, backend, zeek::iosource::IOSource::READ); rpe->reading = 1; } @@ -101,7 +99,7 @@ void redisDelRead(void* privdata) { auto rpe = static_cast(privdata); auto backend = static_cast(rpe->context->data); - if ( rpe->reading == 1 && ! during_expire ) + if ( rpe->reading == 1 && ! backend->ExpireRunning() ) zeek::iosource_mgr->UnregisterFd(rpe->fd, backend, zeek::iosource::IOSource::READ); rpe->reading = 0; } @@ -111,7 +109,7 @@ void redisAddWrite(void* privdata) { auto rpe = static_cast(privdata); auto backend = static_cast(rpe->context->data); - if ( rpe->writing == 0 && ! during_expire ) + if ( rpe->writing == 0 && ! backend->ExpireRunning() ) zeek::iosource_mgr->RegisterFd(rpe->fd, backend, zeek::iosource::IOSource::WRITE); rpe->writing = 1; } @@ -121,7 +119,7 @@ void redisDelWrite(void* privdata) { auto t = Tracer("delwrite"); auto backend = static_cast(rpe->context->data); - if ( rpe->writing == 1 && ! during_expire ) + if ( rpe->writing == 1 && ! backend->ExpireRunning() ) zeek::iosource_mgr->UnregisterFd(rpe->fd, backend, zeek::iosource::IOSource::WRITE); rpe->writing = 0; } @@ -358,7 +356,7 @@ void Redis::DoExpire(double current_network_time) { auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); - during_expire = true; + expire_running = true; int status = redisAsyncCommand(async_ctx, redisGeneric, NULL, "ZRANGEBYSCORE %s_expire -inf %f", key_prefix.data(), current_network_time); @@ -366,7 +364,7 @@ void Redis::DoExpire(double current_network_time) { if ( status == REDIS_ERR ) { // TODO: do something with the error? printf("ZRANGEBYSCORE command failed: %s\n", async_ctx->errstr); - during_expire = false; + expire_running = false; return; } @@ -380,7 +378,7 @@ void Redis::DoExpire(double current_network_time) { if ( reply->elements == 0 ) { freeReplyObject(reply); - during_expire = false; + expire_running = false; return; } diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index 1be442b82d..f77a334d95 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -47,6 +47,8 @@ public: */ bool IsOpen() override { return connected; } + bool ExpireRunning() const { return expire_running.load(); } + private: OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; OperationResult DoClose(OperationResultCallback* cb = nullptr) override; @@ -71,8 +73,10 @@ private: std::string server_addr; std::string key_prefix; + std::atomic connected = false; - int active_ops = 0; + std::atomic expire_running = false; + std::atomic active_ops = 0; }; } // namespace zeek::storage::backend::redis From e6f1eea1b735ab3986921f3e0f0d941904743542 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 10 Mar 2025 13:51:05 -0700 Subject: [PATCH 45/52] Remove default argument for callbacks, reorder function arguments --- src/storage/Backend.cc | 18 +++++++------- src/storage/Backend.h | 24 +++++++++---------- src/storage/Manager.cc | 6 ++--- src/storage/Manager.h | 6 ++--- src/storage/backend/redis/Redis.cc | 10 ++++---- src/storage/backend/redis/Redis.h | 12 +++++----- src/storage/backend/sqlite/SQLite.cc | 18 +++++++------- src/storage/backend/sqlite/SQLite.h | 12 +++++----- src/storage/storage.bif | 16 ++++++------- .../storage-plugin/src/StorageDummy.cc | 10 ++++---- .../plugins/storage-plugin/src/StorageDummy.h | 15 +++++------- 11 files changed, 72 insertions(+), 75 deletions(-) diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index daf74b8c8f..f47e29eeea 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -76,12 +76,12 @@ void OpenResultCallback::Complete(OperationResult res) { trigger->Release(); } -OperationResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb) { +OperationResult Backend::Open(OpenResultCallback* cb, RecordValPtr options, TypePtr kt, TypePtr vt) { key_type = std::move(kt); val_type = std::move(vt); backend_options = options; - auto ret = DoOpen(std::move(options), cb); + auto ret = DoOpen(cb, std::move(options)); if ( ! ret.value ) ret.value = cb->Backend(); @@ -90,8 +90,8 @@ OperationResult Backend::Open(RecordValPtr options, TypePtr kt, TypePtr vt, Open OperationResult Backend::Close(OperationResultCallback* cb) { return DoClose(cb); } -OperationResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double expiration_time, - OperationResultCallback* cb) { +OperationResult Backend::Put(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite, + double expiration_time) { // The intention for this method is to do some other heavy lifting in regard // to backends that need to pass data through the manager instead of directly // through the workers. For the first versions of the storage framework it @@ -107,10 +107,10 @@ OperationResult Backend::Put(ValPtr key, ValPtr value, bool overwrite, double ex return ret; } - return DoPut(std::move(key), std::move(value), overwrite, expiration_time, cb); + return DoPut(cb, std::move(key), std::move(value), overwrite, expiration_time); } -OperationResult Backend::Get(ValPtr key, OperationResultCallback* cb) { +OperationResult Backend::Get(OperationResultCallback* cb, ValPtr key) { // See the note in Put(). if ( ! same_type(key->GetType(), key_type) ) { auto ret = OperationResult{ReturnCode::KEY_TYPE_MISMATCH}; @@ -118,10 +118,10 @@ OperationResult Backend::Get(ValPtr key, OperationResultCallback* cb) { return ret; } - return DoGet(std::move(key), cb); + return DoGet(cb, std::move(key)); } -OperationResult Backend::Erase(ValPtr key, OperationResultCallback* cb) { +OperationResult Backend::Erase(OperationResultCallback* cb, ValPtr key) { // See the note in Put(). if ( ! same_type(key->GetType(), key_type) ) { auto ret = OperationResult{ReturnCode::KEY_TYPE_MISMATCH}; @@ -129,7 +129,7 @@ OperationResult Backend::Erase(ValPtr key, OperationResultCallback* cb) { return ret; } - return DoErase(std::move(key), cb); + return DoErase(cb, std::move(key)); } void Backend::CompleteCallback(ResultCallback* cb, const OperationResult& data) const { diff --git a/src/storage/Backend.h b/src/storage/Backend.h index 80810cd97f..98578c43fb 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -78,8 +78,8 @@ public: * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ - OperationResult Put(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, - OperationResultCallback* cb = nullptr); + OperationResult Put(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite = true, + double expiration_time = 0); /** * Retrieve a value from the backend for a provided key. @@ -89,7 +89,7 @@ public: * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ - OperationResult Get(ValPtr key, OperationResultCallback* cb = nullptr); + OperationResult Get(OperationResultCallback* cb, ValPtr key); /** * Erases the value for a key from the backend. @@ -99,7 +99,7 @@ public: * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ - OperationResult Erase(ValPtr key, OperationResultCallback* cb = nullptr); + OperationResult Erase(OperationResultCallback* cb, ValPtr key); /** * Returns whether the backend is opened. @@ -146,7 +146,7 @@ protected: * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ - OperationResult Open(RecordValPtr options, TypePtr kt, TypePtr vt, OpenResultCallback* cb = nullptr); + OperationResult Open(OpenResultCallback* cb, RecordValPtr options, TypePtr kt, TypePtr vt); /** * Finalizes the backend when it's being closed. @@ -155,7 +155,7 @@ protected: * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ - OperationResult Close(OperationResultCallback* cb = nullptr); + OperationResult Close(OperationResultCallback* cb); /** * Removes any entries in the backend that have expired. Can be overridden by @@ -188,12 +188,12 @@ protected: std::string tag; private: - virtual OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) = 0; - virtual OperationResult DoClose(OperationResultCallback* cb = nullptr) = 0; - virtual OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, - OperationResultCallback* cb = nullptr) = 0; - virtual OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) = 0; - virtual OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) = 0; + virtual OperationResult DoOpen(OpenResultCallback* cb, RecordValPtr options) = 0; + virtual OperationResult DoClose(OperationResultCallback* cb) = 0; + virtual OperationResult DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite = true, + double expiration_time = 0) = 0; + virtual OperationResult DoGet(OperationResultCallback* cb, ValPtr key) = 0; + virtual OperationResult DoErase(OperationResultCallback* cb, ValPtr key) = 0; virtual void DoPoll() {} virtual void DoExpire(double current_network_time) {} diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index a9cf0bf7ff..b36e073606 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -73,9 +73,9 @@ zeek::expected Manager::Instantiate(const Tag& type) { return bp; } -OperationResult Manager::OpenBackend(BackendPtr backend, RecordValPtr options, TypePtr key_type, TypePtr val_type, - OpenResultCallback* cb) { - auto res = backend->Open(std::move(options), std::move(key_type), std::move(val_type), cb); +OperationResult Manager::OpenBackend(BackendPtr backend, OpenResultCallback* cb, RecordValPtr options, TypePtr key_type, + TypePtr val_type) { + auto res = backend->Open(cb, std::move(options), std::move(key_type), std::move(val_type)); if ( res.code != ReturnCode::SUCCESS && res.code != ReturnCode::IN_PROGRESS ) { res.err_str = util::fmt("Failed to open backend %s: %s", backend->Tag(), res.err_str.c_str()); return res; diff --git a/src/storage/Manager.h b/src/storage/Manager.h index badd2c6c5f..a382c533ba 100644 --- a/src/storage/Manager.h +++ b/src/storage/Manager.h @@ -60,8 +60,8 @@ public: * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ - OperationResult OpenBackend(BackendPtr backend, RecordValPtr options, TypePtr key_type, TypePtr val_type, - OpenResultCallback* cb = nullptr); + OperationResult OpenBackend(BackendPtr backend, OpenResultCallback* cb, RecordValPtr options, TypePtr key_type, + TypePtr val_type); /** * Closes a storage backend. @@ -71,7 +71,7 @@ public: * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ - OperationResult CloseBackend(BackendPtr backend, OperationResultCallback* cb = nullptr); + OperationResult CloseBackend(BackendPtr backend, OperationResultCallback* cb); void Expire(); diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 57fe3c5727..0cd2ca004c 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -143,7 +143,7 @@ storage::BackendPtr Redis::Instantiate(std::string_view tag) { return make_intru /** * Called by the manager system to open the backend. */ -OperationResult Redis::DoOpen(RecordValPtr options, OpenResultCallback* cb) { +OperationResult Redis::DoOpen(OpenResultCallback* cb, RecordValPtr options) { RecordValPtr backend_options = options->GetField("redis"); key_prefix = backend_options->GetField("key_prefix")->ToStdString(); @@ -254,8 +254,8 @@ OperationResult Redis::DoClose(OperationResultCallback* cb) { /** * The workhorse method for Put(). This must be implemented by plugins. */ -OperationResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, - OperationResultCallback* cb) { +OperationResult Redis::DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite, + double expiration_time) { // The async context will queue operations until it's connected fully. if ( ! connected && ! async_ctx ) return {ReturnCode::NOT_CONNECTED}; @@ -308,7 +308,7 @@ OperationResult Redis::DoPut(ValPtr key, ValPtr value, bool overwrite, double ex /** * The workhorse method for Get(). This must be implemented for plugins. */ -OperationResult Redis::DoGet(ValPtr key, OperationResultCallback* cb) { +OperationResult Redis::DoGet(OperationResultCallback* cb, ValPtr key) { // The async context will queue operations until it's connected fully. if ( ! connected && ! async_ctx ) return {ReturnCode::NOT_CONNECTED}; @@ -331,7 +331,7 @@ OperationResult Redis::DoGet(ValPtr key, OperationResultCallback* cb) { /** * The workhorse method for Erase(). This must be implemented for plugins. */ -OperationResult Redis::DoErase(ValPtr key, OperationResultCallback* cb) { +OperationResult Redis::DoErase(OperationResultCallback* cb, ValPtr key) { // The async context will queue operations until it's connected fully. if ( ! connected && ! async_ctx ) return {ReturnCode::NOT_CONNECTED}; diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index f77a334d95..c5ac1271de 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -50,12 +50,12 @@ public: bool ExpireRunning() const { return expire_running.load(); } private: - OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; - OperationResult DoClose(OperationResultCallback* cb = nullptr) override; - OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, - OperationResultCallback* cb = nullptr) override; - OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) override; - OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) override; + OperationResult DoOpen(OpenResultCallback* cb, RecordValPtr options) override; + OperationResult DoClose(OperationResultCallback* cb) override; + OperationResult DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite = true, + double expiration_time = 0) override; + OperationResult DoGet(OperationResultCallback* cb, ValPtr key) override; + OperationResult DoErase(OperationResultCallback* cb, ValPtr key) override; void DoExpire(double current_network_time) override; void DoPoll() override; diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index 53678f68ab..d8e123a5a7 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -14,7 +14,7 @@ storage::BackendPtr SQLite::Instantiate(std::string_view tag) { return make_intr /** * Called by the manager system to open the backend. */ -OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { +OperationResult SQLite::DoOpen(OpenResultCallback* cb, RecordValPtr options) { if ( sqlite3_threadsafe() == 0 ) { std::string res = "SQLite reports that it is not threadsafe. Zeek needs a threadsafe version of " @@ -51,7 +51,7 @@ OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { std::string err = util::fmt("Error executing table creation statement: %s", errorMsg); Error(err.c_str()); sqlite3_free(errorMsg); - Close(); + Close(nullptr); return {ReturnCode::INITIALIZATION_FAILED, err}; } @@ -59,7 +59,7 @@ OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { std::string err = util::fmt("Error executing integrity check: %s", errorMsg); Error(err.c_str()); sqlite3_free(errorMsg); - Close(); + Close(nullptr); return {ReturnCode::INITIALIZATION_FAILED, err}; } @@ -73,7 +73,7 @@ OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { std::string err = util::fmt("Error executing tuning pragma statement: %s", errorMsg); Error(err.c_str()); sqlite3_free(errorMsg); - Close(); + Close(nullptr); return {ReturnCode::INITIALIZATION_FAILED, err}; } } @@ -94,7 +94,7 @@ OperationResult SQLite::DoOpen(RecordValPtr options, OpenResultCallback* cb) { sqlite3_stmt* ps; if ( auto prep_res = CheckError(sqlite3_prepare_v2(db, stmt.c_str(), stmt.size(), &ps, NULL)); prep_res.code != ReturnCode::SUCCESS ) { - Close(); + Close(nullptr); return prep_res; } @@ -146,8 +146,8 @@ OperationResult SQLite::DoClose(OperationResultCallback* cb) { /** * The workhorse method for Put(). This must be implemented by plugins. */ -OperationResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, - OperationResultCallback* cb) { +OperationResult SQLite::DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite, + double expiration_time) { if ( ! db ) return {ReturnCode::NOT_CONNECTED}; @@ -193,7 +193,7 @@ OperationResult SQLite::DoPut(ValPtr key, ValPtr value, bool overwrite, double e /** * The workhorse method for Get(). This must be implemented for plugins. */ -OperationResult SQLite::DoGet(ValPtr key, OperationResultCallback* cb) { +OperationResult SQLite::DoGet(OperationResultCallback* cb, ValPtr key) { if ( ! db ) return {ReturnCode::NOT_CONNECTED}; @@ -213,7 +213,7 @@ OperationResult SQLite::DoGet(ValPtr key, OperationResultCallback* cb) { /** * The workhorse method for Erase(). This must be implemented for plugins. */ -OperationResult SQLite::DoErase(ValPtr key, OperationResultCallback* cb) { +OperationResult SQLite::DoErase(OperationResultCallback* cb, ValPtr key) { if ( ! db ) return {ReturnCode::NOT_CONNECTED}; diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index 3fc7bcfcf7..e8d1781ec2 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -23,12 +23,12 @@ public: bool IsOpen() override { return db != nullptr; } private: - OperationResult DoOpen(RecordValPtr options, OpenResultCallback* cb = nullptr) override; - OperationResult DoClose(OperationResultCallback* cb = nullptr) override; - OperationResult DoPut(ValPtr key, ValPtr value, bool overwrite = true, double expiration_time = 0, - OperationResultCallback* cb = nullptr) override; - OperationResult DoGet(ValPtr key, OperationResultCallback* cb = nullptr) override; - OperationResult DoErase(ValPtr key, OperationResultCallback* cb = nullptr) override; + OperationResult DoOpen(OpenResultCallback* cb, RecordValPtr options) override; + OperationResult DoClose(OperationResultCallback* cb) override; + OperationResult DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite = true, + double expiration_time = 0) override; + OperationResult DoGet(OperationResultCallback* cb, ValPtr key) override; + OperationResult DoErase(OperationResultCallback* cb, ValPtr key) override; void DoExpire(double current_network_time) override; /** diff --git a/src/storage/storage.bif b/src/storage/storage.bif index 82d85e6afb..b32818734e 100644 --- a/src/storage/storage.bif +++ b/src/storage/storage.bif @@ -109,7 +109,7 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; - auto op_result = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); + auto op_result = storage_mgr->OpenBackend(b.value(), cb, options_val, kt, vt); handle_async_result(b.value(), cb, op_result); @@ -156,7 +156,7 @@ function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; - auto op_result = (*b)->backend->Put(key_v, val_v, overwrite, expire_time, cb); + auto op_result = (*b)->backend->Put(cb, key_v, val_v, overwrite, expire_time); handle_async_result((*b)->backend, cb, op_result); return nullptr; @@ -177,7 +177,7 @@ function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: } auto key_v = IntrusivePtr{NewRef{}, key}; - auto op_result = (*b)->backend->Get(key_v, cb); + auto op_result = (*b)->backend->Get(cb, key_v); handle_async_result((*b)->backend, cb, op_result); return nullptr; @@ -198,7 +198,7 @@ function Storage::Async::__erase%(backend: opaque of Storage::BackendHandle, key } auto key_v = IntrusivePtr{NewRef{}, key}; - auto op_result = (*b)->backend->Erase(key_v, cb); + auto op_result = (*b)->backend->Erase(cb, key_v); handle_async_result((*b)->backend, cb, op_result); return nullptr; @@ -224,7 +224,7 @@ function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, k auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; - auto op_result = storage_mgr->OpenBackend(b.value(), options_val, kt, vt, cb); + auto op_result = storage_mgr->OpenBackend(b.value(), cb, options_val, kt, vt); // If the backend only supports async, block until it's ready and then pull the result out of // the callback. @@ -277,7 +277,7 @@ function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: a auto cb = new OperationResultCallback(); auto key_v = IntrusivePtr{NewRef{}, key}; auto val_v = IntrusivePtr{NewRef{}, value}; - op_result = (*b)->backend->Put(key_v, val_v, overwrite, expire_time, cb); + op_result = (*b)->backend->Put(cb, key_v, val_v, overwrite, expire_time); // If the backend only supports async, block until it's ready and then pull the result out of // the callback. @@ -302,7 +302,7 @@ function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: a else { auto cb = new OperationResultCallback(); auto key_v = IntrusivePtr{NewRef{}, key}; - op_result = (*b)->backend->Get(key_v, cb); + op_result = (*b)->backend->Get(cb, key_v); // If the backend only supports async, block until it's ready and then pull the result out of // the callback. @@ -327,7 +327,7 @@ function Storage::Sync::__erase%(backend: opaque of Storage::BackendHandle, key: else { auto cb = new OperationResultCallback(); auto key_v = IntrusivePtr{NewRef{}, key}; - op_result = (*b)->backend->Erase(key_v, cb); + op_result = (*b)->backend->Erase(cb, key_v); // If the backend only supports async, block until it's ready and then pull the result out of // the callback. diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc index 03e4360553..8cf64094c3 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -20,7 +20,7 @@ BackendPtr StorageDummy::Instantiate(std::string_view tag) { return make_intrusi * implementation must call \a Opened(); if not, it must call Error() * with a corresponding message. */ -OperationResult StorageDummy::DoOpen(RecordValPtr options, OpenResultCallback* cb) { +OperationResult StorageDummy::DoOpen(OpenResultCallback* cb, RecordValPtr options) { RecordValPtr backend_options = options->GetField("dummy"); bool open_fail = backend_options->GetField("open_fail")->Get(); if ( open_fail ) @@ -42,8 +42,8 @@ OperationResult StorageDummy::DoClose(OperationResultCallback* cb) { /** * The workhorse method for Put(). This must be implemented by plugins. */ -OperationResult StorageDummy::DoPut(ValPtr key, ValPtr value, bool overwrite, double expiration_time, - OperationResultCallback* cb) { +OperationResult StorageDummy::DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite, + double expiration_time) { auto json_key = key->ToJSON()->ToStdString(); auto json_value = value->ToJSON()->ToStdString(); data[json_key] = json_value; @@ -53,7 +53,7 @@ OperationResult StorageDummy::DoPut(ValPtr key, ValPtr value, bool overwrite, do /** * The workhorse method for Get(). This must be implemented for plugins. */ -OperationResult StorageDummy::DoGet(ValPtr key, OperationResultCallback* cb) { +OperationResult StorageDummy::DoGet(OperationResultCallback* cb, ValPtr key) { auto json_key = key->ToJSON(); auto it = data.find(json_key->ToStdString()); if ( it == data.end() ) @@ -71,7 +71,7 @@ OperationResult StorageDummy::DoGet(ValPtr key, OperationResultCallback* cb) { /** * The workhorse method for Erase(). This must be implemented for plugins. */ -OperationResult StorageDummy::DoErase(ValPtr key, OperationResultCallback* cb) { +OperationResult StorageDummy::DoErase(OperationResultCallback* cb, ValPtr key) { auto json_key = key->ToJSON(); auto it = data.find(json_key->ToStdString()); if ( it == data.end() ) diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h index fbd0abc032..d326cb1bde 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.h +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -21,8 +21,7 @@ public: /** * Called by the manager system to open the backend. */ - zeek::storage::OperationResult DoOpen(zeek::RecordValPtr options, - zeek::storage::OpenResultCallback* cb = nullptr) override; + zeek::storage::OperationResult DoOpen(zeek::storage::OpenResultCallback* cb, zeek::RecordValPtr options) override; /** * Finalizes the backend when it's being closed. @@ -37,21 +36,19 @@ public: /** * The workhorse method for Put(). */ - zeek::storage::OperationResult DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite = true, - double expiration_time = 0, - zeek::storage::OperationResultCallback* cb = nullptr) override; + zeek::storage::OperationResult DoPut(zeek::storage::OperationResultCallback* cb, zeek::ValPtr key, + zeek::ValPtr value, bool overwrite = true, + double expiration_time = 0) override; /** * The workhorse method for Get(). */ - zeek::storage::OperationResult DoGet(zeek::ValPtr key, - zeek::storage::OperationResultCallback* cb = nullptr) override; + zeek::storage::OperationResult DoGet(zeek::storage::OperationResultCallback* cb, zeek::ValPtr key) override; /** * The workhorse method for Erase(). */ - zeek::storage::OperationResult DoErase(zeek::ValPtr key, - zeek::storage::OperationResultCallback* cb = nullptr) override; + zeek::storage::OperationResult DoErase(zeek::storage::OperationResultCallback* cb, zeek::ValPtr key) override; private: std::map data; From b067a6e5889870c1621ecaaccabcce977ab092e5 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 10 Mar 2025 15:06:19 -0700 Subject: [PATCH 46/52] Redis: Fix sync erase, add btest for it --- src/storage/backend/redis/Redis.cc | 22 ++++----- .../out | 7 +++ .../base/frameworks/storage/redis-erase.zeek | 48 +++++++++++++++++++ 3 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.redis-erase/out create mode 100644 testing/btest/scripts/base/frameworks/storage/redis-erase.zeek diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 0cd2ca004c..869d89a22a 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -447,21 +447,17 @@ void Redis::HandleGetResult(redisReply* reply, OperationResultCallback* callback void Redis::HandleEraseResult(redisReply* reply, OperationResultCallback* callback) { --active_ops; - if ( callback->IsSyncCallback() ) - reply_queue.push_back(reply); - else { - OperationResult res{ReturnCode::SUCCESS}; + OperationResult res{ReturnCode::SUCCESS}; - if ( ! connected ) - res = {ReturnCode::NOT_CONNECTED}; - else if ( ! reply ) - res = {ReturnCode::OPERATION_FAILED, "Async erase operation returned null reply"}; - else if ( reply && reply->type == REDIS_REPLY_ERROR ) - res = {ReturnCode::OPERATION_FAILED, util::fmt("Async erase operation failed: %s", reply->str)}; + if ( ! connected ) + res = {ReturnCode::NOT_CONNECTED}; + else if ( ! reply ) + res = {ReturnCode::OPERATION_FAILED, "Async erase operation returned null reply"}; + else if ( reply && reply->type == REDIS_REPLY_ERROR ) + res = {ReturnCode::OPERATION_FAILED, util::fmt("Async erase operation failed: %s", reply->str)}; - freeReplyObject(reply); - CompleteCallback(callback, res); - } + freeReplyObject(reply); + CompleteCallback(callback, res); } void Redis::HandleGeneric(redisReply* reply) { diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-erase/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-erase/out new file mode 100644 index 0000000000..c6ddb5d53b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-erase/out @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +open_result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value1234] +get result same as inserted, T +erase result, [code=Storage::SUCCESS, error_str=, value=] +get result 2, [code=Storage::KEY_NOT_FOUND, error_str=, value=] diff --git a/testing/btest/scripts/base/frameworks/storage/redis-erase.zeek b/testing/btest/scripts/base/frameworks/storage/redis-erase.zeek new file mode 100644 index 0000000000..edc3752958 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/redis-erase.zeek @@ -0,0 +1,48 @@ +# @TEST-DOC: Tests basic Redis storage backend functions in sync mode, including overwriting + +# @TEST-REQUIRES: have-redis +# @TEST-PORT: REDIS_PORT + +# @TEST-EXEC: btest-bg-run redis-server run-redis-server ${REDIS_PORT%/tcp} +# @TEST-EXEC: zeek -b %INPUT | sed 's|=[0-9]*/tcp|=xxxx/tcp|g' > out +# @TEST-EXEC: btest-bg-wait -k 0 + +# @TEST-EXEC: btest-diff out + +@load base/frameworks/storage/sync +@load policy/frameworks/storage/backend/redis + +# Create a typename here that can be passed down into open_backend() +type str: string; + +event zeek_init() + { + local opts: Storage::BackendOptions; + opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( + "REDIS_PORT")), $key_prefix="testing" ]; + + local key = "key1234"; + local value = "value1234"; + + local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, str, str); + print "open_result", open_res; + + local b = open_res$value; + + local res = Storage::Sync::put(b, [ $key=key, $value=value ]); + print "put result", res; + + res = Storage::Sync::get(b, key); + print "get result", res; + if ( res$code == Storage::SUCCESS && res?$value ) + print "get result same as inserted", value == ( res$value as string ); + + res = Storage::Sync::erase(b, key); + print "erase result", res; + + res = Storage::Sync::get(b, key); + if ( res$code != Storage::SUCCESS ) + print "get result 2", res; + + Storage::Sync::close_backend(b); + } From a40db844ebc0b530c846ba0e691142b1781f4037 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 10 Mar 2025 15:32:03 -0700 Subject: [PATCH 47/52] Redis: Handle disconnection correctly via callback --- scripts/base/init-bare.zeek | 5 +++-- src/storage/backend/redis/Redis.cc | 32 ++++++++++++++++-------------- src/storage/backend/redis/Redis.h | 1 + 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index a45c5904f1..98b16e103e 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -6233,7 +6233,7 @@ export { NOT_CONNECTED, ## Operation timed out. TIMEOUT, - ## Connection to backed was lost. + ## Connection to backed was lost unexpectedly. CONNECTION_LOST, ## Generic operation failed. OPERATION_FAILED, @@ -6241,7 +6241,8 @@ export { KEY_NOT_FOUND, ## Key requested for overwrite already exists. KEY_EXISTS, - ## Generic connection failure. + ## Generic connection-setup failure. This is not if the connection + ## was lost, but if it failed to be setup in the first place. CONNECTION_FAILED, ## Generic disconnection failure. DISCONNECTION_FAILED, diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 869d89a22a..5d357239d2 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -173,6 +173,9 @@ OperationResult Redis::DoOpen(OpenResultCallback* cb, RecordValPtr options) { struct timeval timeout = {5, 0}; opt.connect_timeout = &timeout; + // The connection request below should be operation #1. + active_ops = 1; + async_ctx = redisAsyncConnectWithOptions(&opt); if ( async_ctx == nullptr || async_ctx->err ) { // This block doesn't necessarily mean the connection failed. It means @@ -189,8 +192,6 @@ OperationResult Redis::DoOpen(OpenResultCallback* cb, RecordValPtr options) { return {ReturnCode::CONNECTION_FAILED, errmsg}; } - ++active_ops; - // There's no way to pass privdata down to the connect handler like there is for // the other callbacks. Store the open callback so that it can be dealt with from // OnConnect(). @@ -236,19 +237,12 @@ OperationResult Redis::DoClose(OperationResultCallback* cb) { auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); connected = false; + close_cb = cb; redisAsyncDisconnect(async_ctx); ++active_ops; - if ( cb->IsSyncCallback() && ! zeek::run_state::terminating ) { - Poll(); - // TODO: handle response - } - - redisAsyncFree(async_ctx); - async_ctx = nullptr; - - return {ReturnCode::SUCCESS}; + return {ReturnCode::IN_PROGRESS}; } /** @@ -487,13 +481,21 @@ void Redis::OnConnect(int status) { void Redis::OnDisconnect(int status) { DBG_LOG(DBG_STORAGE, "Redis backend: disconnection event"); - --active_ops; connected = false; - - if ( status == REDIS_ERR ) + if ( status == REDIS_ERR ) { + // An error status indicates that the connection was lost unexpectedly and not + // via a request from backend. EnqueueBackendLost(async_ctx->errstr); - else + } + else { + --active_ops; + EnqueueBackendLost("Client disconnected"); + CompleteCallback(close_cb, {ReturnCode::SUCCESS}); + } + + redisAsyncFree(async_ctx); + async_ctx = nullptr; } void Redis::ProcessFd(int fd, int flags) { diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index c5ac1271de..2af6d9f501 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -69,6 +69,7 @@ private: std::deque reply_queue; OpenResultCallback* open_cb; + OperationResultCallback* close_cb; std::mutex expire_mutex; std::string server_addr; From d0741c800173cf768a438838919e31311f4da114 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 10 Mar 2025 16:10:38 -0700 Subject: [PATCH 48/52] Allow sync methods to be called from when conditions, add related btest --- .../.stderr | 2 + .../out | 6 ++ .../storage/sqlite-basic-sync-in-when.zeek | 74 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-sync-in-when/.stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-sync-in-when/out create mode 100644 testing/btest/scripts/base/frameworks/storage/sqlite-basic-sync-in-when.zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-sync-in-when/.stderr b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-sync-in-when/.stderr new file mode 100644 index 0000000000..e3f6131b1d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-sync-in-when/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-sync-in-when/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-sync-in-when/out new file mode 100644 index 0000000000..c9184cdc98 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic-sync-in-when/out @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +open result, [code=Storage::SUCCESS, error_str=, value=] +put result, [code=Storage::SUCCESS, error_str=, value=] +get result, [code=Storage::SUCCESS, error_str=, value=value5678] +get result same as inserted, T +closed succesfully diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-sync-in-when.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-sync-in-when.zeek new file mode 100644 index 0000000000..9679ab56eb --- /dev/null +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-sync-in-when.zeek @@ -0,0 +1,74 @@ +# @TEST-DOC: Basic functionality for storage: opening/closing an sqlite backend, storing/retrieving data, using sync methods in when conditions +# @TEST-EXEC: zeek -b %INPUT > out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff .stderr + +@load base/frameworks/storage/sync +@load policy/frameworks/storage/backend/sqlite + +redef exit_only_after_terminate = T; + +# Create a typename here that can be passed down into get(). +type str: string; + +event Storage::backend_opened(tag: string, config: any) { + print "Storage::backend_opened", tag, config; +} + +event zeek_init() + { + # Create a database file in the .tmp directory with a 'testing' table + local opts: Storage::BackendOptions; + opts$sqlite = [ $database_path="test.sqlite", $table_name="testing" ]; + + local key = "key1234"; + local value = "value5678"; + + # Test inserting/retrieving a key/value pair that we know won't be in + # the backend yet. + when [opts, key, value] ( local open_res = Storage::Sync::open_backend( + Storage::SQLITE, opts, str, str) ) + { + print "open result", open_res; + local b = open_res$value; + + when [b, key, value] ( local put_res = Storage::Sync::put(b, [ $key=key, + $value=value ]) ) + { + print "put result", put_res; + + when [b, key, value] ( local get_res = Storage::Sync::get(b, key) ) + { + print "get result", get_res; + if ( get_res$code == Storage::SUCCESS && get_res?$value ) + print "get result same as inserted", value == ( get_res$value as string ); + + when [b] ( local close_res = Storage::Sync::close_backend(b) ) + { + print "closed succesfully"; + terminate(); + } + timeout 5sec + { + print "close request timed out"; + terminate(); + } + } + timeout 5sec + { + print "get request timed out"; + terminate(); + } + } + timeout 5sec + { + print "put request timed out"; + terminate(); + } + } + timeout 5sec + { + print "open request timed out"; + terminate(); + } + } From f40947f6ac17f9af1362f006f3534b1df9e38fbe Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 17 Mar 2025 14:00:14 -0700 Subject: [PATCH 49/52] Update comments in script files, run zeek-format on all of them --- scripts/base/frameworks/storage/async.zeek | 76 +++++++++++-------- scripts/base/frameworks/storage/main.zeek | 6 +- scripts/base/frameworks/storage/sync.zeek | 60 +++++++++------ .../storage/backend/redis/main.zeek | 16 ++-- .../storage/backend/sqlite/main.zeek | 23 +++--- 5 files changed, 104 insertions(+), 77 deletions(-) diff --git a/scripts/base/frameworks/storage/async.zeek b/scripts/base/frameworks/storage/async.zeek index 8fbadb6e1b..3a30755afe 100644 --- a/scripts/base/frameworks/storage/async.zeek +++ b/scripts/base/frameworks/storage/async.zeek @@ -1,6 +1,4 @@ -##! Asynchronous operation methods for the storage framework. These methods must -##! be called as part of a :zeek:see:`when` statement. An error will be returned -##! otherwise. +##! Asynchronous operation methods for the storage framework. @load ./main @@ -8,6 +6,8 @@ module Storage::Async; export { ## Opens a new backend connection based on a configuration object asynchronously. + ## This method must be called via a :zeek:see:`when` condition or an error will + ## be returned. ## ## btype: A tag indicating what type of backend should be opened. These are ## defined by the backend plugins loaded. @@ -25,18 +25,22 @@ export { ## Returns: A record containing the status of the operation, and either an error ## string on failure or a value on success. The value returned here will ## be an ``opaque of BackendHandle``. - global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, - val_type: any): Storage::OperationResult; + global open_backend: function(btype: Storage::Backend, + options: Storage::BackendOptions, key_type: any, val_type: any) + : Storage::OperationResult; - ## Closes an existing backend connection asynchronously. + ## Closes an existing backend connection asynchronously. This method must be + ## called via a :zeek:see:`when` condition or an error will be returned. ## ## backend: A handle to a backend connection. ## ## Returns: A record containing the status of the operation and an optional error ## string for failures. - global close_backend: function(backend: opaque of Storage::BackendHandle): Storage::OperationResult; + global close_backend: function(backend: opaque of Storage::BackendHandle) + : Storage::OperationResult; - ## Inserts a new entry into a backend asynchronously. + ## Inserts a new entry into a backend asynchronously. This method must be called + ## via a :zeek:see:`when` condition or an error will be returned. ## ## backend: A handle to a backend connection. ## @@ -45,9 +49,11 @@ export { ## ## Returns: A record containing the status of the operation and an optional error ## string for failures. - global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): Storage::OperationResult; + global put: function(backend: opaque of Storage::BackendHandle, + args: Storage::PutArgs): Storage::OperationResult; - ## Gets an entry from the backend asynchronously. + ## Gets an entry from the backend asynchronously. This method must be called via a + ## :zeek:see:`when` condition or an error will be returned. ## ## backend: A handle to a backend connection. ## @@ -56,10 +62,12 @@ export { ## Returns: A record containing the status of the operation, an optional error ## string for failures, and an optional value for success. The value ## returned here will be of the type passed into - ## :zeek:see:`Storage::open_backend`. - global get: function(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult; + ## :zeek:see:`Storage::Async::open_backend`. + global get: function(backend: opaque of Storage::BackendHandle, key: any) + : Storage::OperationResult; - ## Erases an entry from the backend asynchronously. + ## Erases an entry from the backend asynchronously. This method must be called via + ## a :zeek:see:`when` condition or an error will be returned. ## ## backend: A handle to a backend connection. ## @@ -67,31 +75,37 @@ export { ## ## Returns: A record containing the status of the operation and an optional error ## string for failures. - global erase: function(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult; + global erase: function(backend: opaque of Storage::BackendHandle, key: any) + : Storage::OperationResult; } -function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, - val_type: any): Storage::OperationResult -{ +function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, + key_type: any, val_type: any): Storage::OperationResult + { return Storage::Async::__open_backend(btype, options, key_type, val_type); -} + } -function close_backend(backend: opaque of Storage::BackendHandle): Storage::OperationResult -{ +function close_backend(backend: opaque of Storage::BackendHandle) + : Storage::OperationResult + { return Storage::Async::__close_backend(backend); -} + } -function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): Storage::OperationResult -{ - return Storage::Async::__put(backend, args$key, args$value, args$overwrite, args$expire_time); -} +function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs) + : Storage::OperationResult + { + return Storage::Async::__put(backend, args$key, args$value, args$overwrite, + args$expire_time); + } -function get(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult -{ +function get(backend: opaque of Storage::BackendHandle, key: any) + : Storage::OperationResult + { return Storage::Async::__get(backend, key); -} + } -function erase(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult -{ +function erase(backend: opaque of Storage::BackendHandle, key: any) + : Storage::OperationResult + { return Storage::Async::__erase(backend, key); -} + } diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index ee928cbc28..7a11a6c25a 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -5,8 +5,10 @@ module Storage; export { - ## Base record for backend options. Backend plugins can redef this record to add - ## relevant fields to it. + ## Base record for backend options that can be passed to + ## :zeek:see:`Storage::Async::open_backend` and + ## :zeek:see:`Storage::Sync::open_backend`. Backend plugins can redef this record + ## to add relevant fields to it. type BackendOptions: record { }; ## Record for passing arguments to :zeek:see:`Storage::Async::put` and diff --git a/scripts/base/frameworks/storage/sync.zeek b/scripts/base/frameworks/storage/sync.zeek index 7eb5fa5a1d..acc06dfbb4 100644 --- a/scripts/base/frameworks/storage/sync.zeek +++ b/scripts/base/frameworks/storage/sync.zeek @@ -23,8 +23,9 @@ export { ## Returns: A record containing the status of the operation, and either an error ## string on failure or a value on success. The value returned here will ## be an ``opaque of BackendHandle``. - global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, - val_type: any): Storage::OperationResult; + global open_backend: function(btype: Storage::Backend, + options: Storage::BackendOptions, key_type: any, val_type: any) + : Storage::OperationResult; ## Closes an existing backend connection. ## @@ -32,18 +33,20 @@ export { ## ## Returns: A record containing the status of the operation and an optional error ## string for failures. - global close_backend: function(backend: opaque of Storage::BackendHandle): Storage::OperationResult; + global close_backend: function(backend: opaque of Storage::BackendHandle) + : Storage::OperationResult; ## Inserts a new entry into a backend. ## ## backend: A handle to a backend connection. ## ## args: A :zeek:see:`Storage::PutArgs` record containing the arguments for the - ## operation. + ## operation. ## ## Returns: A record containing the status of the operation and an optional error ## string for failures. - global put: function(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): Storage::OperationResult; + global put: function(backend: opaque of Storage::BackendHandle, + args: Storage::PutArgs): Storage::OperationResult; ## Gets an entry from the backend. ## @@ -54,8 +57,9 @@ export { ## Returns: A record containing the status of the operation, an optional error ## string for failures, and an optional value for success. The value ## returned here will be of the type passed into - ## :zeek:see:`Storage::open_backend`. - global get: function(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult; + ## :zeek:see:`Storage::Sync::open_backend`. + global get: function(backend: opaque of Storage::BackendHandle, key: any) + : Storage::OperationResult; ## Erases an entry from the backend. ## @@ -65,31 +69,37 @@ export { ## ## Returns: A record containing the status of the operation and an optional error ## string for failures. - global erase: function(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult; + global erase: function(backend: opaque of Storage::BackendHandle, key: any) + : Storage::OperationResult; } -function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, key_type: any, - val_type: any): Storage::OperationResult -{ +function open_backend(btype: Storage::Backend, options: Storage::BackendOptions, + key_type: any, val_type: any): Storage::OperationResult + { return Storage::Sync::__open_backend(btype, options, key_type, val_type); -} + } -function close_backend(backend: opaque of Storage::BackendHandle): Storage::OperationResult -{ +function close_backend(backend: opaque of Storage::BackendHandle) + : Storage::OperationResult + { return Storage::Sync::__close_backend(backend); -} + } -function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs): Storage::OperationResult -{ - return Storage::Sync::__put(backend, args$key, args$value, args$overwrite, args$expire_time); -} +function put(backend: opaque of Storage::BackendHandle, args: Storage::PutArgs) + : Storage::OperationResult + { + return Storage::Sync::__put(backend, args$key, args$value, args$overwrite, + args$expire_time); + } -function get(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult -{ +function get(backend: opaque of Storage::BackendHandle, key: any) + : Storage::OperationResult + { return Storage::Sync::__get(backend, key); -} + } -function erase(backend: opaque of Storage::BackendHandle, key: any): Storage::OperationResult -{ +function erase(backend: opaque of Storage::BackendHandle, key: any) + : Storage::OperationResult + { return Storage::Sync::__erase(backend, key); -} + } diff --git a/scripts/policy/frameworks/storage/backend/redis/main.zeek b/scripts/policy/frameworks/storage/backend/redis/main.zeek index a921fefc3a..2f037fb769 100644 --- a/scripts/policy/frameworks/storage/backend/redis/main.zeek +++ b/scripts/policy/frameworks/storage/backend/redis/main.zeek @@ -7,20 +7,20 @@ module Storage::Backend::Redis; export { ## Options record for the built-in Redis backend. type Options: record { - # Address or hostname of the server + # Address or hostname of the server. server_host: string &optional; - # Port for the server + # Port for the server. server_port: port &default=6379/tcp; - # Server unix socket file. This can be used instead of the - # address and port above to connect to a local server. + # Server unix socket file. This can be used instead of the address and + # port above to connect to a local server. In order to use this, the + # ``server_host`` field must be unset. server_unix_socket: string &optional; - # Prefix used in key values stored to differentiate varying - # types of data on the same server. Defaults to an empty string, - # but preferably should be set to a unique value per Redis - # backend opened. + # Prefix used in key values stored to differentiate varying types of data + # on the same server. Defaults to an empty string, but preferably should + # be set to a unique value per Redis backend opened. key_prefix: string &default=""; }; } diff --git a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek index 24e15fd857..81de566a8e 100644 --- a/scripts/policy/frameworks/storage/backend/sqlite/main.zeek +++ b/scripts/policy/frameworks/storage/backend/sqlite/main.zeek @@ -7,21 +7,22 @@ module Storage::Backend::SQLite; export { ## Options record for the built-in SQLite backend. type Options: record { - ## Path to the database file on disk. Setting this to ":memory:" - ## will tell SQLite to use an in-memory database. Relative paths - ## will be opened relative to the directory where Zeek was - ## started from. Zeek will not create intermediate directories - ## if they do not already exist. See - ## https://www.sqlite.org/c3ref/open.html for more rules on - ## paths that can be passed here. + ## Path to the database file on disk. Setting this to ":memory:" will tell + ## SQLite to use an in-memory database. Relative paths will be opened + ## relative to the directory where Zeek was started from. Zeek will not + ## create intermediate directories if they do not already exist. See + ## https://www.sqlite.org/c3ref/open.html for more rules on paths that can + ## be passed here. database_path: string; - ## Name of the table used for storing data. + ## Name of the table used for storing data. It is possible to use the same + ## database file for two separate tables, as long as the this value is + ## different between the two. table_name: string; - ## Key/value table for passing tuning parameters when opening - ## the database. These must be pairs that can be passed to the - ## ``pragma`` command in sqlite. + ## Key/value table for passing tuning parameters when opening the + ## database. These must be pairs that can be passed to the ``pragma`` + ## command in sqlite. tuning_params: table[string] of string &default=table( ["journal_mode"] = "WAL", ["synchronous"] = "normal", From c7015e8250dc53539241ab9f411ef49fac1f8fde Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 17 Mar 2025 14:38:50 -0700 Subject: [PATCH 50/52] Split storage.bif file into events/sync/async, add more comments --- scripts/base/frameworks/storage/main.zeek | 2 - src/storage/Backend.cc | 2 +- src/storage/CMakeLists.txt | 4 +- src/storage/backend/redis/Redis.cc | 1 - src/storage/storage-async.bif | 190 ++++++++++ src/storage/storage-events.bif | 31 ++ src/storage/storage-sync.bif | 165 +++++++++ src/storage/storage.bif | 343 ------------------ .../canonified_loaded_scripts.log | 4 +- .../canonified_loaded_scripts.log | 4 +- testing/btest/Baseline/plugins.hooks/output | 24 +- 11 files changed, 414 insertions(+), 356 deletions(-) create mode 100644 src/storage/storage-async.bif create mode 100644 src/storage/storage-events.bif create mode 100644 src/storage/storage-sync.bif delete mode 100644 src/storage/storage.bif diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index 7a11a6c25a..5c05503c33 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -1,7 +1,5 @@ ##! The storage framework provides a way to store long-term data to disk. -@load base/bif/storage.bif - module Storage; export { diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index f47e29eeea..53dd4c0335 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -5,7 +5,7 @@ #include "zeek/Trigger.h" #include "zeek/broker/Data.h" #include "zeek/storage/ReturnCode.h" -#include "zeek/storage/storage.bif.h" +#include "zeek/storage/storage-events.bif.h" namespace zeek::storage { diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index d1401a3ee1..b64c7d6016 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -6,6 +6,8 @@ zeek_add_subdir_library( Component.cc ReturnCode.cc BIFS - storage.bif) + storage-async.bif + storage-events.bif + storage-sync.bif) add_subdirectory(backend) diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 5d357239d2..f597a1370f 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -8,7 +8,6 @@ #include "zeek/Val.h" #include "zeek/iosource/Manager.h" #include "zeek/storage/ReturnCode.h" -#include "zeek/storage/storage.bif.h" #include "hiredis/adapters/poll.h" #include "hiredis/async.h" diff --git a/src/storage/storage-async.bif b/src/storage/storage-async.bif new file mode 100644 index 0000000000..5afca4feec --- /dev/null +++ b/src/storage/storage-async.bif @@ -0,0 +1,190 @@ +##! Functions related to asynchronous storage operations. + +%%{ +#include "zeek/Frame.h" +#include "zeek/Trigger.h" +#include "zeek/storage/Backend.h" +#include "zeek/storage/Manager.h" +#include "zeek/storage/ReturnCode.h" + +using namespace zeek; +using namespace zeek::storage; + +// Utility method for initializing a trigger from a Frame passed into a BIF. This is +// used by the asynchronous methods to make sure the trigger is setup before starting +// the operations. It also does some sanity checking to ensure the trigger is valid. + +static zeek::detail::trigger::TriggerPtr init_trigger(zeek::detail::Frame* frame) { + auto trigger = frame->GetTrigger(); + + if ( ! trigger ) { + emit_builtin_error("Asynchronous storage operations must be called via a when-condition"); + return nullptr; + } + + if ( auto timeout = trigger->TimeoutValue(); timeout < 0 ) { + emit_builtin_error("Asynchronous storage operations must specify a timeout block"); + return nullptr; + } + + frame->SetDelayed(); + trigger->Hold(); + + return {NewRef{}, trigger}; +} + +// Utility method to cast the handle val passed into BIF methods into a form that can +// be used to start storage operations. The method is also used by the BIFs in sync.bif. +static zeek::expected cast_handle(Val* handle) { + auto b = static_cast(handle); + + if ( ! b ) + return zeek::unexpected( + OperationResult{ReturnCode::OPERATION_FAILED, "Invalid storage handlle"}); + else if ( ! b->backend->IsOpen() ) + return zeek::unexpected(OperationResult{ReturnCode::NOT_CONNECTED, "Backend is closed"}); + + return b; +} + +static void handle_async_result(const IntrusivePtr& backend, ResultCallback* cb, + const OperationResult& op_result) { + if ( op_result.code != ReturnCode::IN_PROGRESS || ! backend->SupportsAsync() ) { + // We need to complete the callback early if: + // 1. The operation didn't start up successfully. For async operations, this means + // it didn't report back IN_PROGRESS. + // 2. The backend doesn't support async. This means we already blocked in order + // to get here already. + cb->Complete(op_result); + delete cb; + } + else if ( run_state::reading_traces ) { + // If the backend is truly async and we're reading traces, we need to fake being + // in sync mode because otherwise time doesn't move forward correctly. + backend->Poll(); + } +} + +%%} + +module Storage::Async; + +function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): Storage::OperationResult + %{ + auto trigger = init_trigger(frame); + if ( ! trigger ) + return nullptr; + + auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; + Tag tag{btype_val}; + + auto b = storage_mgr->Instantiate(tag); + + if ( ! b.has_value() ) { + trigger->Cache( + frame->GetTriggerAssoc(), + new StringVal(util::fmt("Failed to instantiate backend: %s", b.error().c_str()))); + trigger->Release(); + return nullptr; + } + + auto bh = make_intrusive(b.value()); + + auto cb = new OpenResultCallback(trigger, frame->GetTriggerAssoc(), bh); + auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; + auto op_result = storage_mgr->OpenBackend(b.value(), cb, options_val, kt, vt); + + handle_async_result(b.value(), cb, op_result); + + return nullptr; + %} + +function Storage::Async::__close_backend%(backend: opaque of Storage::BackendHandle%) : Storage::OperationResult + %{ + auto trigger = init_trigger(frame); + if ( ! trigger ) + return nullptr; + + auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); + auto b = cast_handle(backend); + if ( ! b ) { + cb->Complete(b.error()); + delete cb; + return nullptr; + } + + auto op_result = storage_mgr->CloseBackend((*b)->backend, cb); + handle_async_result((*b)->backend, cb, op_result); + + return nullptr; + %} + +function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, + overwrite: bool, expire_time: interval%): Storage::OperationResult + %{ + auto trigger = init_trigger(frame); + if ( ! trigger ) + return nullptr; + + auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); + auto b = cast_handle(backend); + if ( ! b ) { + cb->Complete(b.error()); + delete cb; + return nullptr; + } + + if ( expire_time > 0.0 ) + expire_time += run_state::network_time; + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto val_v = IntrusivePtr{NewRef{}, value}; + auto op_result = (*b)->backend->Put(cb, key_v, val_v, overwrite, expire_time); + handle_async_result((*b)->backend, cb, op_result); + + return nullptr; + %} + +function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult + %{ + auto trigger = init_trigger(frame); + if ( ! trigger ) + return nullptr; + + auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); + auto b = cast_handle(backend); + if ( ! b ) { + cb->Complete(b.error()); + delete cb; + return nullptr; + } + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto op_result = (*b)->backend->Get(cb, key_v); + handle_async_result((*b)->backend, cb, op_result); + + return nullptr; + %} + +function Storage::Async::__erase%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult + %{ + auto trigger = init_trigger(frame); + if ( ! trigger ) + return nullptr; + + auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); + auto b = cast_handle(backend); + if ( ! b ) { + cb->Complete(b.error()); + delete cb; + return nullptr; + } + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto op_result = (*b)->backend->Erase(cb, key_v); + handle_async_result((*b)->backend, cb, op_result); + + return nullptr; + %} diff --git a/src/storage/storage-events.bif b/src/storage/storage-events.bif new file mode 100644 index 0000000000..b0aa5c4f72 --- /dev/null +++ b/src/storage/storage-events.bif @@ -0,0 +1,31 @@ +##! Events related to storage operations. + +module Storage; + +## Generated automatically when a new backend connection is opened successfully. +## +## tag: A string describing the backend that enqueued this event. This is typically +## generated by the ``Tag()`` method in the backend plugin. +## +## options: A copy of the configuration options passed to +## :zeek:see:`Storage::Async::open_backend` or +## :zeek:see:`Storage::Sync::open_backend` when the backend was initially opened. +## +## .. zeek:see:: Storage::backend_lost +event Storage::backend_opened%(tag: string, options: any%); + +## May be generated when a backend connection is lost, both normally and +## unexpectedly. This event depends on the backends implementing handling for +## it, and is not generated automatically by the storage framework. +## +## tag: A string describing the backend that enqueued this event. This is typically +## generated by the ``Tag()`` method in the backend plugin. +## +## options: A copy of the configuration options passed to +## :zeek:see:`Storage::Async::open_backend` or +## :zeek:see:`Storage::Sync::open_backend` when the backend was initially opened. +## +## reason: A string describing why the connection was lost. +## +## .. zeek:see:: Storage::backend_opened +event Storage::backend_lost%(tag: string, options: any, reason: string%); diff --git a/src/storage/storage-sync.bif b/src/storage/storage-sync.bif new file mode 100644 index 0000000000..24ad138ee3 --- /dev/null +++ b/src/storage/storage-sync.bif @@ -0,0 +1,165 @@ +##! Functions related to synchronous storage operations. + +%%{ +#include "zeek/storage/Backend.h" +#include "zeek/storage/Manager.h" +#include "zeek/storage/ReturnCode.h" + +using namespace zeek; +using namespace zeek::storage; + +// Utility method to cast the handle val passed into BIF methods into a form that can +// be used to start storage operations. This is a duplicate of the method in sync.bif +// due to how utility methods are built by bifcl. +/* +static zeek::expected cast_handle(Val* handle) { + auto b = static_cast(handle); + + if ( ! b ) + return zeek::unexpected( + OperationResult{ReturnCode::OPERATION_FAILED, "Invalid storage handlle"}); + else if ( ! b->backend->IsOpen() ) + return zeek::unexpected(OperationResult{ReturnCode::NOT_CONNECTED, "Backend is closed"}); + + return b; +} +*/ +%%} + +module Storage::Sync; + +function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): Storage::OperationResult + %{ + auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; + Tag tag{btype_val}; + + auto b = storage_mgr->Instantiate(tag); + + if ( ! b.has_value() ) { + emit_builtin_error(b.error().c_str()); + return val_mgr->Bool(false); + } + + auto bh = make_intrusive(b.value()); + + auto cb = new OpenResultCallback(bh); + auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; + auto op_result = storage_mgr->OpenBackend(b.value(), cb, options_val, kt, vt); + + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! b.value()->SupportsSync() ) { + b.value()->Poll(); + op_result = cb->Result(); + } + + delete cb; + + return op_result.BuildVal(); + %} + +function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHandle%) : Storage::OperationResult + %{ + OperationResult op_result; + + auto b = cast_handle(backend); + if ( ! b ) + op_result = b.error(); + else { + auto cb = new OperationResultCallback(); + op_result = storage_mgr->CloseBackend((*b)->backend, cb); + + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! (*b)->backend->SupportsSync() ) { + (*b)->backend->Poll(); + op_result = cb->Result(); + } + + delete cb; + } + + return op_result.BuildVal(); + %} + +function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, + overwrite: bool, expire_time: interval%): Storage::OperationResult + %{ + OperationResult op_result; + + auto b = cast_handle(backend); + if ( ! b ) + op_result = b.error(); + else { + if ( expire_time > 0.0 ) + expire_time += run_state::network_time; + + auto cb = new OperationResultCallback(); + auto key_v = IntrusivePtr{NewRef{}, key}; + auto val_v = IntrusivePtr{NewRef{}, value}; + op_result = (*b)->backend->Put(cb, key_v, val_v, overwrite, expire_time); + + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! (*b)->backend->SupportsSync() ) { + (*b)->backend->Poll(); + op_result = cb->Result(); + } + + delete cb; + } + + return op_result.BuildVal(); + %} + +function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult + %{ + OperationResult op_result; + + auto b = cast_handle(backend); + if ( ! b ) + op_result = b.error(); + else { + auto cb = new OperationResultCallback(); + auto key_v = IntrusivePtr{NewRef{}, key}; + op_result = (*b)->backend->Get(cb, key_v); + + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! (*b)->backend->SupportsSync() ) { + (*b)->backend->Poll(); + op_result = cb->Result(); + } + + delete cb; + } + + return op_result.BuildVal(); + %} + +function Storage::Sync::__erase%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult + %{ + OperationResult op_result; + + auto b = cast_handle(backend); + if ( ! b ) + op_result = b.error(); + else { + auto cb = new OperationResultCallback(); + auto key_v = IntrusivePtr{NewRef{}, key}; + op_result = (*b)->backend->Erase(cb, key_v); + + // If the backend only supports async, block until it's ready and then pull the result out of + // the callback. + if ( ! (*b)->backend->SupportsSync() ) { + (*b)->backend->Poll(); + op_result = cb->Result(); + } + + delete cb; + } + + return op_result.BuildVal(); + %} diff --git a/src/storage/storage.bif b/src/storage/storage.bif deleted file mode 100644 index b32818734e..0000000000 --- a/src/storage/storage.bif +++ /dev/null @@ -1,343 +0,0 @@ -%%{ -#include "zeek/Trigger.h" -#include "zeek/Frame.h" -#include "zeek/storage/Backend.h" -#include "zeek/storage/Manager.h" -#include "zeek/storage/ReturnCode.h" - -using namespace zeek; -using namespace zeek::storage; - -static zeek::detail::trigger::TriggerPtr init_trigger(zeek::detail::Frame* frame) { - auto trigger = frame->GetTrigger(); - - if ( ! trigger ) { - emit_builtin_error("Asynchronous storage operations must be called via a when-condition"); - return nullptr; - } - - if ( auto timeout = trigger->TimeoutValue(); timeout < 0 ) { - emit_builtin_error("Async Storage operations must specify a timeout block"); - return nullptr; - } - - frame->SetDelayed(); - trigger->Hold(); - - return {NewRef{}, trigger}; -} - -static IntrusivePtr make_backend_handle(Val* btype) { - auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; - Tag tag{btype_val}; - - auto b = storage_mgr->Instantiate(tag); - - if ( ! b.has_value() ) { - emit_builtin_error(b.error().c_str()); - return nullptr; - } - - return make_intrusive(b.value()); -} - -static zeek::expected cast_handle(Val* handle) { - auto b = static_cast(handle); - - if ( ! b ) - return zeek::unexpected(OperationResult{ReturnCode::OPERATION_FAILED, "Invalid storage handlle"}); - else if ( ! b->backend->IsOpen() ) - return zeek::unexpected(OperationResult{ReturnCode::NOT_CONNECTED, "Backend is closed"}); - - return b; -} - -static void handle_async_result(const IntrusivePtr& backend, ResultCallback* cb, - const OperationResult& op_result) { - if ( op_result.code != ReturnCode::IN_PROGRESS || ! backend->SupportsAsync() ) { - // We need to complete the callback early if: - // 1. The operation didn't start up successfully. For async operations, this means - // it didn't report back IN_PROGRESS. - // 2. The backend doesn't support async. This means we already blocked in order - // to get here already. - cb->Complete(op_result); - delete cb; - } - else if ( run_state::reading_traces ) { - // If the backend is truly async and we're reading traces, we need to fake being in sync mode - // because otherwise time doesn't move forward correctly. - backend->Poll(); - } -} - -%%} - -module Storage; - -## Generated automatically when a new backend connection is opened successfully. -event Storage::backend_opened%(tag: string, options: any%); - -## May be generated when a backend connection is lost, both normally and -## unexpectedly. This event depends on the backends implementing handling for -## it, and is not generated automatically by the storage framework. -event Storage::backend_lost%(tag: string, options: any, reason: string%); - -module Storage::Async; - -function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): Storage::OperationResult - %{ - auto trigger = init_trigger(frame); - if ( ! trigger ) - return nullptr; - - auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; - Tag tag{btype_val}; - - auto b = storage_mgr->Instantiate(tag); - - if ( ! b.has_value() ) { - trigger->Cache( - frame->GetTriggerAssoc(), - new StringVal(util::fmt("Failed to instantiate backend: %s", b.error().c_str()))); - trigger->Release(); - return nullptr; - } - - auto bh = make_intrusive(b.value()); - - auto cb = new OpenResultCallback(trigger, frame->GetTriggerAssoc(), bh); - auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); - auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); - auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; - auto op_result = storage_mgr->OpenBackend(b.value(), cb, options_val, kt, vt); - - handle_async_result(b.value(), cb, op_result); - - return nullptr; - %} - -function Storage::Async::__close_backend%(backend: opaque of Storage::BackendHandle%) : Storage::OperationResult - %{ - auto trigger = init_trigger(frame); - if ( ! trigger ) - return nullptr; - - auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); - auto b = cast_handle(backend); - if ( ! b ) { - cb->Complete(b.error()); - delete cb; - return nullptr; - } - - auto op_result = storage_mgr->CloseBackend((*b)->backend, cb); - handle_async_result((*b)->backend, cb, op_result); - - return nullptr; - %} - -function Storage::Async::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, - overwrite: bool, expire_time: interval%): Storage::OperationResult - %{ - auto trigger = init_trigger(frame); - if ( ! trigger ) - return nullptr; - - auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); - auto b = cast_handle(backend); - if ( ! b ) { - cb->Complete(b.error()); - delete cb; - return nullptr; - } - - if ( expire_time > 0.0 ) - expire_time += run_state::network_time; - - auto key_v = IntrusivePtr{NewRef{}, key}; - auto val_v = IntrusivePtr{NewRef{}, value}; - auto op_result = (*b)->backend->Put(cb, key_v, val_v, overwrite, expire_time); - handle_async_result((*b)->backend, cb, op_result); - - return nullptr; - %} - -function Storage::Async::__get%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult - %{ - auto trigger = init_trigger(frame); - if ( ! trigger ) - return nullptr; - - auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); - auto b = cast_handle(backend); - if ( ! b ) { - cb->Complete(b.error()); - delete cb; - return nullptr; - } - - auto key_v = IntrusivePtr{NewRef{}, key}; - auto op_result = (*b)->backend->Get(cb, key_v); - handle_async_result((*b)->backend, cb, op_result); - - return nullptr; - %} - -function Storage::Async::__erase%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult - %{ - auto trigger = init_trigger(frame); - if ( ! trigger ) - return nullptr; - - auto cb = new OperationResultCallback(trigger, frame->GetTriggerAssoc()); - auto b = cast_handle(backend); - if ( ! b ) { - cb->Complete(b.error()); - delete cb; - return nullptr; - } - - auto key_v = IntrusivePtr{NewRef{}, key}; - auto op_result = (*b)->backend->Erase(cb, key_v); - handle_async_result((*b)->backend, cb, op_result); - - return nullptr; - %} - -module Storage::Sync; - -function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): Storage::OperationResult - %{ - auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; - Tag tag{btype_val}; - - auto b = storage_mgr->Instantiate(tag); - - if ( ! b.has_value() ) { - emit_builtin_error(b.error().c_str()); - return val_mgr->Bool(false); - } - - auto bh = make_intrusive(b.value()); - - auto cb = new OpenResultCallback(bh); - auto kt = key_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); - auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); - auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; - auto op_result = storage_mgr->OpenBackend(b.value(), cb, options_val, kt, vt); - - // If the backend only supports async, block until it's ready and then pull the result out of - // the callback. - if ( ! b.value()->SupportsSync() ) { - b.value()->Poll(); - op_result = cb->Result(); - } - - delete cb; - - return op_result.BuildVal(); - %} - -function Storage::Sync::__close_backend%(backend: opaque of Storage::BackendHandle%) : Storage::OperationResult - %{ - OperationResult op_result; - - auto b = cast_handle(backend); - if ( ! b ) - op_result = b.error(); - else { - auto cb = new OperationResultCallback(); - op_result = storage_mgr->CloseBackend((*b)->backend, cb); - - // If the backend only supports async, block until it's ready and then pull the result out of - // the callback. - if ( ! (*b)->backend->SupportsSync() ) { - (*b)->backend->Poll(); - op_result = cb->Result(); - } - - delete cb; - } - - return op_result.BuildVal(); - %} - -function Storage::Sync::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, - overwrite: bool, expire_time: interval%): Storage::OperationResult - %{ - OperationResult op_result; - - auto b = cast_handle(backend); - if ( ! b ) - op_result = b.error(); - else { - if ( expire_time > 0.0 ) - expire_time += run_state::network_time; - - auto cb = new OperationResultCallback(); - auto key_v = IntrusivePtr{NewRef{}, key}; - auto val_v = IntrusivePtr{NewRef{}, value}; - op_result = (*b)->backend->Put(cb, key_v, val_v, overwrite, expire_time); - - // If the backend only supports async, block until it's ready and then pull the result out of - // the callback. - if ( ! (*b)->backend->SupportsSync() ) { - (*b)->backend->Poll(); - op_result = cb->Result(); - } - - delete cb; - } - - return op_result.BuildVal(); - %} - -function Storage::Sync::__get%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult - %{ - OperationResult op_result; - - auto b = cast_handle(backend); - if ( ! b ) - op_result = b.error(); - else { - auto cb = new OperationResultCallback(); - auto key_v = IntrusivePtr{NewRef{}, key}; - op_result = (*b)->backend->Get(cb, key_v); - - // If the backend only supports async, block until it's ready and then pull the result out of - // the callback. - if ( ! (*b)->backend->SupportsSync() ) { - (*b)->backend->Poll(); - op_result = cb->Result(); - } - - delete cb; - } - - return op_result.BuildVal(); - %} - -function Storage::Sync::__erase%(backend: opaque of Storage::BackendHandle, key: any%): Storage::OperationResult - %{ - OperationResult op_result; - - auto b = cast_handle(backend); - if ( ! b ) - op_result = b.error(); - else { - auto cb = new OperationResultCallback(); - auto key_v = IntrusivePtr{NewRef{}, key}; - op_result = (*b)->backend->Erase(cb, key_v); - - // If the backend only supports async, block until it's ready and then pull the result out of - // the callback. - if ( ! (*b)->backend->SupportsSync() ) { - (*b)->backend->Poll(); - op_result = cb->Result(); - } - - delete cb; - } - - return op_result.BuildVal(); - %} diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index dedda53b20..e08c150e8d 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -160,7 +160,9 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek - build/scripts/base/bif/storage.bif.zeek + build/scripts/base/bif/storage-async.bif.zeek + build/scripts/base/bif/storage-events.bif.zeek + build/scripts/base/bif/storage-sync.bif.zeek build/scripts/base/bif/spicy.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index c6a813054b..febd84103a 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -160,7 +160,9 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek - build/scripts/base/bif/storage.bif.zeek + build/scripts/base/bif/storage-async.bif.zeek + build/scripts/base/bif/storage-events.bif.zeek + build/scripts/base/bif/storage-sync.bif.zeek build/scripts/base/bif/spicy.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index b66bf362d9..9f6e9b79be 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -505,7 +505,9 @@ 0.000000 MetaHookPost LoadFile(0, ./sftp, <...>/sftp.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./storage.bif.zeek, <...>/storage.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./storage-async.bif.zeek, <...>/storage-async.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./storage-events.bif.zeek, <...>/storage-events.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./storage-sync.bif.zeek, <...>/storage-sync.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./store, <...>/store.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./store.bif.zeek, <...>/store.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> -1 @@ -816,7 +818,9 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./sftp, <...>/sftp.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./storage.bif.zeek, <...>/storage.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./storage-async.bif.zeek, <...>/storage-async.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./storage-events.bif.zeek, <...>/storage-events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./storage-sync.bif.zeek, <...>/storage-sync.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./store, <...>/store.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> (-1, ) @@ -1460,7 +1464,9 @@ 0.000000 MetaHookPre LoadFile(0, ./sftp, <...>/sftp.zeek) 0.000000 MetaHookPre LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./storage.bif.zeek, <...>/storage.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./storage-async.bif.zeek, <...>/storage-async.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./storage-events.bif.zeek, <...>/storage-events.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./storage-sync.bif.zeek, <...>/storage-sync.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./store, <...>/store.zeek) 0.000000 MetaHookPre LoadFile(0, ./store.bif.zeek, <...>/store.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek) @@ -1771,7 +1777,9 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./sftp, <...>/sftp.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./storage.bif.zeek, <...>/storage.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./storage-async.bif.zeek, <...>/storage-async.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./storage-events.bif.zeek, <...>/storage-events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./storage-sync.bif.zeek, <...>/storage-sync.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./store, <...>/store.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek) @@ -2426,7 +2434,9 @@ 0.000000 | HookLoadFile ./sftp <...>/sftp.zeek 0.000000 | HookLoadFile ./spicy.bif.zeek <...>/spicy.bif.zeek 0.000000 | HookLoadFile ./stats.bif.zeek <...>/stats.bif.zeek -0.000000 | HookLoadFile ./storage.bif.zeek <...>/storage.bif.zeek +0.000000 | HookLoadFile ./storage-async.bif.zeek <...>/storage-async.bif.zeek +0.000000 | HookLoadFile ./storage-events.bif.zeek <...>/storage-events.bif.zeek +0.000000 | HookLoadFile ./storage-sync.bif.zeek <...>/storage-sync.bif.zeek 0.000000 | HookLoadFile ./store <...>/store.zeek 0.000000 | HookLoadFile ./store.bif.zeek <...>/store.bif.zeek 0.000000 | HookLoadFile ./strings.bif.zeek <...>/strings.bif.zeek @@ -2737,7 +2747,9 @@ 0.000000 | HookLoadFileExtended ./sftp <...>/sftp.zeek 0.000000 | HookLoadFileExtended ./spicy.bif.zeek <...>/spicy.bif.zeek 0.000000 | HookLoadFileExtended ./stats.bif.zeek <...>/stats.bif.zeek -0.000000 | HookLoadFileExtended ./storage.bif.zeek <...>/storage.bif.zeek +0.000000 | HookLoadFileExtended ./storage-async.bif.zeek <...>/storage-async.bif.zeek +0.000000 | HookLoadFileExtended ./storage-events.bif.zeek <...>/storage-events.bif.zeek +0.000000 | HookLoadFileExtended ./storage-sync.bif.zeek <...>/storage-sync.bif.zeek 0.000000 | HookLoadFileExtended ./store <...>/store.zeek 0.000000 | HookLoadFileExtended ./store.bif.zeek <...>/store.bif.zeek 0.000000 | HookLoadFileExtended ./strings.bif.zeek <...>/strings.bif.zeek From 8bca6a8594435ec142fd06b13aa37147f0dfb09e Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 17 Mar 2025 14:56:16 -0700 Subject: [PATCH 51/52] Cleanup/update comments across the storage C++ files --- src/storage/Backend.h | 137 +++++++++++++++++++++++++--- src/storage/Manager.cc | 5 +- src/storage/Manager.h | 31 ++++--- src/storage/backend/redis/Redis.h | 4 +- src/storage/backend/sqlite/SQLite.h | 4 +- 5 files changed, 147 insertions(+), 34 deletions(-) diff --git a/src/storage/Backend.h b/src/storage/Backend.h index 98578c43fb..ea8219b54c 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -14,35 +14,79 @@ namespace zeek::storage { class Manager; +/** + * A structure mapped to the script-level Storage::OperationResult type for returning + * status from storage operations. + */ struct OperationResult { + /** + * One of a set of return values used to return a code-based status. The default set + * of these values is automatically looked up by the `ReturnCode` class, but + * additional codes may be added by backends. See the script-level + * `Storage::ReturnCode` enum for documentation for the default available statuses. + */ EnumValPtr code; + + /** + * An optional error string that can be passed in the result in the case of failure. + */ std::string err_str; + + /** + * A generic value pointer for operations that can return values, such as `Open()` and + * `Get()`. + */ ValPtr value; + /** + * Returns a RecordVal of the script-level type `Storage::OperationResult` from the + * values stored. + */ RecordValPtr BuildVal(); + + /** + * Static version of `BuildVal()` that returns a RecordVal of the script-level type + * `Storage::OperationResult` from the values provided. + */ static RecordValPtr MakeVal(EnumValPtr code, std::string_view err_str = "", ValPtr value = nullptr); }; - -// Base callback object for async operations. This is just here to allow some -// code reuse in the other callback methods. +/** + * Base callback object for asynchronous operations. + */ class ResultCallback { public: ResultCallback() = default; ResultCallback(detail::trigger::TriggerPtr trigger, const void* assoc); virtual ~ResultCallback() = default; + + /** + * Called on the callback when an operation times out. Sets the resulting status to + * TIMEOUT and times out the trigger. + */ void Timeout(); + + /** + * Returns whether the callback was created in an async context. This can be used to + * determine whether an operation was called synchronously or asynchronously. + */ bool IsSyncCallback() const { return ! trigger; } + /** + * Completes a callback, releasing the trigger if it was valid or storing the result + * for later usage if needed. + */ virtual void Complete(OperationResult res) = 0; protected: - void CompleteWithVal(Val* result); - zeek::detail::trigger::TriggerPtr trigger; const void* assoc = nullptr; }; +/** + * A callback that returns an `OperationResult` when it is complete. This is used by most + * of the storage operations for returning status. + */ class OperationResultCallback : public ResultCallback { public: OperationResultCallback() = default; @@ -56,6 +100,10 @@ private: class OpenResultCallback; +/** + * A list of available modes that backends can support. A combination of these is passed + * to `Backend::Backend` during plugin initialization. + */ enum SupportedModes : uint8_t { SYNC = 0x01, ASYNC = 0x02 }; class Backend : public zeek::Obj { @@ -68,13 +116,14 @@ public: /** * Store a new key/value pair in the backend. * - * @param key the key for the pair. - * @param value the value for the pair. + * @param cb A callback object for returning status if being called via an async + * context. + * @param key the key for the data being inserted. + * @param value the value for the data being inserted. * @param overwrite whether an existing value for a key should be overwritten. * @param expiration_time the time when this entry should be automatically * removed. Set to zero to disable expiration. This time is based on the current network * time. - * @param cb An optional callback object if being called via an async context. * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ @@ -84,8 +133,9 @@ public: /** * Retrieve a value from the backend for a provided key. * + * @param cb A callback object for returning status if being called via an async + * context. * @param key the key to lookup in the backend. - * @param cb An optional callback object if being called via an async context. * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ @@ -94,8 +144,9 @@ public: /** * Erases the value for a key from the backend. * + * @param cb A callback object for returning status if being called via an async + * context. * @param key the key to erase - * @param cb An optional callback object if being called via an async context. * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ @@ -115,6 +166,10 @@ public: */ void Poll() { DoPoll(); } + /** + * Returns the options record that was passed to `Manager::OpenBackend` when the + * backend was opened. + */ const RecordValPtr& Options() const { return backend_options; } protected: @@ -137,12 +192,13 @@ protected: /** * Called by the manager system to open the backend. * + * @param cb A callback object for returning status if being called via an async + * context. * @param options A record storing configuration options for the backend. * @param kt The script-side type of the keys stored in the backend. Used for * validation of types. * @param vt The script-side type of the values stored in the backend. Used for * validation of types and conversion during retrieval. - * @param cb An optional callback object if being called via an async context. * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ @@ -151,7 +207,8 @@ protected: /** * Finalizes the backend when it's being closed. * - * @param cb An optional callback object if being called via an async context. + * @param cb A callback object for returning status if being called via an async + * context. * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ @@ -179,6 +236,11 @@ protected: */ void EnqueueBackendLost(std::string_view reason); + /** + * Completes a callback and cleans up the memory if the callback was from a sync + * context. This should be called by backends instead of calling the callback's + * \a`Complete` method directly. + */ void CompleteCallback(ResultCallback* cb, const OperationResult& data) const; TypePtr key_type; @@ -188,13 +250,52 @@ protected: std::string tag; private: + /** + * Workhorse method for calls to `Manager::OpenBackend()`. See that method for + * documentation of the arguments. This must be overridden by all backends. + */ virtual OperationResult DoOpen(OpenResultCallback* cb, RecordValPtr options) = 0; + + /** + * Workhorse method for calls to `Manager::CloseBackend()`. See that method for + * documentation of the arguments. This must be overridden by all backends. + */ virtual OperationResult DoClose(OperationResultCallback* cb) = 0; - virtual OperationResult DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite = true, - double expiration_time = 0) = 0; + + /** + * Workhorse method for calls to `Backend::Put()`. See that method for + * documentation of the arguments. This must be overridden by all backends. + */ + virtual OperationResult DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite, + double expiration_time) = 0; + + /** + * Workhorse method for calls to `Backend::Get()`. See that method for + * documentation of the arguments. This must be overridden by all backends. + */ virtual OperationResult DoGet(OperationResultCallback* cb, ValPtr key) = 0; + + /** + * Workhorse method for calls to `Backend::Erase()`. See that method for + * documentation of the arguments. This must be overridden by all backends. + */ virtual OperationResult DoErase(OperationResultCallback* cb, ValPtr key) = 0; + + /** + * Optional method for backends to override to provide direct polling. This should be + * implemented to support synchronous operations on backends that only provide + * asynchronous communication. See the built-in Redis backend for an example. + */ virtual void DoPoll() {} + + /** + * Optional method for backends to override to provide non-native expiration of + * items. This is called by the manager on a timer. This can also be used to implement + * expiration when reading packet captures. + * + * @param current_network_time The current network time at which expiration is + * happening. + */ virtual void DoExpire(double current_network_time) {} uint8_t modes; @@ -206,6 +307,9 @@ namespace detail { extern OpaqueTypePtr backend_opaque; +/** + * OpaqueVal interface for returning BackendHandle objects to script-land. + */ class BackendHandleVal : public OpaqueVal { public: BackendHandleVal() : OpaqueVal(detail::backend_opaque) {} @@ -222,7 +326,10 @@ protected: } // namespace detail -// A callback for the Backend::Open() method that returns an error or a backend handle. +/** + * A specialized version of callback for returning from `open` operations. This returns a + * `BackendHandleVal` in the `value` field of the result when successful. + */ class OpenResultCallback : public ResultCallback { public: OpenResultCallback(IntrusivePtr backend); diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index b36e073606..68d6df9f89 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -8,8 +8,6 @@ #include "zeek/RunState.h" #include "zeek/storage/ReturnCode.h" -#include "const.bif.netvar_h" - std::atomic_flag expire_running; namespace zeek::storage { @@ -39,6 +37,9 @@ Manager::~Manager() { // Don't leave all of these static objects to leak. ReturnCode::Cleanup(); + + // NOTE: The expiration_thread object is a jthread and will be automatically joined + // here as the object is destroyed. } void Manager::InitPostScript() { diff --git a/src/storage/Manager.h b/src/storage/Manager.h index a382c533ba..44d91a194b 100644 --- a/src/storage/Manager.h +++ b/src/storage/Manager.h @@ -30,19 +30,18 @@ public: ~Manager(); /** - * Initialization of the manager. This is called late during Zeek's - * initialization after any scripts are processed. + * Initialization of the manager. This is called late during Zeek's initialization + * after any scripts are processed. */ void InitPostScript(); /** - * Instantiates a new backend object. The backend will be in a closed state, - * and OpenBackend() will need to be called to fully initialize it. + * Instantiates a new backend object. The backend will be in a closed state, and + * OpenBackend() will need to be called to fully initialize it. * * @param type The tag for the type of backend being opened. - * @return A std::expected containing either a valid BackendPtr with the - * result of the operation or a string containing an error message for - * failure. + * @return A std::expected containing either a valid BackendPtr with the result of the + * operation or a string containing an error message for failure. */ zeek::expected Instantiate(const Tag& type); @@ -50,13 +49,13 @@ public: * Opens a new storage backend. * * @param backend The backend object to open. - * @param options A record val representing the configuration for this type of - * backend. + * @param cb A callback object for returning status if being called via an async + * context. * @param key_type The script-side type of the keys stored in the backend. Used for - * validation of types. + * validation of types for `key` arguments during all operations. * @param val_type The script-side type of the values stored in the backend. Used for - * validation of types and conversion during retrieval. - * @param cb An optional callback object if being called via an async context. + * validation of types for `put` operations and type conversion during `get` + * operations. * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ @@ -67,12 +66,18 @@ public: * Closes a storage backend. * * @param backend A pointer to the backend being closed. - * @param cb An optional callback object if being called via an async context. + * @param cb A callback object for returning status if being called via an async + * context. * @return A struct describing the result of the operation, containing a code, an * optional error string, and a ValPtr for operations that return values. */ OperationResult CloseBackend(BackendPtr backend, OperationResultCallback* cb); + /** + * Runs an expire operation on all open backends. This is called by the expiration + * timer and shouldn't be called directly otherwise, since it should only happen on a + * separate thread. + */ void Expire(); protected: diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index 2af6d9f501..bab7cdfddc 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -52,8 +52,8 @@ public: private: OperationResult DoOpen(OpenResultCallback* cb, RecordValPtr options) override; OperationResult DoClose(OperationResultCallback* cb) override; - OperationResult DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite = true, - double expiration_time = 0) override; + OperationResult DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite, + double expiration_time) override; OperationResult DoGet(OperationResultCallback* cb, ValPtr key) override; OperationResult DoErase(OperationResultCallback* cb, ValPtr key) override; void DoExpire(double current_network_time) override; diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index e8d1781ec2..960b2eb6c7 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -25,8 +25,8 @@ public: private: OperationResult DoOpen(OpenResultCallback* cb, RecordValPtr options) override; OperationResult DoClose(OperationResultCallback* cb) override; - OperationResult DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite = true, - double expiration_time = 0) override; + OperationResult DoPut(OperationResultCallback* cb, ValPtr key, ValPtr value, bool overwrite, + double expiration_time) override; OperationResult DoGet(OperationResultCallback* cb, ValPtr key) override; OperationResult DoErase(OperationResultCallback* cb, ValPtr key) override; void DoExpire(double current_network_time) override; From 6fa2202826bcaf1cb175143c84925647ba370366 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 17 Mar 2025 17:07:22 -0700 Subject: [PATCH 52/52] Update docs submodule [nomail] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 625c534db5..66a9cf6283 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 625c534db57c54b1eaf410eb63e0e261ecad3df0 +Subproject commit 66a9cf6283e3086c9f95bbe114abcc20e172e119