diff --git a/CMakeLists.txt b/CMakeLists.txt index da8ad5a5f8..88d90578a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1143,6 +1143,7 @@ include(FindKqueue) include(FindPrometheusCpp) include_directories(BEFORE "auxil/out_ptr/include") +include_directories(BEFORE "auxil/expected-lite/include") if ((OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0")) set(ZEEK_HAVE_OPENSSL_1_1 true CACHE INTERNAL "" FORCE) diff --git a/scripts/base/frameworks/storage/__load__.zeek b/scripts/base/frameworks/storage/__load__.zeek new file mode 100644 index 0000000000..d551be57d3 --- /dev/null +++ b/scripts/base/frameworks/storage/__load__.zeek @@ -0,0 +1 @@ +@load ./main \ No newline at end of file diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek new file mode 100644 index 0000000000..e879c64504 --- /dev/null +++ b/scripts/base/frameworks/storage/main.zeek @@ -0,0 +1,85 @@ +##! The storage framework provides a way to store long-term data to disk. + +@load base/bif/storage.bif + +module Storage; + +export { + ## Opens a new backend connection based on a configuration object. + ## + ## btype: A tag indicating what type of backend should be opened. + ## + ## options: A record containing the configuration for the connection. + ## + ## Returns: A handle to the new backend connection, or null if the + ## connection failed. + global open_backend: function(btype: Storage::Backend, options: any): opaque of Storage::BackendHandle; + + ## Closes an existing backend connection. + ## + ## backend: A handle to a backend connection. + ## + ## Returns: A boolean indicating success or failure of the operation. + global close_backend: function(backend: opaque of Storage::BackendHandle): bool; + + ## Inserts a new entry into a backend. + ## + ## backend: A handle to a backend connection. + ## + ## key: A key value. + ## + ## value: A corresponding value. + ## + ## overwrite: A flag indicating whether this value should overwrite an + ## existing entry for the key. + ## + ## Returns: A boolean indicating success or failure of the operation. + global put: function(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool): bool; + + ## Gets an entry from the backend. + ## + ## backend: A handle to a backend connection. + ## + ## key: The key to look up. + ## + ## val_type: The type of the value to return. + ## + ## Returns: A boolean indicating success or failure of the + ## operation. Type conversion failures for the value will + ## return false. + global get: function(backend: opaque of Storage::BackendHandle, key: any, val_type: any): any; + + ## Erases an entry from the backend. + ## + ## backend: A handle to a backend connection. + ## + ## key: The key to erase. + ## + ## Returns: A boolean indicating success or failure of the operation. + global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool; +} + +function open_backend(btype: Storage::Backend, options: any): opaque of Storage::BackendHandle +{ + return Storage::__open_backend(btype, options); +} + +function close_backend(backend: opaque of Storage::BackendHandle): bool +{ + return Storage::__close_backend(backend); +} + +function put(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool): bool +{ + return Storage::__put(backend, key, value, overwrite); +} + +function get(backend: opaque of Storage::BackendHandle, key: any, val_type: any): any +{ + return Storage::__get(backend, key, val_type); +} + +function erase(backend: opaque of Storage::BackendHandle, key: any): bool +{ + return Storage::__erase(backend, key); +} diff --git a/scripts/base/init-default.zeek b/scripts/base/init-default.zeek index 35c75875d2..90ccaf3445 100644 --- a/scripts/base/init-default.zeek +++ b/scripts/base/init-default.zeek @@ -44,6 +44,7 @@ @load base/frameworks/openflow @load base/frameworks/netcontrol @load base/frameworks/telemetry +@load base/frameworks/storage @if ( have_spicy() ) @load base/frameworks/spicy diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 356a6eff4c..67347bafca 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -205,6 +205,7 @@ add_subdirectory(iosource) add_subdirectory(logging) add_subdirectory(probabilistic) add_subdirectory(session) +add_subdirectory(storage) if (HAVE_SPICY) add_subdirectory(spicy) diff --git a/src/script_opt/FuncInfo.cc b/src/script_opt/FuncInfo.cc index 8d836facb3..dea2ba7524 100644 --- a/src/script_opt/FuncInfo.cc +++ b/src/script_opt/FuncInfo.cc @@ -148,6 +148,11 @@ static std::unordered_map func_attrs = { {"Reporter::warning", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Spicy::__resource_usage", ATTR_NO_ZEEK_SIDE_EFFECTS}, {"Spicy::__toggle_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::__close_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::__erase", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::__get", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::__open_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"Storage::__put", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Supervisor::__create", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Supervisor::__destroy", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Supervisor::__is_supervised", ATTR_IDEMPOTENT}, diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc new file mode 100644 index 0000000000..fa9e3e2487 --- /dev/null +++ b/src/storage/Backend.cc @@ -0,0 +1,42 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/Backend.h" + +#include "zeek/broker/Data.h" + +namespace zeek::storage { + +ErrorResult Backend::Open(RecordValPtr options) { return DoOpen(std::move(options)); } + +ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite) { + // The intention for this method is to do some other heavy lifting in regard + // to backends that need to pass data through the manager instead of directly + // through the workers. For the first versions of the storage framework it + // just calls the backend itself directly. + return DoPut(std::move(key), std::move(value), overwrite); +} + +ValResult Backend::Get(ValPtr key, TypePtr value_type) { + // See the note in Put(). + return DoGet(std::move(key), std::move(value_type)); +} + +ErrorResult Backend::Erase(ValPtr key) { + // See the note in Put(). + return DoErase(std::move(key)); +} + +zeek::OpaqueTypePtr detail::backend_opaque; +IMPLEMENT_OPAQUE_VALUE(detail::BackendHandleVal) + +std::optional detail::BackendHandleVal::DoSerializeData() const { + // Cannot serialize. + return std::nullopt; +} + +bool detail::BackendHandleVal::DoUnserializeData(BrokerDataView) { + // Cannot unserialize. + return false; +} + +} // namespace zeek::storage diff --git a/src/storage/Backend.h b/src/storage/Backend.h new file mode 100644 index 0000000000..fedabe08c5 --- /dev/null +++ b/src/storage/Backend.h @@ -0,0 +1,137 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/OpaqueVal.h" +#include "zeek/Val.h" +#include "zeek/util.h" + +namespace zeek::storage { + +class Manager; + +// Result from storage operations that may return an error message. If the +// optional value is unset, the operation succeeded. +using ErrorResult = std::optional; + +// Result from storage operations that return Vals. The ValPtr is an +// IntrusivePtr to some result, and can be null if the operation failed. The +// string value will store an error message if the result is null. +using ValResult = zeek::expected; + +class Backend : public zeek::Obj { +public: + /** + * Returns a descriptive tag representing the source for debugging. + */ + const char* Tag() { return tag.c_str(); } + + /** + * Store a new key/value pair in the backend. + * + * @param key the key for the pair + * @param value the value for the pair + * @param overwrite whether an existing value for a key should be overwritten. + * @return A result pair containing a bool with the success state, and a + * possible error string if the operation failed. + */ + ErrorResult Put(ValPtr key, ValPtr value, bool overwrite = true); + + /** + * Retrieve a value from the backend for a provided key. + * + * @param key the key to lookup in the backend. + * @param value_type The script-land type to be used when retrieving values + * from the backend. + * @return A result pair containing a ValPtr with the resulting value or + * nullptr retrieval failed, and a string with the error message if the + * operation failed. + */ + ValResult Get(ValPtr key, TypePtr value_type); + + /** + * Erases the value for a key from the backend. + * + * @return An optional value potentially containing an error string if + * needed. Will be unset if the operation succeeded. + * possible error string if the operation failed. + */ + ErrorResult Erase(ValPtr key); + + /** + * Returns whether the backend is opened. + */ + virtual bool IsOpen() = 0; + +protected: + // Allow the manager to call Open/Close. + friend class storage::Manager; + + /** + * Constructor + * + * @param tag A string representation of the tag for this backend. This + * is passed from the Manager through the component factory. + */ + Backend(std::string_view tag) : tag(tag) {} + + /** + * Called by the manager system to open the backend. + * + * @param options A record storing configuration options for the backend. + * @return A result pair containing a bool with the success state, and a + * possible error string if the operation failed. + */ + ErrorResult Open(RecordValPtr options); + + /** + * Finalizes the backend when it's being closed. Can be overridden by + * derived classes. + */ + virtual void Close() {} + + /** + * The workhorse method for Open(). + */ + virtual ErrorResult DoOpen(RecordValPtr options) = 0; + + /** + * The workhorse method for Put(). + */ + virtual ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true) = 0; + + /** + * The workhorse method for Get(). + */ + virtual ValResult DoGet(ValPtr key, TypePtr vt) = 0; + + /** + * The workhorse method for Erase(). + */ + virtual ErrorResult DoErase(ValPtr key) = 0; + + std::string tag; +}; + +using BackendPtr = zeek::IntrusivePtr; + +namespace detail { + +extern OpaqueTypePtr backend_opaque; + +class BackendHandleVal : public OpaqueVal { +public: + BackendHandleVal() : OpaqueVal(detail::backend_opaque) {} + BackendHandleVal(BackendPtr backend) : OpaqueVal(detail::backend_opaque), backend(std::move(backend)) {} + ~BackendHandleVal() override = default; + + BackendPtr backend; + +protected: + IntrusivePtr DoClone(CloneState* state) override { return {NewRef{}, this}; } + + DECLARE_OPAQUE_VALUE_DATA(BackendHandleVal) +}; + +} // namespace detail +} // namespace zeek::storage diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt new file mode 100644 index 0000000000..86071e81bd --- /dev/null +++ b/src/storage/CMakeLists.txt @@ -0,0 +1,10 @@ +zeek_add_subdir_library( + storage + SOURCES + Manager.cc + Backend.cc + Component.cc + BIFS + storage.bif) + +add_subdirectory(backend) diff --git a/src/storage/Component.cc b/src/storage/Component.cc new file mode 100644 index 0000000000..1f5bb1da6e --- /dev/null +++ b/src/storage/Component.cc @@ -0,0 +1,25 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/Component.h" + +#include "zeek/Desc.h" +#include "zeek/storage/Manager.h" + +namespace zeek::storage { + +Component::Component(const std::string& name, factory_callback arg_factory) + : plugin::Component(plugin::component::STORAGE_BACKEND, name, 0, storage_mgr->GetTagType()) { + factory = arg_factory; +} + +void Component::Initialize() { + InitializeTag(); + storage_mgr->RegisterComponent(this); +} + +void Component::DoDescribe(ODesc* d) const { + d->Add("Storage::STORAGE_BACKEND_"); + d->Add(CanonicalName()); +} + +} // namespace zeek::storage diff --git a/src/storage/Component.h b/src/storage/Component.h new file mode 100644 index 0000000000..de8c9899ce --- /dev/null +++ b/src/storage/Component.h @@ -0,0 +1,59 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/plugin/Component.h" + +namespace zeek::storage { + +class Backend; + +/** + * Component description for plugins providing storage backends. + */ +class Component : public plugin::Component { +public: + using factory_callback = IntrusivePtr (*)(std::string_view); + + /** + * Constructor. + * + * @param name The name of the provided backend. This name is used + * across the system to identify the backend. + * + * @param factory A factory function to instantiate instances of the + * backend's class, which must be derived directly or indirectly from + * storage::Backend. This is typically a static \c Instantiate() + * method inside the class that just allocates and returns a new + * instance. + */ + Component(const std::string& name, factory_callback factory); + + /** + * Destructor. + */ + ~Component() override = default; + + /** + * Initialization function. This function has to be called before any + * plugin component functionality is used; it is used to add the + * plugin component to the list of components and to initialize tags + */ + void Initialize() override; + + /** + * Returns the backend's factory function. + */ + factory_callback Factory() const { return factory; } + +protected: + /** + * Overridden from plugin::Component. + */ + void DoDescribe(ODesc* d) const override; + +private: + factory_callback factory; +}; + +} // namespace zeek::storage diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc new file mode 100644 index 0000000000..8a52c04271 --- /dev/null +++ b/src/storage/Manager.cc @@ -0,0 +1,58 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/Manager.h" + +#include "zeek/Desc.h" + +namespace zeek::storage { + +Manager::Manager() : plugin::ComponentManager("Storage", "Backend") {} + +void Manager::InitPostScript() { detail::backend_opaque = make_intrusive("Storage::Backend"); } + +zeek::expected Manager::OpenBackend(const Tag& type, RecordValPtr options) { + Component* c = Lookup(type); + if ( ! c ) { + return zeek::unexpected( + util::fmt("Request to open unknown backend (%d:%d)", type.Type(), type.Subtype())); + } + + if ( ! c->Factory() ) { + return zeek::unexpected( + util::fmt("Factory invalid for backend %s", GetComponentName(type).c_str())); + } + + ODesc d; + type.AsVal()->Describe(&d); + + BackendPtr bp = c->Factory()(d.Description()); + + if ( ! bp ) { + return zeek::unexpected( + util::fmt("Failed to instantiate backend %s", GetComponentName(type).c_str())); + } + + if ( auto res = bp->Open(std::move(options)); res.has_value() ) { + return zeek::unexpected( + util::fmt("Failed to open backend %s: %s", GetComponentName(type).c_str(), res.value().c_str())); + } + + // TODO: post Storage::backend_opened event + + backends.push_back(bp); + + return bp; +} + +void Manager::CloseBackend(BackendPtr backend) { + auto it = std::find(backends.begin(), backends.end(), backend); + if ( it == backends.end() ) + return; + + backends.erase(it); + backend->Close(); + + // TODO: post Storage::backend_lost event +} + +} // namespace zeek::storage diff --git a/src/storage/Manager.h b/src/storage/Manager.h new file mode 100644 index 0000000000..74cbbc8bee --- /dev/null +++ b/src/storage/Manager.h @@ -0,0 +1,52 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/plugin/ComponentManager.h" +#include "zeek/storage/Backend.h" +#include "zeek/storage/Component.h" + +namespace zeek::storage { + +class Manager final : public plugin::ComponentManager { +public: + Manager(); + ~Manager() = default; + + /** + * Initialization of the manager. This is called late during Zeek's + * initialization after any scripts are processed. + */ + void InitPostScript(); + + /** + * Opens a new storage backend. + * + * @param type The tag for the type of backend being opened. + * @param options A record val representing the configuration for this + * type of backend. + * @return A pair containing a pointer to a backend and a string for + * returning error messages if needed. + */ + zeek::expected OpenBackend(const Tag& type, RecordValPtr options); + + /** + * Closes a storage backend. + */ + void CloseBackend(BackendPtr backend); + + // TODO: + // - Hooks for storage-backed tables? + // - Handling aggregation from workers on a single manager? + +private: + std::vector backends; +}; + +} // namespace zeek::storage + +namespace zeek { + +extern storage::Manager* storage_mgr; + +} // namespace zeek diff --git a/src/storage/backend/CMakeLists.txt b/src/storage/backend/CMakeLists.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/src/storage/backend/CMakeLists.txt @@ -0,0 +1 @@ + diff --git a/src/storage/storage.bif b/src/storage/storage.bif new file mode 100644 index 0000000000..326ae9e114 --- /dev/null +++ b/src/storage/storage.bif @@ -0,0 +1,116 @@ +%%{ +#include "zeek/storage/Backend.h" +#include "zeek/storage/Manager.h" + +using namespace zeek; +using namespace zeek::storage; +%%} + +module Storage; + +# Generated when a new backend connection is opened +event Storage::backend_opened%(%); + +# Generated when a backend connection is lost +event Storage::backend_lost%(%); + +function Storage::__open_backend%(btype: Storage::Backend, options: any%): opaque of Storage::BackendHandle + %{ + auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; + Tag tag{btype_val}; + + auto options_val = IntrusivePtr{NewRef{}, options->AsRecordVal()}; + + auto b = storage_mgr->OpenBackend(tag, options_val); + + if ( ! b.has_value() ) { + emit_builtin_error(b.error().c_str()); + return val_mgr->Bool(false); + } + + return make_intrusive(b.value()); + %} + +function Storage::__close_backend%(backend: opaque of Storage::BackendHandle%) : bool + %{ + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + // Return true here since the backend is already closed + return val_mgr->Bool(true); + + storage_mgr->CloseBackend(b->backend); + + return val_mgr->Bool(true); + %} + +function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool%): bool + %{ + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + return val_mgr->Bool(false); + + // TODO: add support for when statements (see broker/store.bif) + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto val_v = IntrusivePtr{NewRef{}, value}; + auto result = b->backend->Put(key_v, val_v, overwrite); + if ( result.has_value() ) { + emit_builtin_error(util::fmt("Failed to store data: %s", result.value().c_str())); + return val_mgr->Bool(false); + } + + return val_mgr->Bool(true); + %} + +function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, val_type: any%): any + %{ + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + return val_mgr->Bool(false); + + // TODO: add support for when statements (see broker/store.bif) + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType(); + auto result = b->backend->Get(key_v, vt); + if ( ! result.has_value() ) { + emit_builtin_error(util::fmt("Failed to retrieve data: %s", result.error().c_str())); + return val_mgr->Bool(false); + } + + return result.value(); + %} + +function Storage::__erase%(backend: opaque of Storage::BackendHandle, key: any%): bool + %{ + auto b = dynamic_cast(backend); + if ( ! b ) { + emit_builtin_error("Invalid storage handle", backend); + return val_mgr->Bool(false); + } + else if ( ! b->backend->IsOpen() ) + return val_mgr->Bool(false); + + // TODO: add support for when statements (see broker/store.bif) + + auto key_v = IntrusivePtr{NewRef{}, key}; + auto result = b->backend->Erase(key_v); + if ( result.has_value() ) { + emit_builtin_error(util::fmt("Failed to erase data for key: %s", result.value().c_str())); + return val_mgr->Bool(false); + } + + return val_mgr->Bool(true); + %} diff --git a/src/util.h b/src/util.h index f261506310..1c493949f1 100644 --- a/src/util.h +++ b/src/util.h @@ -97,6 +97,15 @@ namespace filesystem = ghc::filesystem; inline constexpr std::string_view path_list_separator = ":"; #endif +#include "zeek/3rdparty/nonstd/expected.hpp" +namespace zeek { +template +using expected = nonstd::expected; + +template +using unexpected = nonstd::unexpected; +} // namespace zeek + using zeek_int_t = int64_t; using zeek_uint_t = uint64_t; diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 05e4e0fb52..ba5dc89e1f 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -65,6 +65,7 @@ #ifdef HAVE_SPICY #include "zeek/spicy/manager.h" #endif +#include "zeek/storage/Manager.h" #include "zeek/supervisor/Supervisor.h" #include "zeek/telemetry/Manager.h" #include "zeek/threading/Manager.h" @@ -178,6 +179,7 @@ zeek::detail::trigger::Manager* zeek::detail::trigger_mgr = nullptr; #ifdef HAVE_SPICY zeek::spicy::Manager* zeek::spicy_mgr = nullptr; #endif +zeek::storage::Manager* zeek::storage_mgr = nullptr; zeek::cluster::Manager* zeek::cluster::manager = nullptr; zeek::cluster::Backend* zeek::cluster::backend = nullptr; @@ -414,6 +416,7 @@ static void terminate_zeek() { #ifdef HAVE_SPICY delete spicy_mgr; #endif + delete storage_mgr; // free the global scope pop_scope(); @@ -686,6 +689,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) { #ifdef HAVE_SPICY spicy_mgr = new spicy::Manager(); // registers as plugin with the plugin manager #endif + storage_mgr = new storage::Manager(); plugin_mgr->InitPreScript(); file_mgr->InitPreScript(); @@ -873,6 +877,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) { timer_mgr->InitPostScript(); event_mgr.InitPostScript(); + storage_mgr->InitPostScript(); if ( supervisor_mgr ) supervisor_mgr->InitPostScript(); diff --git a/src/zeekygen/ScriptInfo.cc b/src/zeekygen/ScriptInfo.cc index 70347baa88..71464fa87d 100644 --- a/src/zeekygen/ScriptInfo.cc +++ b/src/zeekygen/ScriptInfo.cc @@ -321,6 +321,10 @@ void ScriptInfo::DoInitPostScript() { const auto& log_serializer_id = zeek::detail::global_scope()->Find("Cluster::LogSerializerTag"); types.push_back(new IdentifierInfo(log_serializer_id, this)); } + else if ( name == "base/frameworks/storage/main.zeek" ) { + const auto& backend_id = zeek::detail::global_scope()->Find("Storage::Backend"); + types.push_back(new IdentifierInfo(backend_id, this)); + } } vector ScriptInfo::GetComments() const { return comments; } diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index d99e27e5ec..dedda53b20 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -160,6 +160,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek + build/scripts/base/bif/storage.bif.zeek build/scripts/base/bif/spicy.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 4e61a3a752..ebf298afa4 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -160,6 +160,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek + build/scripts/base/bif/storage.bif.zeek build/scripts/base/bif/spicy.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek @@ -367,6 +368,8 @@ scripts/base/init-default.zeek scripts/base/frameworks/telemetry/__load__.zeek scripts/base/frameworks/telemetry/main.zeek scripts/base/misc/version.zeek + scripts/base/frameworks/storage/__load__.zeek + scripts/base/frameworks/storage/main.zeek scripts/base/frameworks/spicy/__load__.zeek scripts/base/frameworks/spicy/main.zeek scripts/base/protocols/conn/__load__.zeek diff --git a/testing/btest/Baseline/opt.ZAM-bif-tracking/output b/testing/btest/Baseline/opt.ZAM-bif-tracking/output index 0731586167..4fab29b90a 100644 --- a/testing/btest/Baseline/opt.ZAM-bif-tracking/output +++ b/testing/btest/Baseline/opt.ZAM-bif-tracking/output @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -546 seen BiFs, 0 unseen BiFs (), 0 new BiFs () +551 seen BiFs, 0 unseen BiFs (), 0 new BiFs () diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 9174cce539..b66bf362d9 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -505,6 +505,7 @@ 0.000000 MetaHookPost LoadFile(0, ./sftp, <...>/sftp.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./storage.bif.zeek, <...>/storage.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./store, <...>/store.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./store.bif.zeek, <...>/store.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> -1 @@ -815,6 +816,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./sftp, <...>/sftp.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./storage.bif.zeek, <...>/storage.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./store, <...>/store.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> (-1, ) @@ -1458,6 +1460,7 @@ 0.000000 MetaHookPre LoadFile(0, ./sftp, <...>/sftp.zeek) 0.000000 MetaHookPre LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./storage.bif.zeek, <...>/storage.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./store, <...>/store.zeek) 0.000000 MetaHookPre LoadFile(0, ./store.bif.zeek, <...>/store.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek) @@ -1768,6 +1771,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./sftp, <...>/sftp.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./storage.bif.zeek, <...>/storage.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./store, <...>/store.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek) @@ -2422,6 +2426,7 @@ 0.000000 | HookLoadFile ./sftp <...>/sftp.zeek 0.000000 | HookLoadFile ./spicy.bif.zeek <...>/spicy.bif.zeek 0.000000 | HookLoadFile ./stats.bif.zeek <...>/stats.bif.zeek +0.000000 | HookLoadFile ./storage.bif.zeek <...>/storage.bif.zeek 0.000000 | HookLoadFile ./store <...>/store.zeek 0.000000 | HookLoadFile ./store.bif.zeek <...>/store.bif.zeek 0.000000 | HookLoadFile ./strings.bif.zeek <...>/strings.bif.zeek @@ -2732,6 +2737,7 @@ 0.000000 | HookLoadFileExtended ./sftp <...>/sftp.zeek 0.000000 | HookLoadFileExtended ./spicy.bif.zeek <...>/spicy.bif.zeek 0.000000 | HookLoadFileExtended ./stats.bif.zeek <...>/stats.bif.zeek +0.000000 | HookLoadFileExtended ./storage.bif.zeek <...>/storage.bif.zeek 0.000000 | HookLoadFileExtended ./store <...>/store.zeek 0.000000 | HookLoadFileExtended ./store.bif.zeek <...>/store.bif.zeek 0.000000 | HookLoadFileExtended ./strings.bif.zeek <...>/strings.bif.zeek diff --git a/testing/btest/Baseline/plugins.storage/output b/testing/btest/Baseline/plugins.storage/output new file mode 100644 index 0000000000..0812d557db --- /dev/null +++ b/testing/btest/Baseline/plugins.storage/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +results of trying to use closed handle: get: 0, put: 0, erase: 0 diff --git a/testing/btest/Baseline/plugins.storage/zeek-stderr b/testing/btest/Baseline/plugins.storage/zeek-stderr new file mode 100644 index 0000000000..4963c5ee83 --- /dev/null +++ b/testing/btest/Baseline/plugins.storage/zeek-stderr @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/storage.zeek, line 37: Failed to retrieve data: Failed to find key (Storage::get(b, to_any_coerce key, to_any_coerce str)) +error in <...>/storage.zeek, line 50: Failed to open backend STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts)) +error in <...>/storage.zeek, line 51: Invalid storage handle (Storage::close_backend(b2) and F) diff --git a/testing/btest/opt/ZAM-bif-tracking.zeek b/testing/btest/opt/ZAM-bif-tracking.zeek index 1668c4af35..2a906c365f 100644 --- a/testing/btest/opt/ZAM-bif-tracking.zeek +++ b/testing/btest/opt/ZAM-bif-tracking.zeek @@ -177,6 +177,11 @@ global known_BiFs = set( "Reporter::warning", "Spicy::__resource_usage", "Spicy::__toggle_analyzer", + "Storage::__close_backend", + "Storage::__erase", + "Storage::__get", + "Storage::__open_backend", + "Storage::__put", "Supervisor::__create", "Supervisor::__destroy", "Supervisor::__is_supervised", diff --git a/testing/btest/plugins/storage-plugin/.btest-ignore b/testing/btest/plugins/storage-plugin/.btest-ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/plugins/storage-plugin/CMakeLists.txt b/testing/btest/plugins/storage-plugin/CMakeLists.txt new file mode 100644 index 0000000000..da332c5eb9 --- /dev/null +++ b/testing/btest/plugins/storage-plugin/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.15) + +project(Zeek-Plugin-Storage-Demo) + +if (NOT ZEEK_DIST) + message(FATAL_ERROR "ZEEK_DIST not set") +endif () + +set(CMAKE_MODULE_PATH ${ZEEK_DIST}/cmake) + +include(ZeekPlugin) + +zeek_plugin_begin(Testing StorageDummy) +zeek_plugin_cc(src/Plugin.cc) +zeek_plugin_cc(src/StorageDummy.cc) +zeek_plugin_end() diff --git a/testing/btest/plugins/storage-plugin/src/Plugin.cc b/testing/btest/plugins/storage-plugin/src/Plugin.cc new file mode 100644 index 0000000000..ca7e42f4e6 --- /dev/null +++ b/testing/btest/plugins/storage-plugin/src/Plugin.cc @@ -0,0 +1,23 @@ +#include "Plugin.h" + +#include "zeek/storage/Component.h" + +#include "StorageDummy.h" + +namespace btest::plugin::Testing_StorageDummy { +Plugin plugin; +} + +using namespace btest::plugin::Testing_StorageDummy; + +zeek::plugin::Configuration Plugin::Configure() { + AddComponent(new zeek::storage::Component("StorageDummy", btest::storage::backend::StorageDummy::Instantiate)); + + zeek::plugin::Configuration config; + config.name = "Testing::StorageDummy"; + config.description = "A dummy storage plugin"; + config.version.major = 1; + config.version.minor = 0; + config.version.patch = 0; + return config; +} diff --git a/testing/btest/plugins/storage-plugin/src/Plugin.h b/testing/btest/plugins/storage-plugin/src/Plugin.h new file mode 100644 index 0000000000..1a386be0a6 --- /dev/null +++ b/testing/btest/plugins/storage-plugin/src/Plugin.h @@ -0,0 +1,16 @@ + +#pragma once + +#include + +namespace btest::plugin::Testing_StorageDummy { + +class Plugin : public zeek::plugin::Plugin { +protected: + // Overridden from plugin::Plugin. + virtual zeek::plugin::Configuration Configure(); +}; + +extern Plugin plugin; + +} // namespace btest::plugin::Testing_StorageDummy diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc new file mode 100644 index 0000000000..0f1c319fa4 --- /dev/null +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -0,0 +1,77 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "StorageDummy.h" + +#include "zeek/Func.h" +#include "zeek/Val.h" + +namespace btest::storage::backend { + +zeek::storage::BackendPtr StorageDummy::Instantiate(std::string_view tag) { + return zeek::make_intrusive(tag); +} + +/** + * Called by the manager system to open the backend. + * + * Derived classes must implement this method. If successful, the + * implementation must call \a Opened(); if not, it must call Error() + * with a corresponding message. + */ +zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options) { + bool open_fail = options->GetField("open_fail")->Get(); + if ( open_fail ) + return "open_fail was set to true, returning error"; + + open = true; + + return std::nullopt; +} + +/** + * Finalizes the backend when it's being closed. + */ +void StorageDummy::Close() { open = false; } + +/** + * The workhorse method for Put(). This must be implemented by plugins. + */ +zeek::storage::ErrorResult StorageDummy::DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite) { + auto json_key = key->ToJSON()->ToStdString(); + auto json_value = value->ToJSON()->ToStdString(); + data[json_key] = json_value; + return std::nullopt; +} + +/** + * The workhorse method for Get(). This must be implemented for plugins. + */ +zeek::storage::ValResult StorageDummy::DoGet(zeek::ValPtr key, zeek::TypePtr vt) { + auto json_key = key->ToJSON(); + auto it = data.find(json_key->ToStdString()); + if ( it == data.end() ) + return zeek::unexpected("Failed to find key"); + + auto val = zeek::detail::ValFromJSON(it->second.c_str(), vt, zeek::Func::nil); + if ( std::holds_alternative(val) ) { + zeek::ValPtr val_v = std::get(val); + return val_v; + } + + return zeek::unexpected(std::get(val)); +} + +/** + * The workhorse method for Erase(). This must be implemented for plugins. + */ +zeek::storage::ErrorResult StorageDummy::DoErase(zeek::ValPtr key) { + auto json_key = key->ToJSON(); + auto it = data.find(json_key->ToStdString()); + if ( it == data.end() ) + return "Failed to find key"; + + data.erase(it); + return std::nullopt; +} + +} // namespace btest::storage::backend diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h new file mode 100644 index 0000000000..cce7fb0293 --- /dev/null +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -0,0 +1,56 @@ + +#pragma once + +#include +#include + +#include "zeek/storage/Backend.h" + +namespace btest::storage::backend { + +/** + * A Foo reader to measure performance of the input framework. + */ +class StorageDummy : public zeek::storage::Backend { +public: + StorageDummy(std::string_view tag) : Backend(tag) {} + ~StorageDummy() override = default; + + static zeek::storage::BackendPtr Instantiate(std::string_view tag); + + /** + * Called by the manager system to open the backend. + */ + zeek::storage::ErrorResult DoOpen(zeek::RecordValPtr options) override; + + /** + * Finalizes the backend when it's being closed. + */ + void Close() override; + + /** + * Returns whether the backend is opened. + */ + bool IsOpen() override { return open; } + + /** + * The workhorse method for Put(). + */ + zeek::storage::ErrorResult DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite = true) override; + + /** + * The workhorse method for Get(). + */ + zeek::storage::ValResult DoGet(zeek::ValPtr key, zeek::TypePtr vt) override; + + /** + * The workhorse method for Erase(). + */ + zeek::storage::ErrorResult DoErase(zeek::ValPtr key) override; + +private: + std::map data; + bool open = false; +}; + +} // namespace btest::storage::backend diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek new file mode 100644 index 0000000000..8034d27cf4 --- /dev/null +++ b/testing/btest/plugins/storage.zeek @@ -0,0 +1,52 @@ +# @TEST-DOC: Basic test of a plugin implmenting a backend for the storage framework +# @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1" + +# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Testing StorageDummy +# @TEST-EXEC: cp -r %DIR/storage-plugin/* . +# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make +# @TEST-EXEC: ZEEK_PLUGIN_PATH=$(pwd) zeek -b Testing::StorageDummy %INPUT >> output 2>zeek-stderr +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff zeek-stderr + +@load base/frameworks/storage + +# Create a typename here that can be passed down into get(). +type str: string; + +type StorageDummyOpts : record { + open_fail: bool; +}; + +event zeek_init() { + local opts : StorageDummyOpts; + opts$open_fail = F; + + local key = "key1234"; + local value = "value5678"; + + # Test basic operation. The second get() should return an error + # as the key should have been erased. + local b = Storage::open_backend(Storage::STORAGEDUMMY, opts); + local put_res = Storage::put(b, key, value, F); + local get_res = Storage::get(b, key, str); + if ( get_res is bool ) { + print("Got an invalid value in response!"); + } + + local erase_res = Storage::erase(b, key); + get_res = Storage::get(b, key, str); + Storage::close_backend(b); + + # Test attempting to use the closed handle. + put_res = Storage::put(b, "a", "b", F); + get_res = Storage::get(b, "a", str); + erase_res = Storage::erase(b, "a"); + + print(fmt("results of trying to use closed handle: get: %d, put: %d, erase: %d", + get_res, put_res, erase_res)); + + # Test failing to open the handle and test closing an invalid handle. + opts$open_fail = T; + local b2 = Storage::open_backend(Storage::STORAGEDUMMY, opts); + Storage::close_backend(b2); +}