Lay out initial parts for the Storage framework

This includes a manager, component manager, BIF and script code, and
parts to support new storage backend plugins.

CMakeLists.txt

@@ -1143,6 +1143,7 @@ include(FindKqueue)
 
 include(FindPrometheusCpp)
 include_directories(BEFORE "auxil/out_ptr/include")
+include_directories(BEFORE "auxil/expected-lite/include")
 
 if ((OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0"))
     set(ZEEK_HAVE_OPENSSL_1_1 true CACHE INTERNAL "" FORCE)

scripts/base/frameworks/storage/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/storage/main.zeek

@@ -0,0 +1,85 @@
+##! The storage framework provides a way to store long-term data to disk.
+
+@load base/bif/storage.bif
+
+module Storage;
+
+export {
+	## Opens a new backend connection based on a configuration object.
+	##
+	## btype: A tag indicating what type of backend should be opened.
+	##
+	## options: A record containing the configuration for the connection.
+	##
+	## Returns: A handle to the new backend connection, or null if the
+	##          connection failed.
+	global open_backend: function(btype: Storage::Backend, options: any): opaque of Storage::BackendHandle;
+
+	## Closes an existing backend connection.
+	##
+	## backend: A handle to a backend connection.
+	##
+	## Returns: A boolean indicating success or failure of the operation.
+	global close_backend: function(backend: opaque of Storage::BackendHandle): bool;
+
+	## Inserts a new entry into a backend.
+	##
+	## backend: A handle to a backend connection.
+	##
+	## key: A key value.
+	##
+	## value: A corresponding value.
+	##
+	## overwrite: A flag indicating whether this value should overwrite an
+	##            existing entry for the key.
+	##
+	## Returns: A boolean indicating success or failure of the operation.
+	global put: function(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool): bool;
+
+	## Gets an entry from the backend.
+	##
+	## backend: A handle to a backend connection.
+	##
+	## key: The key to look up.
+	##
+	## val_type: The type of the value to return.
+	##
+	## Returns: A boolean indicating success or failure of the
+	##          operation. Type conversion failures for the value will
+	##          return false.
+	global get: function(backend: opaque of Storage::BackendHandle, key: any, val_type: any): any;
+
+	## Erases an entry from the backend.
+	##
+	## backend: A handle to a backend connection.
+	##
+	## key: The key to erase.
+	##
+	## Returns: A boolean indicating success or failure of the operation.
+	global erase: function(backend: opaque of Storage::BackendHandle, key: any): bool;
+}
+
+function open_backend(btype: Storage::Backend, options: any): opaque of Storage::BackendHandle
+{
+	return Storage::__open_backend(btype, options);
+}
+
+function close_backend(backend: opaque of Storage::BackendHandle): bool
+{
+	return Storage::__close_backend(backend);
+}
+
+function put(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool): bool
+{
+	return Storage::__put(backend, key, value, overwrite);
+}
+
+function get(backend: opaque of Storage::BackendHandle, key: any, val_type: any): any
+{
+	return Storage::__get(backend, key, val_type);
+}
+
+function erase(backend: opaque of Storage::BackendHandle, key: any): bool
+{
+	return Storage::__erase(backend, key);
+}

scripts/base/init-default.zeek

@@ -44,6 +44,7 @@
 @load base/frameworks/openflow
 @load base/frameworks/netcontrol
 @load base/frameworks/telemetry
+@load base/frameworks/storage
 
 @if ( have_spicy() )
 @load base/frameworks/spicy

src/CMakeLists.txt

@@ -205,6 +205,7 @@ add_subdirectory(iosource)
 add_subdirectory(logging)
 add_subdirectory(probabilistic)
 add_subdirectory(session)
+add_subdirectory(storage)
 
 if (HAVE_SPICY)
     add_subdirectory(spicy)

src/script_opt/FuncInfo.cc

@@ -148,6 +148,11 @@ static std::unordered_map<std::string, unsigned int> func_attrs = {
     {"Reporter::warning", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"Spicy::__resource_usage", ATTR_NO_ZEEK_SIDE_EFFECTS},
     {"Spicy::__toggle_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS},
+    {"Storage::__close_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS},
+    {"Storage::__erase", ATTR_NO_SCRIPT_SIDE_EFFECTS},
+    {"Storage::__get", ATTR_NO_SCRIPT_SIDE_EFFECTS},
+    {"Storage::__open_backend", ATTR_NO_SCRIPT_SIDE_EFFECTS},
+    {"Storage::__put", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"Supervisor::__create", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"Supervisor::__destroy", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"Supervisor::__is_supervised", ATTR_IDEMPOTENT},

src/storage/Backend.cc

@@ -0,0 +1,42 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/storage/Backend.h"
+
+#include "zeek/broker/Data.h"
+
+namespace zeek::storage {
+
+ErrorResult Backend::Open(RecordValPtr options) { return DoOpen(std::move(options)); }
+
+ErrorResult Backend::Put(ValPtr key, ValPtr value, bool overwrite) {
+    // The intention for this method is to do some other heavy lifting in regard
+    // to backends that need to pass data through the manager instead of directly
+    // through the workers. For the first versions of the storage framework it
+    // just calls the backend itself directly.
+    return DoPut(std::move(key), std::move(value), overwrite);
+}
+
+ValResult Backend::Get(ValPtr key, TypePtr value_type) {
+    // See the note in Put().
+    return DoGet(std::move(key), std::move(value_type));
+}
+
+ErrorResult Backend::Erase(ValPtr key) {
+    // See the note in Put().
+    return DoErase(std::move(key));
+}
+
+zeek::OpaqueTypePtr detail::backend_opaque;
+IMPLEMENT_OPAQUE_VALUE(detail::BackendHandleVal)
+
+std::optional<BrokerData> detail::BackendHandleVal::DoSerializeData() const {
+    // Cannot serialize.
+    return std::nullopt;
+}
+
+bool detail::BackendHandleVal::DoUnserializeData(BrokerDataView) {
+    // Cannot unserialize.
+    return false;
+}
+
+} // namespace zeek::storage

src/storage/Backend.h

@@ -0,0 +1,137 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include "zeek/OpaqueVal.h"
+#include "zeek/Val.h"
+#include "zeek/util.h"
+
+namespace zeek::storage {
+
+class Manager;
+
+// Result from storage operations that may return an error message. If the
+// optional value is unset, the operation succeeded.
+using ErrorResult = std::optional<std::string>;
+
+// Result from storage operations that return Vals. The ValPtr is an
+// IntrusivePtr to some result, and can be null if the operation failed. The
+// string value will store an error message if the result is null.
+using ValResult = zeek::expected<ValPtr, std::string>;
+
+class Backend : public zeek::Obj {
+public:
+    /**
+     * Returns a descriptive tag representing the source for debugging.
+     */
+    const char* Tag() { return tag.c_str(); }
+
+    /**
+     * Store a new key/value pair in the backend.
+     *
+     * @param key the key for the pair
+     * @param value the value for the pair
+     * @param overwrite whether an existing value for a key should be overwritten.
+     * @return A result pair containing a bool with the success state, and a
+     * possible error string if the operation failed.
+     */
+    ErrorResult Put(ValPtr key, ValPtr value, bool overwrite = true);
+
+    /**
+     * Retrieve a value from the backend for a provided key.
+     *
+     * @param key the key to lookup in the backend.
+     * @param value_type The script-land type to be used when retrieving values
+     * from the backend.
+     * @return A result pair containing a ValPtr with the resulting value or
+     * nullptr retrieval failed, and a string with the error message if the
+     * operation failed.
+     */
+    ValResult Get(ValPtr key, TypePtr value_type);
+
+    /**
+     * Erases the value for a key from the backend.
+     *
+     * @return An optional value potentially containing an error string if
+     * needed. Will be unset if the operation succeeded.
+     * possible error string if the operation failed.
+     */
+    ErrorResult Erase(ValPtr key);
+
+    /**
+     * Returns whether the backend is opened.
+     */
+    virtual bool IsOpen() = 0;
+
+protected:
+    // Allow the manager to call Open/Close.
+    friend class storage::Manager;
+
+    /**
+     * Constructor
+     *
+     * @param tag A string representation of the tag for this backend. This
+     * is passed from the Manager through the component factory.
+     */
+    Backend(std::string_view tag) : tag(tag) {}
+
+    /**
+     * Called by the manager system to open the backend.
+     *
+     * @param options A record storing configuration options for the backend.
+     * @return A result pair containing a bool with the success state, and a
+     * possible error string if the operation failed.
+     */
+    ErrorResult Open(RecordValPtr options);
+
+    /**
+     * Finalizes the backend when it's being closed. Can be overridden by
+     * derived classes.
+     */
+    virtual void Close() {}
+
+    /**
+     * The workhorse method for Open().
+     */
+    virtual ErrorResult DoOpen(RecordValPtr options) = 0;
+
+    /**
+     * The workhorse method for Put().
+     */
+    virtual ErrorResult DoPut(ValPtr key, ValPtr value, bool overwrite = true) = 0;
+
+    /**
+     * The workhorse method for Get().
+     */
+    virtual ValResult DoGet(ValPtr key, TypePtr vt) = 0;
+
+    /**
+     * The workhorse method for Erase().
+     */
+    virtual ErrorResult DoErase(ValPtr key) = 0;
+
+    std::string tag;
+};
+
+using BackendPtr = zeek::IntrusivePtr<Backend>;
+
+namespace detail {
+
+extern OpaqueTypePtr backend_opaque;
+
+class BackendHandleVal : public OpaqueVal {
+public:
+    BackendHandleVal() : OpaqueVal(detail::backend_opaque) {}
+    BackendHandleVal(BackendPtr backend) : OpaqueVal(detail::backend_opaque), backend(std::move(backend)) {}
+    ~BackendHandleVal() override = default;
+
+    BackendPtr backend;
+
+protected:
+    IntrusivePtr<Val> DoClone(CloneState* state) override { return {NewRef{}, this}; }
+
+    DECLARE_OPAQUE_VALUE_DATA(BackendHandleVal)
+};
+
+} // namespace detail
+} // namespace zeek::storage

src/storage/CMakeLists.txt

@@ -0,0 +1,10 @@
+zeek_add_subdir_library(
+    storage
+    SOURCES
+    Manager.cc
+    Backend.cc
+    Component.cc
+    BIFS
+    storage.bif)
+
+add_subdirectory(backend)

src/storage/Component.cc

@@ -0,0 +1,25 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/storage/Component.h"
+
+#include "zeek/Desc.h"
+#include "zeek/storage/Manager.h"
+
+namespace zeek::storage {
+
+Component::Component(const std::string& name, factory_callback arg_factory)
+    : plugin::Component(plugin::component::STORAGE_BACKEND, name, 0, storage_mgr->GetTagType()) {
+    factory = arg_factory;
+}
+
+void Component::Initialize() {
+    InitializeTag();
+    storage_mgr->RegisterComponent(this);
+}
+
+void Component::DoDescribe(ODesc* d) const {
+    d->Add("Storage::STORAGE_BACKEND_");
+    d->Add(CanonicalName());
+}
+
+} // namespace zeek::storage

src/storage/Component.h

@@ -0,0 +1,59 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include "zeek/plugin/Component.h"
+
+namespace zeek::storage {
+
+class Backend;
+
+/**
+ * Component description for plugins providing storage backends.
+ */
+class Component : public plugin::Component {
+public:
+    using factory_callback = IntrusivePtr<Backend> (*)(std::string_view);
+
+    /**
+     * Constructor.
+     *
+     * @param name The name of the provided backend. This name is used
+     * across the system to identify the backend.
+     *
+     * @param factory A factory function to instantiate instances of the
+     * backend's class, which must be derived directly or indirectly from
+     * storage::Backend. This is typically a static \c Instantiate()
+     * method inside the class that just allocates and returns a new
+     * instance.
+     */
+    Component(const std::string& name, factory_callback factory);
+
+    /**
+     * Destructor.
+     */
+    ~Component() override = default;
+
+    /**
+     * Initialization function. This function has to be called before any
+     * plugin component functionality is used; it is used to add the
+     * plugin component to the list of components and to initialize tags
+     */
+    void Initialize() override;
+
+    /**
+     * Returns the backend's factory function.
+     */
+    factory_callback Factory() const { return factory; }
+
+protected:
+    /**
+     * Overridden from plugin::Component.
+     */
+    void DoDescribe(ODesc* d) const override;
+
+private:
+    factory_callback factory;
+};
+
+} // namespace zeek::storage

src/storage/Manager.cc

@@ -0,0 +1,58 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/storage/Manager.h"
+
+#include "zeek/Desc.h"
+
+namespace zeek::storage {
+
+Manager::Manager() : plugin::ComponentManager<storage::Component>("Storage", "Backend") {}
+
+void Manager::InitPostScript() { detail::backend_opaque = make_intrusive<OpaqueType>("Storage::Backend"); }
+
+zeek::expected<BackendPtr, std::string> Manager::OpenBackend(const Tag& type, RecordValPtr options) {
+    Component* c = Lookup(type);
+    if ( ! c ) {
+        return zeek::unexpected<std::string>(
+            util::fmt("Request to open unknown backend (%d:%d)", type.Type(), type.Subtype()));
+    }
+
+    if ( ! c->Factory() ) {
+        return zeek::unexpected<std::string>(
+            util::fmt("Factory invalid for backend %s", GetComponentName(type).c_str()));
+    }
+
+    ODesc d;
+    type.AsVal()->Describe(&d);
+
+    BackendPtr bp = c->Factory()(d.Description());
+
+    if ( ! bp ) {
+        return zeek::unexpected<std::string>(
+            util::fmt("Failed to instantiate backend %s", GetComponentName(type).c_str()));
+    }
+
+    if ( auto res = bp->Open(std::move(options)); res.has_value() ) {
+        return zeek::unexpected<std::string>(
+            util::fmt("Failed to open backend %s: %s", GetComponentName(type).c_str(), res.value().c_str()));
+    }
+
+    // TODO: post Storage::backend_opened event
+
+    backends.push_back(bp);
+
+    return bp;
+}
+
+void Manager::CloseBackend(BackendPtr backend) {
+    auto it = std::find(backends.begin(), backends.end(), backend);
+    if ( it == backends.end() )
+        return;
+
+    backends.erase(it);
+    backend->Close();
+
+    // TODO: post Storage::backend_lost event
+}
+
+} // namespace zeek::storage

src/storage/Manager.h

@@ -0,0 +1,52 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include "zeek/plugin/ComponentManager.h"
+#include "zeek/storage/Backend.h"
+#include "zeek/storage/Component.h"
+
+namespace zeek::storage {
+
+class Manager final : public plugin::ComponentManager<Component> {
+public:
+    Manager();
+    ~Manager() = default;
+
+    /**
+     * Initialization of the manager. This is called late during Zeek's
+     * initialization after any scripts are processed.
+     */
+    void InitPostScript();
+
+    /**
+     * Opens a new storage backend.
+     *
+     * @param type The tag for the type of backend being opened.
+     * @param options A record val representing the configuration for this
+     * type of backend.
+     * @return A pair containing a pointer to a backend and a string for
+     * returning error messages if needed.
+     */
+    zeek::expected<BackendPtr, std::string> OpenBackend(const Tag& type, RecordValPtr options);
+
+    /**
+     * Closes a storage backend.
+     */
+    void CloseBackend(BackendPtr backend);
+
+    // TODO:
+    // - Hooks for storage-backed tables?
+    // - Handling aggregation from workers on a single manager?
+
+private:
+    std::vector<BackendPtr> backends;
+};
+
+} // namespace zeek::storage
+
+namespace zeek {
+
+extern storage::Manager* storage_mgr;
+
+} // namespace zeek

src/storage/backend/CMakeLists.txt

@@ -0,0 +1 @@
+

src/storage/storage.bif

@@ -0,0 +1,116 @@
+%%{
+#include "zeek/storage/Backend.h"
+#include "zeek/storage/Manager.h"
+
+using namespace zeek;
+using namespace zeek::storage;
+%%}
+
+module Storage;
+
+# Generated when a new backend connection is opened
+event Storage::backend_opened%(%);
+
+# Generated when a backend connection is lost
+event Storage::backend_lost%(%);
+
+function Storage::__open_backend%(btype: Storage::Backend, options: any%): opaque of Storage::BackendHandle
+	%{
+	auto btype_val = IntrusivePtr<EnumVal>{NewRef{}, btype->AsEnumVal()};
+	Tag tag{btype_val};
+
+	auto options_val = IntrusivePtr<RecordVal>{NewRef{}, options->AsRecordVal()};
+
+	auto b = storage_mgr->OpenBackend(tag, options_val);
+
+	if ( ! b.has_value() ) {
+		emit_builtin_error(b.error().c_str());
+		return val_mgr->Bool(false);
+	}
+
+	return make_intrusive<storage::detail::BackendHandleVal>(b.value());
+	%}
+
+function Storage::__close_backend%(backend: opaque of Storage::BackendHandle%) : bool
+	%{
+	auto b = dynamic_cast<storage::detail::BackendHandleVal*>(backend);
+	if ( ! b ) {
+		emit_builtin_error("Invalid storage handle", backend);
+		return val_mgr->Bool(false);
+	}
+	else if ( ! b->backend->IsOpen() )
+		// Return true here since the backend is already closed
+		return val_mgr->Bool(true);
+
+	storage_mgr->CloseBackend(b->backend);
+
+	return val_mgr->Bool(true);
+	%}
+
+function Storage::__put%(backend: opaque of Storage::BackendHandle, key: any, value: any, overwrite: bool%): bool
+	%{
+	auto b = dynamic_cast<storage::detail::BackendHandleVal*>(backend);
+	if ( ! b ) {
+		emit_builtin_error("Invalid storage handle", backend);
+		return val_mgr->Bool(false);
+	}
+	else if ( ! b->backend->IsOpen() )
+		return val_mgr->Bool(false);
+
+	// TODO: add support for when statements (see broker/store.bif)
+
+	auto key_v = IntrusivePtr<Val>{NewRef{}, key};
+	auto val_v = IntrusivePtr<Val>{NewRef{}, value};
+	auto result = b->backend->Put(key_v, val_v, overwrite);
+	if ( result.has_value() ) {
+		emit_builtin_error(util::fmt("Failed to store data: %s", result.value().c_str()));
+		return val_mgr->Bool(false);
+	}
+
+	return val_mgr->Bool(true);
+	%}
+
+function Storage::__get%(backend: opaque of Storage::BackendHandle, key: any, val_type: any%): any
+	%{
+	auto b = dynamic_cast<storage::detail::BackendHandleVal*>(backend);
+	if ( ! b ) {
+		emit_builtin_error("Invalid storage handle", backend);
+		return val_mgr->Bool(false);
+	}
+	else if ( ! b->backend->IsOpen() )
+		return val_mgr->Bool(false);
+
+	// TODO: add support for when statements (see broker/store.bif)
+
+	auto key_v = IntrusivePtr<Val>{NewRef{}, key};
+	auto vt = val_type->AsTypeVal()->GetType()->AsTypeType()->GetType();
+	auto result = b->backend->Get(key_v, vt);
+	if ( ! result.has_value() ) {
+		emit_builtin_error(util::fmt("Failed to retrieve data: %s", result.error().c_str()));
+		return val_mgr->Bool(false);
+	}
+
+	return result.value();
+	%}
+
+function Storage::__erase%(backend: opaque of Storage::BackendHandle, key: any%): bool
+	%{
+	auto b = dynamic_cast<storage::detail::BackendHandleVal*>(backend);
+	if ( ! b ) {
+		emit_builtin_error("Invalid storage handle", backend);
+		return val_mgr->Bool(false);
+	}
+	else if ( ! b->backend->IsOpen() )
+		return val_mgr->Bool(false);
+
+	// TODO: add support for when statements (see broker/store.bif)
+
+	auto key_v = IntrusivePtr<Val>{NewRef{}, key};
+	auto result = b->backend->Erase(key_v);
+	if ( result.has_value() ) {
+		emit_builtin_error(util::fmt("Failed to erase data for key: %s", result.value().c_str()));
+		return val_mgr->Bool(false);
+	}
+
+	return val_mgr->Bool(true);
+	%}

src/util.h

@@ -97,6 +97,15 @@ namespace filesystem = ghc::filesystem;
 inline constexpr std::string_view path_list_separator = ":";
 #endif
 
+#include "zeek/3rdparty/nonstd/expected.hpp"
+namespace zeek {
+template<typename T, typename E>
+using expected = nonstd::expected<T, E>;
+
+template<typename E>
+using unexpected = nonstd::unexpected<E>;
+} // namespace zeek
+
 using zeek_int_t = int64_t;
 using zeek_uint_t = uint64_t;
 

src/zeek-setup.cc

@@ -65,6 +65,7 @@
 #ifdef HAVE_SPICY
 #include "zeek/spicy/manager.h"
 #endif
+#include "zeek/storage/Manager.h"
 #include "zeek/supervisor/Supervisor.h"
 #include "zeek/telemetry/Manager.h"
 #include "zeek/threading/Manager.h"
@@ -178,6 +179,7 @@ zeek::detail::trigger::Manager* zeek::detail::trigger_mgr = nullptr;
 #ifdef HAVE_SPICY
 zeek::spicy::Manager* zeek::spicy_mgr = nullptr;
 #endif
+zeek::storage::Manager* zeek::storage_mgr = nullptr;
 
 zeek::cluster::Manager* zeek::cluster::manager = nullptr;
 zeek::cluster::Backend* zeek::cluster::backend = nullptr;
@@ -414,6 +416,7 @@ static void terminate_zeek() {
 #ifdef HAVE_SPICY
     delete spicy_mgr;
 #endif
+    delete storage_mgr;
 
     // free the global scope
     pop_scope();
@@ -686,6 +689,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) {
 #ifdef HAVE_SPICY
     spicy_mgr = new spicy::Manager(); // registers as plugin with the plugin manager
 #endif
+    storage_mgr = new storage::Manager();
 
     plugin_mgr->InitPreScript();
     file_mgr->InitPreScript();
@@ -873,6 +877,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) {
 
         timer_mgr->InitPostScript();
         event_mgr.InitPostScript();
+        storage_mgr->InitPostScript();
 
         if ( supervisor_mgr )
             supervisor_mgr->InitPostScript();

src/zeekygen/ScriptInfo.cc

@@ -321,6 +321,10 @@ void ScriptInfo::DoInitPostScript() {
         const auto& log_serializer_id = zeek::detail::global_scope()->Find("Cluster::LogSerializerTag");
         types.push_back(new IdentifierInfo(log_serializer_id, this));
     }
+    else if ( name == "base/frameworks/storage/main.zeek" ) {
+        const auto& backend_id = zeek::detail::global_scope()->Find("Storage::Backend");
+        types.push_back(new IdentifierInfo(backend_id, this));
+    }
 }
 
 vector<string> ScriptInfo::GetComments() const { return comments; }

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -160,6 +160,7 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/bloom-filter.bif.zeek
     build/scripts/base/bif/cardinality-counter.bif.zeek
     build/scripts/base/bif/top-k.bif.zeek
+    build/scripts/base/bif/storage.bif.zeek
     build/scripts/base/bif/spicy.bif.zeek
   build/scripts/base/bif/plugins/__load__.zeek
     build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -160,6 +160,7 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/bloom-filter.bif.zeek
     build/scripts/base/bif/cardinality-counter.bif.zeek
     build/scripts/base/bif/top-k.bif.zeek
+    build/scripts/base/bif/storage.bif.zeek
     build/scripts/base/bif/spicy.bif.zeek
   build/scripts/base/bif/plugins/__load__.zeek
     build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek
@@ -367,6 +368,8 @@ scripts/base/init-default.zeek
   scripts/base/frameworks/telemetry/__load__.zeek
     scripts/base/frameworks/telemetry/main.zeek
       scripts/base/misc/version.zeek
+  scripts/base/frameworks/storage/__load__.zeek
+    scripts/base/frameworks/storage/main.zeek
   scripts/base/frameworks/spicy/__load__.zeek
     scripts/base/frameworks/spicy/main.zeek
   scripts/base/protocols/conn/__load__.zeek

testing/btest/Baseline/opt.ZAM-bif-tracking/output

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-546 seen BiFs, 0 unseen BiFs (), 0 new BiFs ()
+551 seen BiFs, 0 unseen BiFs (), 0 new BiFs ()

testing/btest/Baseline/plugins.hooks/output

@@ -505,6 +505,7 @@
 0.000000   MetaHookPost  LoadFile(0, ./sftp, <...>/sftp.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./storage.bif.zeek, <...>/storage.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./store, <...>/store.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./store.bif.zeek, <...>/store.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> -1
@@ -815,6 +816,7 @@
 0.000000   MetaHookPost  LoadFileExtended(0, ./sftp, <...>/sftp.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./storage.bif.zeek, <...>/storage.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./store, <...>/store.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> (-1, <no content>)
@@ -1458,6 +1460,7 @@
 0.000000   MetaHookPre   LoadFile(0, ./sftp, <...>/sftp.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./storage.bif.zeek, <...>/storage.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./store, <...>/store.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./store.bif.zeek, <...>/store.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek)
@@ -1768,6 +1771,7 @@
 0.000000   MetaHookPre   LoadFileExtended(0, ./sftp, <...>/sftp.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./storage.bif.zeek, <...>/storage.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./store, <...>/store.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek)
@@ -2422,6 +2426,7 @@
 0.000000 | HookLoadFile  ./sftp <...>/sftp.zeek
 0.000000 | HookLoadFile  ./spicy.bif.zeek <...>/spicy.bif.zeek
 0.000000 | HookLoadFile  ./stats.bif.zeek <...>/stats.bif.zeek
+0.000000 | HookLoadFile  ./storage.bif.zeek <...>/storage.bif.zeek
 0.000000 | HookLoadFile  ./store <...>/store.zeek
 0.000000 | HookLoadFile  ./store.bif.zeek <...>/store.bif.zeek
 0.000000 | HookLoadFile  ./strings.bif.zeek <...>/strings.bif.zeek
@@ -2732,6 +2737,7 @@
 0.000000 | HookLoadFileExtended ./sftp <...>/sftp.zeek
 0.000000 | HookLoadFileExtended ./spicy.bif.zeek <...>/spicy.bif.zeek
 0.000000 | HookLoadFileExtended ./stats.bif.zeek <...>/stats.bif.zeek
+0.000000 | HookLoadFileExtended ./storage.bif.zeek <...>/storage.bif.zeek
 0.000000 | HookLoadFileExtended ./store <...>/store.zeek
 0.000000 | HookLoadFileExtended ./store.bif.zeek <...>/store.bif.zeek
 0.000000 | HookLoadFileExtended ./strings.bif.zeek <...>/strings.bif.zeek

testing/btest/Baseline/plugins.storage/output

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+results of trying to use closed handle: get: 0, put: 0, erase: 0

testing/btest/Baseline/plugins.storage/zeek-stderr

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in <...>/storage.zeek, line 37: Failed to retrieve data: Failed to find key (Storage::get(b, to_any_coerce key, to_any_coerce str))
+error in <...>/storage.zeek, line 50: Failed to open backend STORAGEDUMMY: open_fail was set to true, returning error (Storage::open_backend(Storage::STORAGEDUMMY, to_any_coerce opts))
+error in <...>/storage.zeek, line 51: Invalid storage handle (Storage::close_backend(b2) and F)

testing/btest/opt/ZAM-bif-tracking.zeek

@@ -177,6 +177,11 @@ global known_BiFs = set(
 	"Reporter::warning",
 	"Spicy::__resource_usage",
 	"Spicy::__toggle_analyzer",
+	"Storage::__close_backend",
+	"Storage::__erase",
+	"Storage::__get",
+	"Storage::__open_backend",
+	"Storage::__put",
 	"Supervisor::__create",
 	"Supervisor::__destroy",
 	"Supervisor::__is_supervised",

testing/btest/plugins/storage-plugin/CMakeLists.txt

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.15)
+
+project(Zeek-Plugin-Storage-Demo)
+
+if (NOT ZEEK_DIST)
+    message(FATAL_ERROR "ZEEK_DIST not set")
+endif ()
+
+set(CMAKE_MODULE_PATH ${ZEEK_DIST}/cmake)
+
+include(ZeekPlugin)
+
+zeek_plugin_begin(Testing StorageDummy)
+zeek_plugin_cc(src/Plugin.cc)
+zeek_plugin_cc(src/StorageDummy.cc)
+zeek_plugin_end()

testing/btest/plugins/storage-plugin/src/Plugin.cc

@@ -0,0 +1,23 @@
+#include "Plugin.h"
+
+#include "zeek/storage/Component.h"
+
+#include "StorageDummy.h"
+
+namespace btest::plugin::Testing_StorageDummy {
+Plugin plugin;
+}
+
+using namespace btest::plugin::Testing_StorageDummy;
+
+zeek::plugin::Configuration Plugin::Configure() {
+    AddComponent(new zeek::storage::Component("StorageDummy", btest::storage::backend::StorageDummy::Instantiate));
+
+    zeek::plugin::Configuration config;
+    config.name = "Testing::StorageDummy";
+    config.description = "A dummy storage plugin";
+    config.version.major = 1;
+    config.version.minor = 0;
+    config.version.patch = 0;
+    return config;
+}

testing/btest/plugins/storage-plugin/src/Plugin.h

@@ -0,0 +1,16 @@
+
+#pragma once
+
+#include <plugin/Plugin.h>
+
+namespace btest::plugin::Testing_StorageDummy {
+
+class Plugin : public zeek::plugin::Plugin {
+protected:
+    // Overridden from plugin::Plugin.
+    virtual zeek::plugin::Configuration Configure();
+};
+
+extern Plugin plugin;
+
+} // namespace btest::plugin::Testing_StorageDummy

testing/btest/plugins/storage-plugin/src/StorageDummy.cc

@@ -0,0 +1,77 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "StorageDummy.h"
+
+#include "zeek/Func.h"
+#include "zeek/Val.h"
+
+namespace btest::storage::backend {
+
+zeek::storage::BackendPtr StorageDummy::Instantiate(std::string_view tag) {
+    return zeek::make_intrusive<StorageDummy>(tag);
+}
+
+/**
+ * Called by the manager system to open the backend.
+ *
+ * Derived classes must implement this method. If successful, the
+ * implementation must call \a Opened(); if not, it must call Error()
+ * with a corresponding message.
+ */
+zeek::storage::ErrorResult StorageDummy::DoOpen(zeek::RecordValPtr options) {
+    bool open_fail = options->GetField<zeek::BoolVal>("open_fail")->Get();
+    if ( open_fail )
+        return "open_fail was set to true, returning error";
+
+    open = true;
+
+    return std::nullopt;
+}
+
+/**
+ * Finalizes the backend when it's being closed.
+ */
+void StorageDummy::Close() { open = false; }
+
+/**
+ * The workhorse method for Put(). This must be implemented by plugins.
+ */
+zeek::storage::ErrorResult StorageDummy::DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite) {
+    auto json_key = key->ToJSON()->ToStdString();
+    auto json_value = value->ToJSON()->ToStdString();
+    data[json_key] = json_value;
+    return std::nullopt;
+}
+
+/**
+ * The workhorse method for Get(). This must be implemented for plugins.
+ */
+zeek::storage::ValResult StorageDummy::DoGet(zeek::ValPtr key, zeek::TypePtr vt) {
+    auto json_key = key->ToJSON();
+    auto it = data.find(json_key->ToStdString());
+    if ( it == data.end() )
+        return zeek::unexpected<std::string>("Failed to find key");
+
+    auto val = zeek::detail::ValFromJSON(it->second.c_str(), vt, zeek::Func::nil);
+    if ( std::holds_alternative<zeek::ValPtr>(val) ) {
+        zeek::ValPtr val_v = std::get<zeek::ValPtr>(val);
+        return val_v;
+    }
+
+    return zeek::unexpected<std::string>(std::get<std::string>(val));
+}
+
+/**
+ * The workhorse method for Erase(). This must be implemented for plugins.
+ */
+zeek::storage::ErrorResult StorageDummy::DoErase(zeek::ValPtr key) {
+    auto json_key = key->ToJSON();
+    auto it = data.find(json_key->ToStdString());
+    if ( it == data.end() )
+        return "Failed to find key";
+
+    data.erase(it);
+    return std::nullopt;
+}
+
+} // namespace btest::storage::backend

testing/btest/plugins/storage-plugin/src/StorageDummy.h

@@ -0,0 +1,56 @@
+
+#pragma once
+
+#include <map>
+#include <string>
+
+#include "zeek/storage/Backend.h"
+
+namespace btest::storage::backend {
+
+/**
+ * A Foo reader to measure performance of the input framework.
+ */
+class StorageDummy : public zeek::storage::Backend {
+public:
+    StorageDummy(std::string_view tag) : Backend(tag) {}
+    ~StorageDummy() override = default;
+
+    static zeek::storage::BackendPtr Instantiate(std::string_view tag);
+
+    /**
+     * Called by the manager system to open the backend.
+     */
+    zeek::storage::ErrorResult DoOpen(zeek::RecordValPtr options) override;
+
+    /**
+     * Finalizes the backend when it's being closed.
+     */
+    void Close() override;
+
+    /**
+     * Returns whether the backend is opened.
+     */
+    bool IsOpen() override { return open; }
+
+    /**
+     * The workhorse method for Put().
+     */
+    zeek::storage::ErrorResult DoPut(zeek::ValPtr key, zeek::ValPtr value, bool overwrite = true) override;
+
+    /**
+     * The workhorse method for Get().
+     */
+    zeek::storage::ValResult DoGet(zeek::ValPtr key, zeek::TypePtr vt) override;
+
+    /**
+     * The workhorse method for Erase().
+     */
+    zeek::storage::ErrorResult DoErase(zeek::ValPtr key) override;
+
+private:
+    std::map<std::string, std::string> data;
+    bool open = false;
+};
+
+} // namespace btest::storage::backend

testing/btest/plugins/storage.zeek

@@ -0,0 +1,52 @@
+# @TEST-DOC: Basic test of a plugin implmenting a backend for the storage framework
+# @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1"
+
+# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Testing StorageDummy
+# @TEST-EXEC: cp -r %DIR/storage-plugin/* .
+# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make
+# @TEST-EXEC: ZEEK_PLUGIN_PATH=$(pwd) zeek -b Testing::StorageDummy %INPUT >> output 2>zeek-stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff zeek-stderr
+
+@load base/frameworks/storage
+
+# Create a typename here that can be passed down into get().
+type str: string;
+
+type StorageDummyOpts : record {
+	open_fail: bool;
+};
+
+event zeek_init() {
+	local opts : StorageDummyOpts;
+	opts$open_fail = F;
+
+	local key = "key1234";
+	local value = "value5678";
+
+	# Test basic operation. The second get() should return an error
+	# as the key should have been erased.
+	local b = Storage::open_backend(Storage::STORAGEDUMMY, opts);
+	local put_res = Storage::put(b, key, value, F);
+	local get_res = Storage::get(b, key, str);
+	if ( get_res is bool ) {
+		print("Got an invalid value in response!");
+	}
+
+	local erase_res = Storage::erase(b, key);
+	get_res = Storage::get(b, key, str);
+	Storage::close_backend(b);
+
+	# Test attempting to use the closed handle.
+	put_res = Storage::put(b, "a", "b", F);
+	get_res = Storage::get(b, "a", str);
+	erase_res = Storage::erase(b, "a");
+
+	print(fmt("results of trying to use closed handle: get: %d, put: %d, erase: %d",
+	          get_res, put_res, erase_res));
+
+	# Test failing to open the handle and test closing an invalid handle.
+	opts$open_fail = T;
+	local b2 = Storage::open_backend(Storage::STORAGEDUMMY, opts);
+	Storage::close_backend(b2);
+}