From e545fe825684a2c16fdde3563d17354c1e440a17 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 1 Apr 2025 16:14:35 -0700 Subject: [PATCH 1/9] Ground work for pluggable storage serializers --- scripts/base/frameworks/storage/async.zeek | 5 +- scripts/base/frameworks/storage/main.zeek | 17 ++++--- scripts/base/frameworks/storage/sync.zeek | 5 +- src/plugin/Component.cc | 2 + src/plugin/Component.h | 27 ++++++----- src/storage/Backend.cc | 11 ++++- src/storage/Backend.h | 4 +- src/storage/CMakeLists.txt | 1 + src/storage/Component.cc | 25 ++++++++-- src/storage/Component.h | 55 ++++++++++++++++++++-- src/storage/Manager.cc | 44 ++++++++++++----- src/storage/Manager.h | 20 +++++++- src/storage/ReturnCode.cc | 10 ++++ src/storage/ReturnCode.h | 2 + src/storage/Serializer.h | 50 ++++++++++++++++++++ src/storage/backend/redis/Plugin.cc | 2 +- src/storage/backend/sqlite/Plugin.cc | 2 +- src/storage/serializer/CMakeLists.txt | 0 src/storage/storage-async.bif | 4 +- src/storage/storage-sync.bif | 4 +- src/zeekygen/ScriptInfo.cc | 2 + 21 files changed, 235 insertions(+), 57 deletions(-) create mode 100644 src/storage/Serializer.h create mode 100644 src/storage/serializer/CMakeLists.txt diff --git a/scripts/base/frameworks/storage/async.zeek b/scripts/base/frameworks/storage/async.zeek index 3a30755afe..04398585f1 100644 --- a/scripts/base/frameworks/storage/async.zeek +++ b/scripts/base/frameworks/storage/async.zeek @@ -25,9 +25,8 @@ export { ## Returns: A record containing the status of the operation, and either an error ## string on failure or a value on success. The value returned here will ## be an ``opaque of BackendHandle``. - global open_backend: function(btype: Storage::Backend, - options: Storage::BackendOptions, key_type: any, val_type: any) - : Storage::OperationResult; + global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, + key_type: any, val_type: any): Storage::OperationResult; ## Closes an existing backend connection asynchronously. This method must be ## called via a :zeek:see:`when` condition or an error will be returned. diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index 5c05503c33..a1793dac37 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -7,23 +7,26 @@ export { ## :zeek:see:`Storage::Async::open_backend` and ## :zeek:see:`Storage::Sync::open_backend`. Backend plugins can redef this record ## to add relevant fields to it. - type BackendOptions: record { }; + type BackendOptions: record { + ## The serializer used for converting Zeek data. + serializer: Storage::Serializer &default=Storage::JSON; + }; ## Record for passing arguments to :zeek:see:`Storage::Async::put` and ## :zeek:see:`Storage::Sync::put`. type PutArgs: record { - # The key to store the value under. + ## The key to store the value under. key: any; - # The value to store associated with the key. + ## The value to store associated with the key. value: any; - # Indicates whether this value should overwrite an existing entry for the - # key. + ## Indicates whether this value should overwrite an existing entry for the + ## key. overwrite: bool &default=T; - # An interval of time until the entry is automatically removed from the - # backend. + ## An interval of time until the entry is automatically removed from the + ## backend. expire_time: interval &default=0sec; }; } diff --git a/scripts/base/frameworks/storage/sync.zeek b/scripts/base/frameworks/storage/sync.zeek index acc06dfbb4..042cf96420 100644 --- a/scripts/base/frameworks/storage/sync.zeek +++ b/scripts/base/frameworks/storage/sync.zeek @@ -23,9 +23,8 @@ export { ## Returns: A record containing the status of the operation, and either an error ## string on failure or a value on success. The value returned here will ## be an ``opaque of BackendHandle``. - global open_backend: function(btype: Storage::Backend, - options: Storage::BackendOptions, key_type: any, val_type: any) - : Storage::OperationResult; + global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions, + key_type: any, val_type: any): Storage::OperationResult; ## Closes an existing backend connection. ## diff --git a/src/plugin/Component.cc b/src/plugin/Component.cc index 7374622910..fda9bde98d 100644 --- a/src/plugin/Component.cc +++ b/src/plugin/Component.cc @@ -47,6 +47,8 @@ void Component::Describe(ODesc* d) const { case component::STORAGE_BACKEND: d->Add("Storage Backend"); break; + case component::STORAGE_SERIALIZER: d->Add("Storage Serializer"); break; + default: reporter->InternalWarning("unknown component type in plugin::Component::Describe"); d->Add(""); diff --git a/src/plugin/Component.h b/src/plugin/Component.h index 259fc4ebc8..62f11b0cb9 100644 --- a/src/plugin/Component.h +++ b/src/plugin/Component.h @@ -21,19 +21,20 @@ namespace component { * Component types. */ enum Type { - READER, /// An input reader (not currently used). - WRITER, /// A logging writer (not currently used). - ANALYZER, /// A protocol analyzer. - PACKET_ANALYZER, /// A packet analyzer. - FILE_ANALYZER, /// A file analyzer. - IOSOURCE, /// An I/O source, excluding packet sources. - PKTSRC, /// A packet source. - PKTDUMPER, /// A packet dumper. - SESSION_ADAPTER, /// A session adapter analyzer. - CLUSTER_BACKEND, /// A cluster backend. - EVENT_SERIALIZER, /// A serializer for events, used by cluster backends. - LOG_SERIALIZER, /// A serializer for log batches, used by cluster backends. - STORAGE_BACKEND, /// A backend for the storage framework. + READER, /// An input reader (not currently used). + WRITER, /// A logging writer (not currently used). + ANALYZER, /// A protocol analyzer. + PACKET_ANALYZER, /// A packet analyzer. + FILE_ANALYZER, /// A file analyzer. + IOSOURCE, /// An I/O source, excluding packet sources. + PKTSRC, /// A packet source. + PKTDUMPER, /// A packet dumper. + SESSION_ADAPTER, /// A session adapter analyzer. + CLUSTER_BACKEND, /// A cluster backend. + EVENT_SERIALIZER, /// A serializer for events, used by cluster backends. + LOG_SERIALIZER, /// A serializer for log batches, used by cluster backends. + STORAGE_BACKEND, /// A backend for the storage framework. + STORAGE_SERIALIZER, /// A serializer for the storage framework. }; } // namespace component diff --git a/src/storage/Backend.cc b/src/storage/Backend.cc index e71b9e2603..a8d951b0dc 100644 --- a/src/storage/Backend.cc +++ b/src/storage/Backend.cc @@ -66,7 +66,7 @@ void OpenResultCallback::Complete(OperationResult res) { } Backend::Backend(uint8_t modes, std::string_view tag_name) : modes(modes) { - tag = storage_mgr->GetComponentTag(std::string{tag_name}); + tag = storage_mgr->BackendMgr().GetComponentTag(std::string{tag_name}); tag_str = zeek::obj_desc_short(tag.AsVal().get()); } @@ -75,6 +75,15 @@ OperationResult Backend::Open(OpenResultCallback* cb, RecordValPtr options, Type val_type = std::move(vt); backend_options = options; + auto stype = options->GetField("serializer"); + zeek::Tag stag{stype}; + + auto s = storage_mgr->InstantiateSerializer(stag); + if ( ! s ) + return {ReturnCode::INITIALIZATION_FAILED, s.error()}; + + serializer = std::move(s.value()); + auto ret = DoOpen(cb, std::move(options)); if ( ! ret.value ) ret.value = cb->Backend(); diff --git a/src/storage/Backend.h b/src/storage/Backend.h index 702fde362b..17fb5e33bf 100644 --- a/src/storage/Backend.h +++ b/src/storage/Backend.h @@ -5,6 +5,7 @@ #include "zeek/OpaqueVal.h" #include "zeek/Tag.h" #include "zeek/Val.h" +#include "zeek/storage/Serializer.h" namespace zeek::detail::trigger { class Trigger; @@ -58,7 +59,7 @@ struct OperationResult { class ResultCallback { public: ResultCallback() = default; - ResultCallback(detail::trigger::TriggerPtr trigger, const void* assoc); + ResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc); virtual ~ResultCallback() = default; /** @@ -239,6 +240,7 @@ protected: zeek::Tag tag; std::string tag_str; + std::unique_ptr serializer; private: /** diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index b64c7d6016..c68b7b1af8 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -11,3 +11,4 @@ zeek_add_subdir_library( storage-sync.bif) add_subdirectory(backend) +add_subdirectory(serializer) diff --git a/src/storage/Component.cc b/src/storage/Component.cc index 1f5bb1da6e..cc0ee4cc4c 100644 --- a/src/storage/Component.cc +++ b/src/storage/Component.cc @@ -7,19 +7,34 @@ namespace zeek::storage { -Component::Component(const std::string& name, factory_callback arg_factory) - : plugin::Component(plugin::component::STORAGE_BACKEND, name, 0, storage_mgr->GetTagType()) { +BackendComponent::BackendComponent(const std::string& name, factory_callback arg_factory) + : plugin::Component(plugin::component::STORAGE_BACKEND, name, 0, storage_mgr->BackendMgr().GetTagType()) { factory = arg_factory; } -void Component::Initialize() { +void BackendComponent::Initialize() { InitializeTag(); - storage_mgr->RegisterComponent(this); + storage_mgr->BackendMgr().RegisterComponent(this); } -void Component::DoDescribe(ODesc* d) const { +void BackendComponent::DoDescribe(ODesc* d) const { d->Add("Storage::STORAGE_BACKEND_"); d->Add(CanonicalName()); } +SerializerComponent::SerializerComponent(const std::string& name, factory_callback arg_factory) + : plugin::Component(plugin::component::STORAGE_SERIALIZER, name, 0, storage_mgr->SerializerMgr().GetTagType()) { + factory = arg_factory; +} + +void SerializerComponent::Initialize() { + InitializeTag(); + storage_mgr->SerializerMgr().RegisterComponent(this); +} + +void SerializerComponent::DoDescribe(ODesc* d) const { + d->Add("Storage::STORAGE_SERIALIZER_"); + d->Add(CanonicalName()); +} + } // namespace zeek::storage diff --git a/src/storage/Component.h b/src/storage/Component.h index 6d7f5d1e6f..d7768b1d1a 100644 --- a/src/storage/Component.h +++ b/src/storage/Component.h @@ -7,11 +7,12 @@ namespace zeek::storage { class Backend; +class Serializer; /** * Component description for plugins providing storage backends. */ -class Component : public plugin::Component { +class BackendComponent : public plugin::Component { public: using factory_callback = IntrusivePtr (*)(); @@ -27,12 +28,60 @@ public: * method inside the class that just allocates and returns a new * instance. */ - Component(const std::string& name, factory_callback factory); + BackendComponent(const std::string& name, factory_callback factory); /** * Destructor. */ - ~Component() override = default; + ~BackendComponent() override = default; + + /** + * Initialization function. This function has to be called before any + * plugin component functionality is used; it is used to add the + * plugin component to the list of components and to initialize tags + */ + void Initialize() override; + + /** + * Returns the backend's factory function. + */ + factory_callback Factory() const { return factory; } + +protected: + /** + * Overridden from plugin::Component. + */ + void DoDescribe(ODesc* d) const override; + +private: + factory_callback factory; +}; + +/** + * Component description for plugins providing serialization for storage data.. + */ +class SerializerComponent : public plugin::Component { +public: + using factory_callback = std::unique_ptr (*)(); + + /** + * Constructor. + * + * @param name The name of the provided backend. This name is used + * across the system to identify the backend. + * + * @param factory A factory function to instantiate instances of the + * backend's class, which must be derived directly or indirectly from + * storage::Backend. This is typically a static \c Instantiate() + * method inside the class that just allocates and returns a new + * instance. + */ + SerializerComponent(const std::string& name, factory_callback factory); + + /** + * Destructor. + */ + ~SerializerComponent() override = default; /** * Initialization function. This function has to be called before any diff --git a/src/storage/Manager.cc b/src/storage/Manager.cc index 3ca04d5530..abd2848832 100644 --- a/src/storage/Manager.cc +++ b/src/storage/Manager.cc @@ -28,7 +28,9 @@ void detail::ExpirationTimer::Dispatch(double t, bool is_expire) { storage_mgr->StartExpirationTimer(); } -Manager::Manager() : plugin::ComponentManager("Storage", "Backend") {} +Manager::Manager() + : backend_mgr(plugin::ComponentManager("Storage", "Backend")), + serializer_mgr(plugin::ComponentManager("Storage", "Serializer")) {} Manager::~Manager() { // TODO: should we shut down any existing backends? force-poll until all of their existing @@ -48,24 +50,40 @@ void Manager::InitPostScript() { StartExpirationTimer(); } -zeek::expected Manager::Instantiate(const Tag& type) { - Component* c = Lookup(type); - if ( ! c ) { +zeek::expected Manager::InstantiateBackend(const Tag& type) { + BackendComponent* c = backend_mgr.Lookup(type); + if ( ! c ) return zeek::unexpected( - util::fmt("Request to open unknown backend (%d:%d)", type.Type(), type.Subtype())); - } + util::fmt("Request to instantiate unknown backend type (%d:%d)", type.Type(), type.Subtype())); - if ( ! c->Factory() ) { + if ( ! c->Factory() ) return zeek::unexpected( - util::fmt("Factory invalid for backend %s", GetComponentName(type).c_str())); - } + util::fmt("Factory invalid for backend %s", backend_mgr.GetComponentName(type).c_str())); - BackendPtr bp = c->Factory()(); + auto bp = c->Factory()(); - if ( ! bp ) { + if ( ! bp ) return zeek::unexpected( - util::fmt("Failed to instantiate backend %s", GetComponentName(type).c_str())); - } + util::fmt("Failed to instantiate backend %s", backend_mgr.GetComponentName(type).c_str())); + + return bp; +} + +zeek::expected, std::string> Manager::InstantiateSerializer(const Tag& type) { + SerializerComponent* c = serializer_mgr.Lookup(type); + if ( ! c ) + return zeek::unexpected( + util::fmt("Request to instantiate unknown serializer type (%d:%d)", type.Type(), type.Subtype())); + + if ( ! c->Factory() ) + return zeek::unexpected( + util::fmt("Factory invalid for serializer %s", serializer_mgr.GetComponentName(type).c_str())); + + auto bp = c->Factory()(); + + if ( ! bp ) + return zeek::unexpected( + util::fmt("Failed to instantiate serializer %s", serializer_mgr.GetComponentName(type).c_str())); return bp; } diff --git a/src/storage/Manager.h b/src/storage/Manager.h index ae3c1bab66..cdbc208b04 100644 --- a/src/storage/Manager.h +++ b/src/storage/Manager.h @@ -10,6 +10,7 @@ #include "zeek/plugin/ComponentManager.h" #include "zeek/storage/Backend.h" #include "zeek/storage/Component.h" +#include "zeek/storage/Serializer.h" namespace zeek::storage { @@ -24,7 +25,7 @@ public: } // namespace detail -class Manager final : public plugin::ComponentManager { +class Manager final { public: Manager(); ~Manager(); @@ -43,7 +44,16 @@ public: * @return A std::expected containing either a valid BackendPtr with the result of the * operation or a string containing an error message for failure. */ - zeek::expected Instantiate(const Tag& type); + zeek::expected InstantiateBackend(const Tag& type); + + /** + * Instantiates a new serializer object. + * + * @param type The tag for the type of backend being opened. + * @return A std::expected containing either a valid BackendPtr with the result of the + * operation or a string containing an error message for failure. + */ + zeek::expected, std::string> InstantiateSerializer(const Tag& type); /** * Opens a new storage backend. @@ -82,6 +92,9 @@ public: */ void Expire(double t); + plugin::ComponentManager& BackendMgr() { return backend_mgr; } + plugin::ComponentManager& SerializerMgr() { return serializer_mgr; } + protected: friend class storage::detail::ExpirationTimer; void RunExpireThread(); @@ -94,6 +107,9 @@ protected: private: std::vector backends; std::mutex backends_mtx; + + plugin::ComponentManager backend_mgr; + plugin::ComponentManager serializer_mgr; }; } // namespace zeek::storage diff --git a/src/storage/ReturnCode.cc b/src/storage/ReturnCode.cc index 22a3e8a2e6..3b78c6e4ae 100644 --- a/src/storage/ReturnCode.cc +++ b/src/storage/ReturnCode.cc @@ -19,6 +19,8 @@ EnumValPtr ReturnCode::CONNECTION_FAILED; EnumValPtr ReturnCode::DISCONNECTION_FAILED; EnumValPtr ReturnCode::INITIALIZATION_FAILED; EnumValPtr ReturnCode::IN_PROGRESS; +EnumValPtr ReturnCode::SERIALIZATION_FAILED; +EnumValPtr ReturnCode::UNSERIALIZATION_FAILED; void ReturnCode::Initialize() { static const auto& return_code_type = zeek::id::find_type("Storage::ReturnCode"); @@ -61,6 +63,12 @@ void ReturnCode::Initialize() { tmp = return_code_type->Lookup("Storage::IN_PROGRESS"); IN_PROGRESS = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::SERIALIZATION_FAILED"); + SERIALIZATION_FAILED = return_code_type->GetEnumVal(tmp); + + tmp = return_code_type->Lookup("Storage::UNSERIALIZATION_FAILED"); + UNSERIALIZATION_FAILED = return_code_type->GetEnumVal(tmp); } void ReturnCode::Cleanup() { @@ -77,6 +85,8 @@ void ReturnCode::Cleanup() { DISCONNECTION_FAILED.reset(); INITIALIZATION_FAILED.reset(); IN_PROGRESS.reset(); + SERIALIZATION_FAILED.reset(); + UNSERIALIZATION_FAILED.reset(); } } // namespace zeek::storage diff --git a/src/storage/ReturnCode.h b/src/storage/ReturnCode.h index 8b110641f0..db0bda7f16 100644 --- a/src/storage/ReturnCode.h +++ b/src/storage/ReturnCode.h @@ -32,6 +32,8 @@ public: static EnumValPtr DISCONNECTION_FAILED; static EnumValPtr INITIALIZATION_FAILED; static EnumValPtr IN_PROGRESS; + static EnumValPtr SERIALIZATION_FAILED; + static EnumValPtr UNSERIALIZATION_FAILED; }; } // namespace storage diff --git a/src/storage/Serializer.h b/src/storage/Serializer.h new file mode 100644 index 0000000000..dea136fcb2 --- /dev/null +++ b/src/storage/Serializer.h @@ -0,0 +1,50 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/Span.h" +#include "zeek/Val.h" + +namespace zeek::storage { + +namespace detail { +using byte_buffer = std::vector; +using byte_buffer_span = Span; +} // namespace detail + +/** + * Base class for a serializer used by storage backends. + */ +class Serializer { +public: + virtual ~Serializer() = default; + + /** + * Serializes Zeek Val data into another format. + * + * @param val The data to serialize. + * + * @return On success, a byte buffer containing the serialized data. std::nullopt will + * be returned on failure. + */ + virtual std::optional Serialize(ValPtr val) = 0; + + /** + * Unserializes a byte buffer into Zeek Val objects of a specific type. + * + * @param buf The byte data to unserialize. + * @param type The Zeek script-level type to unserialize the data into. + * + * @return A zeek::expected containing either the unserialized Val data on success, or + * a string containing an error message on failure. + */ + virtual zeek::expected Unserialize(detail::byte_buffer_span buf, TypePtr type) = 0; + +protected: + Serializer(std::string name) : name(std::move(name)) {} + +private: + std::string name; +}; + +} // namespace zeek::storage diff --git a/src/storage/backend/redis/Plugin.cc b/src/storage/backend/redis/Plugin.cc index 751a3996c9..a94707ca45 100644 --- a/src/storage/backend/redis/Plugin.cc +++ b/src/storage/backend/redis/Plugin.cc @@ -10,7 +10,7 @@ namespace zeek::storage::backend::redis { class Plugin : public plugin::Plugin { public: plugin::Configuration Configure() override { - AddComponent(new storage::Component("REDIS", backend::redis::Redis::Instantiate)); + AddComponent(new storage::BackendComponent("REDIS", backend::redis::Redis::Instantiate)); plugin::Configuration config; config.name = "Zeek::Storage_Backend_Redis"; diff --git a/src/storage/backend/sqlite/Plugin.cc b/src/storage/backend/sqlite/Plugin.cc index f6d56e77fb..8498e747d8 100644 --- a/src/storage/backend/sqlite/Plugin.cc +++ b/src/storage/backend/sqlite/Plugin.cc @@ -10,7 +10,7 @@ namespace zeek::storage::backend::sqlite { class Plugin : public plugin::Plugin { public: plugin::Configuration Configure() override { - AddComponent(new storage::Component("SQLITE", backend::sqlite::SQLite::Instantiate)); + AddComponent(new storage::BackendComponent("SQLITE", backend::sqlite::SQLite::Instantiate)); plugin::Configuration config; config.name = "Zeek::Storage_Backend_SQLite"; diff --git a/src/storage/serializer/CMakeLists.txt b/src/storage/serializer/CMakeLists.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/storage/storage-async.bif b/src/storage/storage-async.bif index ff56cef0cd..a705569236 100644 --- a/src/storage/storage-async.bif +++ b/src/storage/storage-async.bif @@ -80,9 +80,9 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, return nullptr; auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; - Tag tag{btype_val}; + Tag btag{btype_val}; - auto b = storage_mgr->Instantiate(tag); + auto b = storage_mgr->InstantiateBackend(btag); if ( ! b.has_value() ) { trigger->Cache( diff --git a/src/storage/storage-sync.bif b/src/storage/storage-sync.bif index a3a9e78e42..38c977bd3b 100644 --- a/src/storage/storage-sync.bif +++ b/src/storage/storage-sync.bif @@ -31,9 +31,9 @@ module Storage::Sync; function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): Storage::OperationResult %{ auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; - Tag tag{btype_val}; + Tag btag{btype_val}; - auto b = storage_mgr->Instantiate(tag); + auto b = storage_mgr->InstantiateBackend(btag); if ( ! b.has_value() ) { emit_builtin_error(b.error().c_str()); diff --git a/src/zeekygen/ScriptInfo.cc b/src/zeekygen/ScriptInfo.cc index 71464fa87d..8495b384f1 100644 --- a/src/zeekygen/ScriptInfo.cc +++ b/src/zeekygen/ScriptInfo.cc @@ -324,6 +324,8 @@ void ScriptInfo::DoInitPostScript() { else if ( name == "base/frameworks/storage/main.zeek" ) { const auto& backend_id = zeek::detail::global_scope()->Find("Storage::Backend"); types.push_back(new IdentifierInfo(backend_id, this)); + const auto& serializer_id = zeek::detail::global_scope()->Find("Storage::Serializer"); + types.push_back(new IdentifierInfo(serializer_id, this)); } } From 201d4508e6eafe4f9ab57cd9f1ee027e87685a30 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 2 Apr 2025 14:44:47 -0700 Subject: [PATCH 2/9] Make ValFromJSON return zeek::expected instead of a variant --- src/Val.cc | 90 +++++++++++++++------------- src/Val.h | 4 +- src/storage/backend/redis/Redis.cc | 6 +- src/storage/backend/sqlite/SQLite.cc | 12 ++-- src/supervisor/Supervisor.cc | 6 +- src/zeek.bif | 6 +- 6 files changed, 64 insertions(+), 60 deletions(-) diff --git a/src/Val.cc b/src/Val.cc index 18c2f7335c..451c9f6428 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -891,8 +891,8 @@ unsigned int StringVal::ComputeFootprint(std::unordered_set* analyze return 1 /* this object */ + static_cast(Len()) / sizeof(Val); } -static std::variant BuildVal(const rapidjson::Value& j, const TypePtr& t, - const FuncPtr& key_func) { +static zeek::expected BuildVal(const rapidjson::Value& j, const TypePtr& t, + const FuncPtr& key_func) { auto mismatch_err = [t, &j]() { std::string json_type; switch ( j.GetType() ) { @@ -906,7 +906,8 @@ static std::variant BuildVal(const rapidjson::Value& j, con default: json_type = "unknown"; } - return util::fmt("cannot convert JSON type '%s' to Zeek type '%s'", json_type.c_str(), type_name(t->Tag())); + return zeek::unexpected( + util::fmt("cannot convert JSON type '%s' to Zeek type '%s'", json_type.c_str(), type_name(t->Tag()))); }; if ( j.IsNull() ) @@ -960,7 +961,7 @@ static std::variant BuildVal(const rapidjson::Value& j, con parts.erase(std::remove_if(parts.begin(), parts.end(), [](auto x) { return x.empty(); }), parts.end()); if ( (parts.size() % 2) != 0 ) - return "wrong interval format, must be pairs of values with units"; + return zeek::unexpected("wrong interval format, must be pairs of values with units"); double interval_secs = 0.0; for ( size_t i = 0; i < parts.size(); i += 2 ) { @@ -980,7 +981,8 @@ static std::variant BuildVal(const rapidjson::Value& j, con else if ( unit == "usec" || unit == "usecs" ) interval_secs += (value * Microseconds); else - return util::fmt("wrong interval format, invalid unit type %s", unit.data()); + return zeek::unexpected( + util::fmt("wrong interval format, invalid unit type %s", unit.data())); } return make_intrusive(interval_secs, Seconds); @@ -991,11 +993,10 @@ static std::variant BuildVal(const rapidjson::Value& j, con case TYPE_PORT: { if ( j.IsString() ) { - int port = 0; if ( j.GetStringLength() > 0 && j.GetStringLength() < 10 ) { char* slash; errno = 0; - port = strtol(j.GetString(), &slash, 10); + auto port = strtol(j.GetString(), &slash, 10); if ( ! errno ) { ++slash; if ( util::streq(slash, "tcp") ) @@ -1009,15 +1010,17 @@ static std::variant BuildVal(const rapidjson::Value& j, con } } - return "wrong port format, string must be /[0-9]{1,5}\\/(tcp|udp|icmp|unknown)/"; + return zeek::unexpected( + "wrong port format, string must be /[0-9]{1,5}\\/(tcp|udp|icmp|unknown)/"); } else if ( j.IsObject() ) { if ( ! j.HasMember("port") || ! j.HasMember("proto") ) - return "wrong port format, object must have 'port' and 'proto' members"; + return zeek::unexpected( + "wrong port format, object must have 'port' and 'proto' members"); if ( ! j["port"].IsNumber() ) - return "wrong port format, port must be a number"; + return zeek::unexpected("wrong port format, port must be a number"); if ( ! j["proto"].IsString() ) - return "wrong port format, protocol must be a string"; + return zeek::unexpected("wrong port format, protocol must be a string"); std::string proto{j["proto"].GetString()}; @@ -1030,10 +1033,10 @@ static std::variant BuildVal(const rapidjson::Value& j, con if ( proto == "unknown" ) return val_mgr->Port(j["port"].GetInt(), TRANSPORT_UNKNOWN); - return "wrong port format, invalid protocol string"; + return zeek::unexpected("wrong port format, invalid protocol string"); } else - return "wrong port format, must be string or object"; + return zeek::unexpected("wrong port format, must be string or object"); } case TYPE_PATTERN: { @@ -1055,7 +1058,7 @@ static std::variant BuildVal(const rapidjson::Value& j, con auto re = std::make_unique(candidate.c_str()); if ( ! re->Compile() ) - return "error compiling pattern"; + return zeek::unexpected("error compiling pattern"); return make_intrusive(re.release()); } @@ -1074,7 +1077,7 @@ static std::variant BuildVal(const rapidjson::Value& j, con std::string_view subnet_sv(j.GetString(), j.GetStringLength()); auto pos = subnet_sv.find('/'); if ( pos == subnet_sv.npos ) - return util::fmt("invalid value for subnet: '%s'", j.GetString()); + return zeek::unexpected(util::fmt("invalid value for subnet: '%s'", j.GetString())); candidate = std::string(j.GetString(), pos); @@ -1082,7 +1085,7 @@ static std::variant BuildVal(const rapidjson::Value& j, con char* end; width = strtol(subnet_sv.data() + pos + 1, &end, 10); if ( subnet_sv.data() + pos + 1 == end || errno ) - return util::fmt("invalid value for subnet: '%s'", j.GetString()); + return zeek::unexpected(util::fmt("invalid value for subnet: '%s'", j.GetString())); } if ( candidate.front() == '[' ) @@ -1104,7 +1107,8 @@ static std::variant BuildVal(const rapidjson::Value& j, con auto intval = et->Lookup({j.GetString(), j.GetStringLength()}); if ( intval < 0 ) - return util::fmt("'%s' is not a valid enum for '%s'.", j.GetString(), et->GetName().c_str()); + return zeek::unexpected( + util::fmt("'%s' is not a valid enum for '%s'.", j.GetString(), et->GetName().c_str())); return et->GetEnumVal(intval); } @@ -1126,19 +1130,19 @@ static std::variant BuildVal(const rapidjson::Value& j, con return mismatch_err(); for ( const auto& item : j.GetArray() ) { - std::variant v; + zeek::expected v; if ( tl->GetTypes().size() == 1 ) v = BuildVal(item, tl->GetPureType(), key_func); else v = BuildVal(item, tl, key_func); - if ( ! get_if(&v) ) + if ( ! v ) return v; - if ( ! std::get(v) ) + if ( v.value() == nullptr ) continue; - tv->Assign(std::move(std::get(v)), nullptr); + tv->Assign(v.value(), nullptr); } return tv; @@ -1151,7 +1155,7 @@ static std::variant BuildVal(const rapidjson::Value& j, con rapidjson::Document idxstr; idxstr.Parse(it->name.GetString(), it->name.GetStringLength()); - std::variant idx; + zeek::expected idx; if ( tl->GetTypes().size() > 1 ) idx = BuildVal(idxstr, tl, key_func); @@ -1163,19 +1167,19 @@ static std::variant BuildVal(const rapidjson::Value& j, con // Parse the string's content, not the full JSON string. idx = BuildVal(idxstr, tl->GetPureType(), key_func); - if ( ! get_if(&idx) ) + if ( ! idx ) return idx; - if ( ! std::get(idx) ) + if ( idx.value() == nullptr ) continue; auto v = BuildVal(it->value, tt->Yield(), key_func); - if ( ! get_if(&v) ) + if ( ! v ) return v; - if ( ! std::get(v) ) + if ( v.value() == nullptr ) continue; - tv->Assign(std::move(std::get(idx)), std::move(std::get(v))); + tv->Assign(idx.value(), v.value()); } return tv; @@ -1202,7 +1206,7 @@ static std::variant BuildVal(const rapidjson::Value& j, con } if ( ! result ) - return "key function error"; + return zeek::unexpected("key function error"); normalized_keys[result->AsStringVal()->CheckString()] = &it->value; } @@ -1226,17 +1230,18 @@ static std::variant BuildVal(const rapidjson::Value& j, con if ( ! td_i->GetAttr(detail::ATTR_OPTIONAL) && ! td_i->GetAttr(detail::ATTR_DEFAULT) ) // jval being set means it is a null JSON value else // it wasn't even there. - return util::fmt("required field %s$%s is %s in JSON", t->GetName().c_str(), td_i->id, - jval ? "null" : "missing"); + return zeek::unexpected(util::fmt("required field %s$%s is %s in JSON", + t->GetName().c_str(), td_i->id, + jval ? "null" : "missing")); continue; } auto v = BuildVal(*jval, td_i->type, key_func); - if ( ! get_if(&v) ) + if ( ! v ) return v; - rv->Assign(i, std::move(std::get(v))); + rv->Assign(i, v.value()); } return rv; @@ -1249,16 +1254,16 @@ static std::variant BuildVal(const rapidjson::Value& j, con auto lt = t->AsTypeList(); if ( j.GetArray().Size() < lt->GetTypes().size() ) - return "index type doesn't match"; + return zeek::unexpected("index type doesn't match"); auto lv = make_intrusive(TYPE_ANY); for ( size_t i = 0; i < lt->GetTypes().size(); i++ ) { auto v = BuildVal(j.GetArray()[i], lt->GetTypes()[i], key_func); - if ( ! get_if(&v) ) + if ( ! v ) return v; - lv->Append(std::move(std::get(v))); + lv->Append(v.value()); } return lv; @@ -1272,29 +1277,30 @@ static std::variant BuildVal(const rapidjson::Value& j, con auto vv = make_intrusive(IntrusivePtr{NewRef{}, vt}); for ( const auto& item : j.GetArray() ) { auto v = BuildVal(item, vt->Yield(), key_func); - if ( ! get_if(&v) ) + if ( ! v ) return v; - if ( ! std::get(v) ) + if ( v.value() == nullptr ) continue; - vv->Assign(vv->Size(), std::move(std::get(v))); + vv->Assign(vv->Size(), v.value()); } return vv; } - default: return util::fmt("type '%s' unsupported", type_name(t->Tag())); + default: return zeek::unexpected(util::fmt("type '%s' unsupported", type_name(t->Tag()))); } } -std::variant detail::ValFromJSON(std::string_view json_str, const TypePtr& t, - const FuncPtr& key_func) { +zeek::expected detail::ValFromJSON(std::string_view json_str, const TypePtr& t, + const FuncPtr& key_func) { rapidjson::Document doc; rapidjson::ParseResult ok = doc.Parse(json_str.data(), json_str.length()); if ( ! ok ) - return util::fmt("JSON parse error: %s Offset: %lu", rapidjson::GetParseError_En(ok.Code()), ok.Offset()); + return zeek::unexpected( + util::fmt("JSON parse error: %s Offset: %lu", rapidjson::GetParseError_En(ok.Code()), ok.Offset())); return BuildVal(doc, t, key_func); } diff --git a/src/Val.h b/src/Val.h index bc6de01ddd..850b89dc81 100644 --- a/src/Val.h +++ b/src/Val.h @@ -1774,8 +1774,8 @@ namespace detail { // // The *key_func* parameter is a Zeek script function called for every JSON key // for normalization. If Func::nil is passed, no normalization happens. -extern std::variant ValFromJSON(std::string_view json_str, const TypePtr& t, - const FuncPtr& key_func); +extern zeek::expected ValFromJSON(std::string_view json_str, const TypePtr& t, + const FuncPtr& key_func); // If the given vector is an empty vector-of-any ("unspecified"), // concretizes it to the given type. *v* gives the vector and *t* the diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 47666a3c4d..3e5a4f30d5 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -440,10 +440,10 @@ void Redis::HandleGetResult(redisReply* reply, ResultCallback* callback) { res = ParseReplyError("get", reply->str); else { auto val = zeek::detail::ValFromJSON(reply->str, val_type, Func::nil); - if ( std::holds_alternative(val) ) - res = {ReturnCode::SUCCESS, "", std::get(val)}; + if ( val ) + res = {ReturnCode::SUCCESS, "", val.value()}; else - res = {ReturnCode::OPERATION_FAILED, std::get(val)}; + res = {ReturnCode::OPERATION_FAILED, val.error()}; } freeReplyObject(reply); diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index 4758ac071c..a51c2f21bf 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -270,13 +270,11 @@ OperationResult SQLite::Step(sqlite3_stmt* stmt, bool parse_value) { const char* text = (const char*)sqlite3_column_text(stmt, 0); auto val = zeek::detail::ValFromJSON(text, val_type, Func::nil); sqlite3_reset(stmt); - if ( std::holds_alternative(val) ) { - ValPtr val_v = std::get(val); - ret = {ReturnCode::SUCCESS, "", val_v}; - } - else { - ret = {ReturnCode::OPERATION_FAILED, std::get(val)}; - } + + if ( val ) + ret = {ReturnCode::SUCCESS, "", val.value()}; + else + ret = {ReturnCode::OPERATION_FAILED, val.error()}; } else { ret = {ReturnCode::OPERATION_FAILED, "sqlite3_step should not have returned a value"}; diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index 0585ec6467..22e43f004d 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -1363,8 +1363,8 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const { auto tt = rt->GetFieldType("cluster"); auto json_res = detail::ValFromJSON(cluster, tt, Func::nil); - if ( auto val = std::get_if(&json_res) ) { - rval->AssignField("cluster", *val); + if ( json_res ) { + rval->AssignField("cluster", json_res.value()); } else { // This should never happen: the JSON data comes from a table[string] of @@ -1372,7 +1372,7 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const { // here can be hard to debug. Other JSON code (see FromJSON()) fails // silently when the JSON is misformatted. We just warn: fprintf(stderr, "Could not parse %s's cluster table from '%s': %s\n", name.c_str(), cluster.c_str(), - std::get(json_res).c_str()); + json_res.error().c_str()); rval->AssignField("cluster", make_intrusive(std::move(tt))); } diff --git a/src/zeek.bif b/src/zeek.bif index 7c5fac0f09..8702a0a7fa 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -5268,15 +5268,15 @@ function from_json%(s: string, t: any, key_func: string_mapper &default=from_jso auto res = zeek::detail::ValFromJSON(s->ToStdStringView(), t->AsType()->AsTypeType()->GetType(), key_func_ptr); - if ( auto val = std::get_if(&res) ) + if ( res ) { - rval->Assign(v_idx, *val); + rval->Assign(v_idx, res.value()); rval->Assign(valid_idx, true); } else { rval->Assign(valid_idx, false); - zeek::emit_builtin_error(std::get(res).c_str()); + zeek::emit_builtin_error(res.error().c_str()); } return std::move(rval); From 88786a28a296819220da648f78a08a4e181a2dd1 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 3 Apr 2025 12:38:18 -0700 Subject: [PATCH 3/9] Add JSON storage serializer, use with existing backends/tests --- src/storage/backend/redis/Redis.cc | 42 ++++++++++++------- src/storage/backend/sqlite/SQLite.cc | 42 +++++++++++-------- src/storage/serializer/CMakeLists.txt | 1 + src/storage/serializer/json/CMakeLists.txt | 3 ++ src/storage/serializer/json/JSON.cc | 29 +++++++++++++ src/storage/serializer/json/JSON.h | 20 +++++++++ src/storage/serializer/json/Plugin.cc | 22 ++++++++++ src/storage/storage-async.bif | 4 +- src/storage/storage-sync.bif | 4 +- .../out | 4 +- .../out | 4 +- .../out | 2 +- .../plugins/storage-plugin/src/Plugin.cc | 3 +- .../storage-plugin/src/StorageDummy.cc | 35 ++++++++-------- .../plugins/storage-plugin/src/StorageDummy.h | 2 +- .../frameworks/storage/redis-disconnect.zeek | 1 + .../base/frameworks/storage/redis-sync.zeek | 1 + .../base/frameworks/storage/sqlite-basic.zeek | 1 + 18 files changed, 161 insertions(+), 59 deletions(-) create mode 100644 src/storage/serializer/json/CMakeLists.txt create mode 100644 src/storage/serializer/json/JSON.cc create mode 100644 src/storage/serializer/json/JSON.h create mode 100644 src/storage/serializer/json/Plugin.cc diff --git a/src/storage/backend/redis/Redis.cc b/src/storage/backend/redis/Redis.cc index 3e5a4f30d5..d5e96d7dfe 100644 --- a/src/storage/backend/redis/Redis.cc +++ b/src/storage/backend/redis/Redis.cc @@ -254,24 +254,30 @@ OperationResult Redis::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); - std::string format = "SET %s:%s %s"; + std::string format = "SET %s:%b %b"; if ( ! overwrite ) format.append(" NX"); - auto json_key = key->ToJSON()->ToStdString(); - auto json_value = value->ToJSON()->ToStdString(); + auto key_data = serializer->Serialize(key); + if ( ! key_data ) + return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"}; + + auto val_data = serializer->Serialize(value); + if ( ! val_data ) + return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize value"}; int status; // Use built-in expiration if reading live data, since time will move // forward consistently. If reading pcaps, we'll do something else. if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) { format.append(" PXAT %" PRIu64); - status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), - json_value.data(), static_cast(expiration_time * 1e3)); + status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), key_data->data(), + key_data->size(), val_data->data(), val_data->size(), + static_cast(expiration_time * 1e3)); } else - status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), - json_value.data()); + status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), key_data->data(), + key_data->size(), val_data->data(), val_data->size()); if ( connected && status == REDIS_ERR ) return {ReturnCode::OPERATION_FAILED, util::fmt("Failed to queue put operation: %s", async_ctx->errstr)}; @@ -284,10 +290,10 @@ OperationResult Redis::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool format = "ZADD %s_expire"; if ( ! overwrite ) format.append(" NX"); - format += " %f %s"; + format += " %f %b"; status = redisAsyncCommand(async_ctx, redisZADD, NULL, format.c_str(), key_prefix.data(), expiration_time, - json_key.data()); + key_data->data(), key_data->size()); if ( connected && status == REDIS_ERR ) return {ReturnCode::OPERATION_FAILED, util::fmt("ZADD operation failed: %s", async_ctx->errstr)}; @@ -307,8 +313,12 @@ OperationResult Redis::DoGet(ResultCallback* cb, ValPtr key) { auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); - int status = redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%s", key_prefix.data(), - key->ToJSON()->ToStdStringView().data()); + auto key_data = serializer->Serialize(key); + if ( ! key_data ) + return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"}; + + int status = + redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%b", key_prefix.data(), key_data->data(), key_data->size()); if ( connected && status == REDIS_ERR ) return {ReturnCode::OPERATION_FAILED, util::fmt("Failed to queue get operation: %s", async_ctx->errstr)}; @@ -330,8 +340,12 @@ OperationResult Redis::DoErase(ResultCallback* cb, ValPtr key) { auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); - int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%s", key_prefix.data(), - key->ToJSON()->ToStdStringView().data()); + auto key_data = serializer->Serialize(key); + if ( ! key_data ) + return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"}; + + int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%b", key_prefix.data(), key_data->data(), + key_data->size()); if ( connected && status == REDIS_ERR ) return {ReturnCode::OPERATION_FAILED, async_ctx->errstr}; @@ -439,7 +453,7 @@ void Redis::HandleGetResult(redisReply* reply, ResultCallback* callback) { else if ( reply->type == REDIS_REPLY_ERROR ) res = ParseReplyError("get", reply->str); else { - auto val = zeek::detail::ValFromJSON(reply->str, val_type, Func::nil); + auto val = serializer->Unserialize({(std::byte*)reply->str, reply->len}, val_type); if ( val ) res = {ReturnCode::SUCCESS, "", val.value()}; else diff --git a/src/storage/backend/sqlite/SQLite.cc b/src/storage/backend/sqlite/SQLite.cc index a51c2f21bf..e5ddbd15b5 100644 --- a/src/storage/backend/sqlite/SQLite.cc +++ b/src/storage/backend/sqlite/SQLite.cc @@ -44,7 +44,7 @@ OperationResult SQLite::DoOpen(OpenResultCallback* cb, RecordValPtr options) { } std::string create = "create table if not exists " + table_name + " ("; - create.append("key_str text primary key, value_str text not null, expire_time real);"); + create.append("key_str blob primary key, value_str blob not null, expire_time real);"); char* errorMsg = nullptr; if ( int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); res != SQLITE_OK ) { @@ -151,8 +151,9 @@ OperationResult SQLite::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool if ( ! db ) return {ReturnCode::NOT_CONNECTED}; - auto json_key = key->ToJSON(); - auto json_value = value->ToJSON(); + auto key_data = serializer->Serialize(key); + if ( ! key_data ) + return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"}; sqlite3_stmt* stmt; if ( ! overwrite ) @@ -160,15 +161,17 @@ OperationResult SQLite::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool else stmt = put_update_stmt.get(); - auto key_str = json_key->ToStdStringView(); - if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); + if ( auto res = CheckError(sqlite3_bind_blob(stmt, 1, key_data->data(), key_data->size(), SQLITE_STATIC)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; } - auto value_str = json_value->ToStdStringView(); - if ( auto res = CheckError(sqlite3_bind_text(stmt, 2, value_str.data(), value_str.size(), SQLITE_STATIC)); + auto val_data = serializer->Serialize(value); + if ( ! val_data ) + return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize value"}; + + if ( auto res = CheckError(sqlite3_bind_blob(stmt, 2, val_data->data(), val_data->size(), SQLITE_STATIC)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; @@ -180,7 +183,7 @@ OperationResult SQLite::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool } if ( overwrite ) { - if ( auto res = CheckError(sqlite3_bind_text(stmt, 4, value_str.data(), value_str.size(), SQLITE_STATIC)); + if ( auto res = CheckError(sqlite3_bind_blob(stmt, 4, val_data->data(), val_data->size(), SQLITE_STATIC)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; @@ -197,11 +200,13 @@ OperationResult SQLite::DoGet(ResultCallback* cb, ValPtr key) { if ( ! db ) return {ReturnCode::NOT_CONNECTED}; - auto json_key = key->ToJSON(); + auto key_data = serializer->Serialize(key); + if ( ! key_data ) + return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"}; + auto stmt = get_stmt.get(); - auto key_str = json_key->ToStdStringView(); - if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); + if ( auto res = CheckError(sqlite3_bind_blob(stmt, 1, key_data->data(), key_data->size(), SQLITE_STATIC)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; @@ -217,11 +222,13 @@ OperationResult SQLite::DoErase(ResultCallback* cb, ValPtr key) { if ( ! db ) return {ReturnCode::NOT_CONNECTED}; - auto json_key = key->ToJSON(); + auto key_data = serializer->Serialize(key); + if ( ! key_data ) + return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"}; + auto stmt = erase_stmt.get(); - auto key_str = json_key->ToStdStringView(); - if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC)); + if ( auto res = CheckError(sqlite3_bind_blob(stmt, 1, key_data->data(), key_data->size(), SQLITE_STATIC)); res.code != ReturnCode::SUCCESS ) { sqlite3_reset(stmt); return res; @@ -266,9 +273,10 @@ OperationResult SQLite::Step(sqlite3_stmt* stmt, bool parse_value) { int step_status = sqlite3_step(stmt); if ( step_status == SQLITE_ROW ) { if ( parse_value ) { - // Column 1 is the value - const char* text = (const char*)sqlite3_column_text(stmt, 0); - auto val = zeek::detail::ValFromJSON(text, val_type, Func::nil); + auto blob = static_cast(sqlite3_column_blob(stmt, 0)); + size_t blob_size = sqlite3_column_bytes(stmt, 0); + + auto val = serializer->Unserialize({blob, blob_size}, val_type); sqlite3_reset(stmt); if ( val ) diff --git a/src/storage/serializer/CMakeLists.txt b/src/storage/serializer/CMakeLists.txt index e69de29bb2..7a340d53da 100644 --- a/src/storage/serializer/CMakeLists.txt +++ b/src/storage/serializer/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(json) diff --git a/src/storage/serializer/json/CMakeLists.txt b/src/storage/serializer/json/CMakeLists.txt new file mode 100644 index 0000000000..2c9a16f9d7 --- /dev/null +++ b/src/storage/serializer/json/CMakeLists.txt @@ -0,0 +1,3 @@ +zeek_add_plugin( + Zeek Storage_Serializer_JSON + SOURCES JSON.cc Plugin.cc) diff --git a/src/storage/serializer/json/JSON.cc b/src/storage/serializer/json/JSON.cc new file mode 100644 index 0000000000..8625eaa0be --- /dev/null +++ b/src/storage/serializer/json/JSON.cc @@ -0,0 +1,29 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/storage/serializer/json/JSON.h" + +#include "zeek/Func.h" + +namespace zeek::storage::serializer::json { + +std::unique_ptr JSON::Instantiate() { return std::make_unique(); } + +JSON::JSON() : Serializer("JSON") {} + +std::optional JSON::Serialize(ValPtr val) { + detail::byte_buffer buf; + auto json = val->ToJSON(); + buf.reserve(json->Len()); + + std::transform(json->Bytes(), json->Bytes() + json->Len(), std::back_inserter(buf), + [](u_char c) { return std::byte(c); }); + + return buf; +} + +zeek::expected JSON::Unserialize(detail::byte_buffer_span buf, TypePtr type) { + std::string_view text{reinterpret_cast(buf.data()), buf.size()}; + return zeek::detail::ValFromJSON(text, type, Func::nil); +} + +} // namespace zeek::storage::serializer::json diff --git a/src/storage/serializer/json/JSON.h b/src/storage/serializer/json/JSON.h new file mode 100644 index 0000000000..2b5475936a --- /dev/null +++ b/src/storage/serializer/json/JSON.h @@ -0,0 +1,20 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/storage/Serializer.h" + +namespace zeek::storage::serializer::json { + +class JSON final : public Serializer { +public: + static std::unique_ptr Instantiate(); + + JSON(); + ~JSON() override = default; + + std::optional Serialize(ValPtr val) override; + zeek::expected Unserialize(detail::byte_buffer_span buf, TypePtr type) override; +}; + +} // namespace zeek::storage::serializer::json diff --git a/src/storage/serializer/json/Plugin.cc b/src/storage/serializer/json/Plugin.cc new file mode 100644 index 0000000000..7f4055b354 --- /dev/null +++ b/src/storage/serializer/json/Plugin.cc @@ -0,0 +1,22 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/plugin/Plugin.h" + +#include "zeek/storage/Component.h" +#include "zeek/storage/serializer/json/JSON.h" + +namespace zeek::storage::serializer::json { + +class Plugin final : public plugin::Plugin { +public: + plugin::Configuration Configure() override { + AddComponent(new storage::SerializerComponent("JSON", serializer::json::JSON::Instantiate)); + + plugin::Configuration config; + config.name = "Zeek::Storage_Serializer_JSON"; + config.description = "JSON serializer for storage framework"; + return config; + } +} plugin; + +} // namespace zeek::storage::serializer::json diff --git a/src/storage/storage-async.bif b/src/storage/storage-async.bif index a705569236..fecfcfe341 100644 --- a/src/storage/storage-async.bif +++ b/src/storage/storage-async.bif @@ -80,9 +80,9 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any, return nullptr; auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; - Tag btag{btype_val}; + Tag tag{btype_val}; - auto b = storage_mgr->InstantiateBackend(btag); + auto b = storage_mgr->InstantiateBackend(tag); if ( ! b.has_value() ) { trigger->Cache( diff --git a/src/storage/storage-sync.bif b/src/storage/storage-sync.bif index 38c977bd3b..8c20ff4d3b 100644 --- a/src/storage/storage-sync.bif +++ b/src/storage/storage-sync.bif @@ -31,9 +31,9 @@ module Storage::Sync; function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, key_type: any, val_type: any%): Storage::OperationResult %{ auto btype_val = IntrusivePtr{NewRef{}, btype->AsEnumVal()}; - Tag btag{btype_val}; + Tag tag{btype_val}; - auto b = storage_mgr->InstantiateBackend(btag); + auto b = storage_mgr->InstantiateBackend(tag); if ( ! b.has_value() ) { emit_builtin_error(b.error().c_str()); diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out index 3308aef14e..d87e524a8b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. open_result, [code=Storage::SUCCESS, error_str=, value=] -Storage::backend_opened, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]] -Storage::backend_lost, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]], Server closed the connection +Storage::backend_opened, Storage::REDIS, [serializer=Storage::JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]] +Storage::backend_lost, Storage::REDIS, [serializer=Storage::JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]], Server closed the connection diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out index 0053e1555d..52cc495f62 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out @@ -9,5 +9,5 @@ get result same as originally inserted, T put result, [code=Storage::SUCCESS, error_str=, value=] get result, [code=Storage::SUCCESS, error_str=, value=value2345] get result same as overwritten, T -Storage::backend_opened, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]] -Storage::backend_lost, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]], Client disconnected +Storage::backend_opened, Storage::REDIS, [serializer=Storage::JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]] +Storage::backend_lost, Storage::REDIS, [serializer=Storage::JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]], Client disconnected diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out index 781e32a333..8ad2a12b65 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out @@ -1,5 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -Storage::backend_opened, Storage::SQLITE, [sqlite=[database_path=test.sqlite, table_name=testing, tuning_params={ +Storage::backend_opened, Storage::SQLITE, [serializer=Storage::JSON, sqlite=[database_path=test.sqlite, table_name=testing, tuning_params={ [synchronous] = normal, [temp_store] = memory, [journal_mode] = WAL diff --git a/testing/btest/plugins/storage-plugin/src/Plugin.cc b/testing/btest/plugins/storage-plugin/src/Plugin.cc index ca7e42f4e6..e73eb1fdfb 100644 --- a/testing/btest/plugins/storage-plugin/src/Plugin.cc +++ b/testing/btest/plugins/storage-plugin/src/Plugin.cc @@ -11,7 +11,8 @@ Plugin plugin; using namespace btest::plugin::Testing_StorageDummy; zeek::plugin::Configuration Plugin::Configure() { - AddComponent(new zeek::storage::Component("StorageDummy", btest::storage::backend::StorageDummy::Instantiate)); + AddComponent( + new zeek::storage::BackendComponent("StorageDummy", btest::storage::backend::StorageDummy::Instantiate)); zeek::plugin::Configuration config; config.name = "Testing::StorageDummy"; diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc index f05f3ed403..59471dac18 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.cc +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.cc @@ -49,9 +49,10 @@ OperationResult StorageDummy::DoPut(ResultCallback* cb, ValPtr key, ValPtr value if ( timeout_put ) return {ReturnCode::TIMEOUT}; - auto json_key = key->ToJSON()->ToStdString(); - auto json_value = value->ToJSON()->ToStdString(); - data[json_key] = json_value; + auto key_data = serializer->Serialize(key); + auto val_data = serializer->Serialize(value); + + data[*key_data] = *val_data; return {ReturnCode::SUCCESS}; } @@ -59,31 +60,31 @@ OperationResult StorageDummy::DoPut(ResultCallback* cb, ValPtr key, ValPtr value * The workhorse method for Get(). This must be implemented for plugins. */ OperationResult StorageDummy::DoGet(ResultCallback* cb, ValPtr key) { - auto json_key = key->ToJSON(); - auto it = data.find(json_key->ToStdString()); + auto key_data = serializer->Serialize(key); + + auto it = data.find(*key_data); if ( it == data.end() ) return {ReturnCode::KEY_NOT_FOUND}; - auto val = zeek::detail::ValFromJSON(it->second.c_str(), val_type, Func::nil); - if ( std::holds_alternative(val) ) { - ValPtr val_v = std::get(val); - return {ReturnCode::SUCCESS, "", val_v}; - } + auto val = serializer->Unserialize(it->second, val_type); + if ( val ) + return {ReturnCode::SUCCESS, "", val.value()}; - return {ReturnCode::OPERATION_FAILED, std::get(val)}; + return {ReturnCode::UNSERIALIZATION_FAILED, val.error()}; } /** * The workhorse method for Erase(). This must be implemented for plugins. */ OperationResult StorageDummy::DoErase(ResultCallback* cb, ValPtr key) { - auto json_key = key->ToJSON(); - auto it = data.find(json_key->ToStdString()); - if ( it == data.end() ) - return {ReturnCode::KEY_NOT_FOUND}; + auto key_data = serializer->Serialize(key); - data.erase(it); - return {ReturnCode::SUCCESS}; + if ( auto it = data.find(*key_data); it != data.end() ) { + data.erase(it); + return {ReturnCode::SUCCESS}; + } + + return {ReturnCode::KEY_NOT_FOUND}; } } // namespace btest::storage::backend diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h index 0fa718fc4c..c295aee82e 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.h +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -50,7 +50,7 @@ public: zeek::storage::OperationResult DoErase(zeek::storage::ResultCallback* cb, zeek::ValPtr key) override; private: - std::map data; + std::map data; bool open = false; }; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek b/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek index 33f66b1d43..55a8f2c320 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek @@ -26,6 +26,7 @@ event Storage::backend_lost(tag: Storage::Backend, config: any, reason: string) event zeek_init() { local opts: Storage::BackendOptions; + opts$serializer = Storage::JSON; opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( "REDIS_PORT")), $key_prefix="testing" ]; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek index ffaf42f4c8..e19503df5d 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -24,6 +24,7 @@ event Storage::backend_lost(tag: Storage::Backend, config: any, reason: string) event zeek_init() { local opts: Storage::BackendOptions; + opts$serializer = Storage::JSON; opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( "REDIS_PORT")), $key_prefix="testing" ]; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index c910353b0e..2e6a815f97 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -16,6 +16,7 @@ event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table local opts: Storage::BackendOptions; + opts$serializer = Storage::JSON; opts$sqlite = [ $database_path="test.sqlite", $table_name="testing" ]; local key = "key1234"; From 98bd85b805bdcb99f65769e466a5930be353b143 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 3 Apr 2025 12:54:08 -0700 Subject: [PATCH 4/9] Mark storage classes as final where appropriate --- src/storage/ReturnCode.h | 2 +- src/storage/backend/redis/Plugin.cc | 2 +- src/storage/backend/redis/Redis.h | 2 +- src/storage/backend/sqlite/Plugin.cc | 2 +- src/storage/backend/sqlite/SQLite.h | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/storage/ReturnCode.h b/src/storage/ReturnCode.h index db0bda7f16..2ddbd6ac25 100644 --- a/src/storage/ReturnCode.h +++ b/src/storage/ReturnCode.h @@ -14,7 +14,7 @@ namespace storage { * A collection of EnumValPtrs for the default set of result codes in the storage framework. * should be kept up-to-date with the Storage::ReturnCodes script-level enum. */ -class ReturnCode { +class ReturnCode final { public: static void Initialize(); static void Cleanup(); diff --git a/src/storage/backend/redis/Plugin.cc b/src/storage/backend/redis/Plugin.cc index a94707ca45..aa94f80294 100644 --- a/src/storage/backend/redis/Plugin.cc +++ b/src/storage/backend/redis/Plugin.cc @@ -7,7 +7,7 @@ namespace zeek::storage::backend::redis { -class Plugin : public plugin::Plugin { +class Plugin final : public plugin::Plugin { public: plugin::Configuration Configure() override { AddComponent(new storage::BackendComponent("REDIS", backend::redis::Redis::Instantiate)); diff --git a/src/storage/backend/redis/Redis.h b/src/storage/backend/redis/Redis.h index 2654ef6c9c..336bcb69de 100644 --- a/src/storage/backend/redis/Redis.h +++ b/src/storage/backend/redis/Redis.h @@ -13,7 +13,7 @@ struct redisReply; struct redisPollEvents; namespace zeek::storage::backend::redis { -class Redis : public Backend, public iosource::IOSource { +class Redis final : public Backend, public iosource::IOSource { public: Redis() : Backend(SupportedModes::ASYNC, "REDIS"), IOSource(true) {} ~Redis() override = default; diff --git a/src/storage/backend/sqlite/Plugin.cc b/src/storage/backend/sqlite/Plugin.cc index 8498e747d8..c6ba89df94 100644 --- a/src/storage/backend/sqlite/Plugin.cc +++ b/src/storage/backend/sqlite/Plugin.cc @@ -7,7 +7,7 @@ namespace zeek::storage::backend::sqlite { -class Plugin : public plugin::Plugin { +class Plugin final : public plugin::Plugin { public: plugin::Configuration Configure() override { AddComponent(new storage::BackendComponent("SQLITE", backend::sqlite::SQLite::Instantiate)); diff --git a/src/storage/backend/sqlite/SQLite.h b/src/storage/backend/sqlite/SQLite.h index 5a35651fde..6f011611b4 100644 --- a/src/storage/backend/sqlite/SQLite.h +++ b/src/storage/backend/sqlite/SQLite.h @@ -10,7 +10,7 @@ struct sqlite3_stmt; namespace zeek::storage::backend::sqlite { -class SQLite : public Backend { +class SQLite final : public Backend { public: SQLite() : Backend(SupportedModes::SYNC, "SQLITE") {} ~SQLite() override = default; From 40b75cb8090353849d2edc33926544a345f1fc4f Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 4 Apr 2025 13:05:45 -0700 Subject: [PATCH 5/9] Remove unnecessary and includes from util.h --- src/util.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/util.h b/src/util.h index d950c9df17..a59d2d1962 100644 --- a/src/util.h +++ b/src/util.h @@ -16,14 +16,12 @@ #include #include -#include #include #include #include #include #include #include -#include // std::unique_ptr #include #include #include From 1169fcf2a2c8966dbdb0c47c5f5b69f4e6bcc31b Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 4 Apr 2025 13:22:53 -0700 Subject: [PATCH 6/9] Move byte_buffer types from cluster and storage into util --- src/broker/Manager.h | 5 ++-- src/cluster/Backend.cc | 11 ++++----- src/cluster/Backend.h | 23 +++++++++---------- src/cluster/Serializer.h | 14 ++++------- src/cluster/backend/zeromq/ZeroMQ.cc | 13 +++++------ src/cluster/backend/zeromq/ZeroMQ.h | 7 +++--- .../binary-serialization-format/Serializer.cc | 6 ++--- .../binary-serialization-format/Serializer.h | 4 ++-- src/cluster/serializer/broker/Serializer.cc | 10 ++++---- src/cluster/serializer/broker/Serializer.h | 8 +++---- src/storage/Serializer.h | 10 ++------ src/storage/serializer/json/JSON.cc | 6 ++--- src/storage/serializer/json/JSON.h | 4 ++-- src/util.h | 6 +++++ .../plugins/storage-plugin/src/StorageDummy.h | 2 +- 15 files changed, 59 insertions(+), 70 deletions(-) diff --git a/src/broker/Manager.h b/src/broker/Manager.h index c9e7f3c8e9..d2dfbf2261 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -400,8 +400,7 @@ private: // This should never be reached, broker itself doesn't call this and overrides // the generic DoPublishEvent() method that would call this. - bool DoPublishEvent(const std::string& topic, const std::string& format, - const cluster::detail::byte_buffer& buf) override { + bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) override { throw std::logic_error("not implemented"); } @@ -416,7 +415,7 @@ private: } bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, - cluster::detail::byte_buffer& buf) override { + byte_buffer& buf) override { // Not implemented by broker. throw std::logic_error("not implemented"); } diff --git a/src/cluster/Backend.cc b/src/cluster/Backend.cc index 8fba80b48b..53b70047cc 100644 --- a/src/cluster/Backend.cc +++ b/src/cluster/Backend.cc @@ -100,7 +100,7 @@ std::optional Backend::MakeClusterEvent(FuncValPtr handler, ArgsS // Default implementation doing the serialization. bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& event) { - cluster::detail::byte_buffer buf; + byte_buffer buf; if ( ! event_serializer->SerializeEvent(buf, event) ) return false; @@ -111,7 +111,7 @@ bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& e // Default implementation doing log record serialization. bool Backend::DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, zeek::Span records) { - cluster::detail::byte_buffer buf; + byte_buffer buf; if ( ! log_serializer->SerializeLogWrite(buf, header, records) ) return false; @@ -123,8 +123,7 @@ void Backend::EnqueueEvent(EventHandlerPtr h, zeek::Args args) { event_handling_strategy->EnqueueLocalEvent(h, std::move(args)); } -bool Backend::ProcessEventMessage(std::string_view topic, std::string_view format, - const detail::byte_buffer_span payload) { +bool Backend::ProcessEventMessage(std::string_view topic, std::string_view format, const byte_buffer_span payload) { if ( format != event_serializer->Name() ) { zeek::reporter->Error("ProcessEventMessage: Wrong format: %s vs %s", std::string{format}.c_str(), event_serializer->Name().c_str()); @@ -143,7 +142,7 @@ bool Backend::ProcessEventMessage(std::string_view topic, std::string_view forma return event_handling_strategy->HandleRemoteEvent(topic, std::move(*r)); } -bool Backend::ProcessLogMessage(std::string_view format, detail::byte_buffer_span payload) { +bool Backend::ProcessLogMessage(std::string_view format, byte_buffer_span payload) { // We could also dynamically lookup the right de-serializer, but // for now assume we just receive what is configured. if ( format != log_serializer->Name() ) { @@ -162,7 +161,7 @@ bool Backend::ProcessLogMessage(std::string_view format, detail::byte_buffer_spa return zeek::log_mgr->WriteBatchFromRemote(result->header, std::move(result->records)); } -bool ThreadedBackend::ProcessBackendMessage(int tag, detail::byte_buffer_span payload) { +bool ThreadedBackend::ProcessBackendMessage(int tag, byte_buffer_span payload) { return DoProcessBackendMessage(tag, payload); } diff --git a/src/cluster/Backend.h b/src/cluster/Backend.h index 5f0495ac18..03f6de3e49 100644 --- a/src/cluster/Backend.h +++ b/src/cluster/Backend.h @@ -311,12 +311,12 @@ protected: /** * Process an incoming event message. */ - bool ProcessEventMessage(std::string_view topic, std::string_view format, detail::byte_buffer_span payload); + bool ProcessEventMessage(std::string_view topic, std::string_view format, byte_buffer_span payload); /** * Process an incoming log message. */ - bool ProcessLogMessage(std::string_view format, detail::byte_buffer_span payload); + bool ProcessLogMessage(std::string_view format, byte_buffer_span payload); private: /** @@ -350,7 +350,7 @@ private: /** * Publish a cluster::detail::Event to the given topic. * - * The default implementation serializes to a detail::byte_buffer and + * The default implementation serializes to a byte_buffer and * calls DoPublishEvent() with the resulting buffer. * * This hook method only exists for the existing Broker implementation that @@ -373,8 +373,7 @@ private: * @param buf the serialized Event. * @return true if the message has been published successfully. */ - virtual bool DoPublishEvent(const std::string& topic, const std::string& format, - const detail::byte_buffer& buf) = 0; + virtual bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) = 0; /** * Register interest in messages that use a certain topic prefix. @@ -405,7 +404,7 @@ private: /** * Serialize a log batch, then forward it to DoPublishLogWrites() below. - * The default implementation serializes to a detail::byte_buffer and + * The default implementation serializes to a byte_buffer and * calls DoPublishLogWrites() with the resulting buffer. * * This hook method only exists for the existing Broker implementation that @@ -440,7 +439,7 @@ private: * @return true if the message has been published successfully. */ virtual bool DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, const std::string& format, - detail::byte_buffer& buf) = 0; + byte_buffer& buf) = 0; std::string name; zeek::Tag tag; @@ -471,7 +470,7 @@ private: struct EventMessage { std::string topic; std::string format; - detail::byte_buffer payload; + byte_buffer payload; auto payload_span() const { return Span(payload.data(), payload.size()); }; }; @@ -481,7 +480,7 @@ struct EventMessage { */ struct LogMessage { std::string format; - detail::byte_buffer payload; + byte_buffer payload; auto payload_span() const { return Span(payload.data(), payload.size()); }; }; @@ -494,7 +493,7 @@ struct LogMessage { */ struct BackendMessage { int tag; - detail::byte_buffer payload; + byte_buffer payload; auto payload_span() const { return Span(payload.data(), payload.size()); }; }; @@ -546,13 +545,13 @@ private: /** * Process a backend specific message queued as BackendMessage. */ - bool ProcessBackendMessage(int tag, detail::byte_buffer_span payload); + bool ProcessBackendMessage(int tag, byte_buffer_span payload); /** * If a cluster backend produces messages of type BackendMessage, * this method will be invoked by the main thread to process it. */ - virtual bool DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) { return false; }; + virtual bool DoProcessBackendMessage(int tag, byte_buffer_span payload) { return false; }; /** * Hook method for OnLooProcess. diff --git a/src/cluster/Serializer.h b/src/cluster/Serializer.h index 2be72ceca2..cc2402641b 100644 --- a/src/cluster/Serializer.h +++ b/src/cluster/Serializer.h @@ -6,7 +6,6 @@ #include #include -#include #include "zeek/Span.h" #include "zeek/logging/Types.h" @@ -15,13 +14,8 @@ namespace zeek::cluster { namespace detail { class Event; - -using byte_buffer = std::vector; -using byte_buffer_span = Span; - } // namespace detail - /** * This class handles encoding of events into byte buffers and back. * @@ -40,7 +34,7 @@ public: * * @returns True on success, false in exceptional cases (e.g. unsupported serialization). */ - virtual bool SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) = 0; + virtual bool SerializeEvent(byte_buffer& buf, const detail::Event& event) = 0; /** * Unserialize an event from a given byte buffer. @@ -49,7 +43,7 @@ public: * * @returns The event, or std::nullopt on error. */ - virtual std::optional UnserializeEvent(detail::byte_buffer_span buf) = 0; + virtual std::optional UnserializeEvent(byte_buffer_span buf) = 0; /** * @returns The name of this event serializer instance. @@ -85,7 +79,7 @@ public: * @param header The log batch header. * @param records The actual log writes. */ - virtual bool SerializeLogWrite(detail::byte_buffer& buf, const logging::detail::LogWriteHeader& header, + virtual bool SerializeLogWrite(byte_buffer& buf, const logging::detail::LogWriteHeader& header, zeek::Span records) = 0; /** @@ -93,7 +87,7 @@ public: * * @param buf The span representing received log writes. */ - virtual std::optional UnserializeLogWrite(detail::byte_buffer_span buf) = 0; + virtual std::optional UnserializeLogWrite(byte_buffer_span buf) = 0; /** * @returns The name of this log serializer instance. diff --git a/src/cluster/backend/zeromq/ZeroMQ.cc b/src/cluster/backend/zeromq/ZeroMQ.cc index 43a48aec89..3e587dd963 100644 --- a/src/cluster/backend/zeromq/ZeroMQ.cc +++ b/src/cluster/backend/zeromq/ZeroMQ.cc @@ -253,8 +253,7 @@ bool ZeroMQBackend::SpawnZmqProxyThread() { return proxy_thread->Start(); } -bool ZeroMQBackend::DoPublishEvent(const std::string& topic, const std::string& format, - const cluster::detail::byte_buffer& buf) { +bool ZeroMQBackend::DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) { // Publishing an event happens as a multipart message with 4 parts: // // * The topic to publish to - this is required by XPUB/XSUB @@ -336,7 +335,7 @@ bool ZeroMQBackend::DoUnsubscribe(const std::string& topic_prefix) { } bool ZeroMQBackend::DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, - cluster::detail::byte_buffer& buf) { + byte_buffer& buf) { ZEROMQ_DEBUG("Publishing %zu bytes of log writes (path %s)", buf.size(), header.path.c_str()); static std::string message_type = "log-write"; @@ -405,7 +404,7 @@ void ZeroMQBackend::Run() { continue; } - detail::byte_buffer payload{msg[3].data(), msg[3].data() + msg[3].size()}; + byte_buffer payload{msg[3].data(), msg[3].data() + msg[3].size()}; LogMessage lm{.format = std::string(msg[2].data(), msg[2].size()), .payload = std::move(payload)}; @@ -487,7 +486,7 @@ void ZeroMQBackend::Run() { QueueMessage qm; auto* start = msg[0].data() + 1; auto* end = msg[0].data() + msg[0].size(); - detail::byte_buffer topic(start, end); + byte_buffer topic(start, end); if ( first == 1 ) { qm = BackendMessage{1, std::move(topic)}; } @@ -516,7 +515,7 @@ void ZeroMQBackend::Run() { if ( sender == NodeId() ) continue; - detail::byte_buffer payload{msg[3].data(), msg[3].data() + msg[3].size()}; + byte_buffer payload{msg[3].data(), msg[3].data() + msg[3].size()}; EventMessage em{.topic = std::string(msg[0].data(), msg[0].size()), .format = std::string(msg[2].data(), msg[2].size()), .payload = std::move(payload)}; @@ -644,7 +643,7 @@ void ZeroMQBackend::Run() { } } -bool ZeroMQBackend::DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) { +bool ZeroMQBackend::DoProcessBackendMessage(int tag, byte_buffer_span payload) { if ( tag == 0 || tag == 1 ) { std::string topic{reinterpret_cast(payload.data()), payload.size()}; zeek::EventHandlerPtr eh; diff --git a/src/cluster/backend/zeromq/ZeroMQ.h b/src/cluster/backend/zeromq/ZeroMQ.h index fed3585394..088f922f34 100644 --- a/src/cluster/backend/zeromq/ZeroMQ.h +++ b/src/cluster/backend/zeromq/ZeroMQ.h @@ -60,17 +60,16 @@ private: void DoTerminate() override; - bool DoPublishEvent(const std::string& topic, const std::string& format, - const cluster::detail::byte_buffer& buf) override; + bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) override; bool DoSubscribe(const std::string& topic_prefix, SubscribeCallback cb) override; bool DoUnsubscribe(const std::string& topic_prefix) override; bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, - cluster::detail::byte_buffer& buf) override; + byte_buffer& buf) override; - bool DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) override; + bool DoProcessBackendMessage(int tag, byte_buffer_span payload) override; // Script level variables. std::string connect_xsub_endpoint; diff --git a/src/cluster/serializer/binary-serialization-format/Serializer.cc b/src/cluster/serializer/binary-serialization-format/Serializer.cc index 1bb2e49710..b26087e34c 100644 --- a/src/cluster/serializer/binary-serialization-format/Serializer.cc +++ b/src/cluster/serializer/binary-serialization-format/Serializer.cc @@ -77,7 +77,7 @@ bool detail::BinarySerializationFormatLogSerializer::SerializeLogWrite(byte_buff } std::optional detail::BinarySerializationFormatLogSerializer::UnserializeLogWrite( - detail::byte_buffer_span buf) { + byte_buffer_span buf) { zeek::detail::BinarySerializationFormat fmt; fmt.StartRead(reinterpret_cast(buf.data()), buf.size()); @@ -145,7 +145,7 @@ std::optional detail::BinarySerializationF TEST_SUITE_BEGIN("cluster serializer binary-serialization-format"); TEST_CASE("roundtrip") { - detail::byte_buffer buf; + zeek::byte_buffer buf; detail::BinarySerializationFormatLogSerializer serializer; static const auto& stream_id_type = zeek::id::find_type("Log::ID"); @@ -161,7 +161,7 @@ TEST_CASE("roundtrip") { 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x16, 0x01, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; std::byte* p = reinterpret_cast(&expected_bytes[0]); - detail::byte_buffer expected{p, p + sizeof(expected_bytes)}; + zeek::byte_buffer expected{p, p + sizeof(expected_bytes)}; auto s = stream_id_type->Lookup("Log::UNKNOWN"); REQUIRE_GE(s, 0); diff --git a/src/cluster/serializer/binary-serialization-format/Serializer.h b/src/cluster/serializer/binary-serialization-format/Serializer.h index 2fa6bbc4c6..79640635e4 100644 --- a/src/cluster/serializer/binary-serialization-format/Serializer.h +++ b/src/cluster/serializer/binary-serialization-format/Serializer.h @@ -13,10 +13,10 @@ class BinarySerializationFormatLogSerializer : public cluster::LogSerializer { public: BinarySerializationFormatLogSerializer() : LogSerializer("zeek-bin-serializer") {} - bool SerializeLogWrite(cluster::detail::byte_buffer& buf, const logging::detail::LogWriteHeader& header, + bool SerializeLogWrite(byte_buffer& buf, const logging::detail::LogWriteHeader& header, zeek::Span records) override; - std::optional UnserializeLogWrite(detail::byte_buffer_span buf) override; + std::optional UnserializeLogWrite(byte_buffer_span buf) override; }; } // namespace zeek::cluster::detail diff --git a/src/cluster/serializer/broker/Serializer.cc b/src/cluster/serializer/broker/Serializer.cc index 5bd32f8738..d47e61018d 100644 --- a/src/cluster/serializer/broker/Serializer.cc +++ b/src/cluster/serializer/broker/Serializer.cc @@ -101,7 +101,7 @@ std::optional detail::to_zeek_event(const broker::zeek::Event& ev return detail::Event{handler, std::move(vl), ts}; } -bool detail::BrokerBinV1_Serializer::SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) { +bool detail::BrokerBinV1_Serializer::SerializeEvent(byte_buffer& buf, const detail::Event& event) { auto ev = to_broker_event(event); if ( ! ev ) return false; @@ -117,7 +117,7 @@ bool detail::BrokerBinV1_Serializer::SerializeEvent(detail::byte_buffer& buf, co return true; } -std::optional detail::BrokerBinV1_Serializer::UnserializeEvent(detail::byte_buffer_span buf) { +std::optional detail::BrokerBinV1_Serializer::UnserializeEvent(byte_buffer_span buf) { auto r = broker::data_envelope::deserialize(broker::endpoint_id::nil(), broker::endpoint_id::nil(), 0, "", buf.data(), buf.size()); if ( ! r ) @@ -152,7 +152,7 @@ bool detail::BrokerJsonV1_Serializer::SerializeEvent(byte_buffer& buf, const det return true; } -std::optional detail::BrokerJsonV1_Serializer::UnserializeEvent(detail::byte_buffer_span buf) { +std::optional detail::BrokerJsonV1_Serializer::UnserializeEvent(byte_buffer_span buf) { broker::variant res; auto err = broker::format::json::v1::decode(std::string_view{reinterpret_cast(buf.data()), buf.size()}, res); @@ -173,7 +173,7 @@ TEST_SUITE_BEGIN("cluster serializer broker"); TEST_CASE("roundtrip") { auto* handler = zeek::event_registry->Lookup("Supervisor::node_status"); detail::Event e{handler, zeek::Args{zeek::make_intrusive("TEST"), zeek::val_mgr->Count(42)}}; - detail::byte_buffer buf; + zeek::byte_buffer buf; SUBCASE("json") { detail::BrokerJsonV1_Serializer serializer; @@ -201,7 +201,7 @@ TEST_CASE("roundtrip") { 0x01, 0x0e, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; std::byte* p = reinterpret_cast(&expected_bytes[0]); - detail::byte_buffer expected{p, p + sizeof(expected_bytes)}; + zeek::byte_buffer expected{p, p + sizeof(expected_bytes)}; serializer.SerializeEvent(buf, e); diff --git a/src/cluster/serializer/broker/Serializer.h b/src/cluster/serializer/broker/Serializer.h index 4b9adae241..60e2691dbf 100644 --- a/src/cluster/serializer/broker/Serializer.h +++ b/src/cluster/serializer/broker/Serializer.h @@ -34,9 +34,9 @@ class BrokerBinV1_Serializer : public EventSerializer { public: BrokerBinV1_Serializer() : EventSerializer("broker-bin-v1") {} - bool SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) override; + bool SerializeEvent(byte_buffer& buf, const detail::Event& event) override; - std::optional UnserializeEvent(detail::byte_buffer_span buf) override; + std::optional UnserializeEvent(byte_buffer_span buf) override; }; // Implementation of the EventSerializer that uses the existing broker::detail::val_to_data() @@ -45,9 +45,9 @@ class BrokerJsonV1_Serializer : public EventSerializer { public: BrokerJsonV1_Serializer() : EventSerializer("broker-json-v1") {} - bool SerializeEvent(zeek::cluster::detail::byte_buffer& buf, const detail::Event& event) override; + bool SerializeEvent(byte_buffer& buf, const detail::Event& event) override; - std::optional UnserializeEvent(detail::byte_buffer_span buf) override; + std::optional UnserializeEvent(byte_buffer_span buf) override; }; } // namespace zeek::cluster::detail diff --git a/src/storage/Serializer.h b/src/storage/Serializer.h index dea136fcb2..8b5643bf1e 100644 --- a/src/storage/Serializer.h +++ b/src/storage/Serializer.h @@ -2,16 +2,10 @@ #pragma once -#include "zeek/Span.h" #include "zeek/Val.h" namespace zeek::storage { -namespace detail { -using byte_buffer = std::vector; -using byte_buffer_span = Span; -} // namespace detail - /** * Base class for a serializer used by storage backends. */ @@ -27,7 +21,7 @@ public: * @return On success, a byte buffer containing the serialized data. std::nullopt will * be returned on failure. */ - virtual std::optional Serialize(ValPtr val) = 0; + virtual std::optional Serialize(ValPtr val) = 0; /** * Unserializes a byte buffer into Zeek Val objects of a specific type. @@ -38,7 +32,7 @@ public: * @return A zeek::expected containing either the unserialized Val data on success, or * a string containing an error message on failure. */ - virtual zeek::expected Unserialize(detail::byte_buffer_span buf, TypePtr type) = 0; + virtual zeek::expected Unserialize(byte_buffer_span buf, TypePtr type) = 0; protected: Serializer(std::string name) : name(std::move(name)) {} diff --git a/src/storage/serializer/json/JSON.cc b/src/storage/serializer/json/JSON.cc index 8625eaa0be..a639112524 100644 --- a/src/storage/serializer/json/JSON.cc +++ b/src/storage/serializer/json/JSON.cc @@ -10,8 +10,8 @@ std::unique_ptr JSON::Instantiate() { return std::make_unique( JSON::JSON() : Serializer("JSON") {} -std::optional JSON::Serialize(ValPtr val) { - detail::byte_buffer buf; +std::optional JSON::Serialize(ValPtr val) { + byte_buffer buf; auto json = val->ToJSON(); buf.reserve(json->Len()); @@ -21,7 +21,7 @@ std::optional JSON::Serialize(ValPtr val) { return buf; } -zeek::expected JSON::Unserialize(detail::byte_buffer_span buf, TypePtr type) { +zeek::expected JSON::Unserialize(byte_buffer_span buf, TypePtr type) { std::string_view text{reinterpret_cast(buf.data()), buf.size()}; return zeek::detail::ValFromJSON(text, type, Func::nil); } diff --git a/src/storage/serializer/json/JSON.h b/src/storage/serializer/json/JSON.h index 2b5475936a..c9789d6b9f 100644 --- a/src/storage/serializer/json/JSON.h +++ b/src/storage/serializer/json/JSON.h @@ -13,8 +13,8 @@ public: JSON(); ~JSON() override = default; - std::optional Serialize(ValPtr val) override; - zeek::expected Unserialize(detail::byte_buffer_span buf, TypePtr type) override; + std::optional Serialize(ValPtr val) override; + zeek::expected Unserialize(byte_buffer_span buf, TypePtr type) override; }; } // namespace zeek::storage::serializer::json diff --git a/src/util.h b/src/util.h index a59d2d1962..b5cff50e6e 100644 --- a/src/util.h +++ b/src/util.h @@ -104,6 +104,8 @@ template using unexpected = nonstd::unexpected; } // namespace zeek +#include "zeek/Span.h" + using zeek_int_t = int64_t; using zeek_uint_t = uint64_t; @@ -119,6 +121,10 @@ namespace zeek { class ODesc; class RecordVal; +// Byte buffer types used by serialization code in storage and cluster. +using byte_buffer = std::vector; +using byte_buffer_span = Span; + namespace util { namespace detail { diff --git a/testing/btest/plugins/storage-plugin/src/StorageDummy.h b/testing/btest/plugins/storage-plugin/src/StorageDummy.h index c295aee82e..3e1e5d2a9d 100644 --- a/testing/btest/plugins/storage-plugin/src/StorageDummy.h +++ b/testing/btest/plugins/storage-plugin/src/StorageDummy.h @@ -50,7 +50,7 @@ public: zeek::storage::OperationResult DoErase(zeek::storage::ResultCallback* cb, zeek::ValPtr key) override; private: - std::map data; + std::map data; bool open = false; }; From dbb3144e2dd659593f44cedfb7f6bd63f84a6e7f Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 7 Apr 2025 13:20:40 -0700 Subject: [PATCH 7/9] Remove unnecessary includes in Val.h --- src/Val.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Val.h b/src/Val.h index 850b89dc81..e5055beec2 100644 --- a/src/Val.h +++ b/src/Val.h @@ -4,9 +4,7 @@ #include // for u_char #include -#include #include -#include #include #include "zeek/IntrusivePtr.h" From 9593db1974161ab4a9cafbe15756f020f0c2b82c Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 7 Apr 2025 16:11:15 -0700 Subject: [PATCH 8/9] Add versioning to JSON serializer --- src/storage/Serializer.h | 2 -- src/storage/serializer/json/JSON.cc | 23 +++++++++++++++++++---- src/storage/serializer/json/JSON.h | 3 +++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/storage/Serializer.h b/src/storage/Serializer.h index 8b5643bf1e..939b06b0be 100644 --- a/src/storage/Serializer.h +++ b/src/storage/Serializer.h @@ -36,8 +36,6 @@ public: protected: Serializer(std::string name) : name(std::move(name)) {} - -private: std::string name; }; diff --git a/src/storage/serializer/json/JSON.cc b/src/storage/serializer/json/JSON.cc index a639112524..ecc4296e37 100644 --- a/src/storage/serializer/json/JSON.cc +++ b/src/storage/serializer/json/JSON.cc @@ -6,24 +6,39 @@ namespace zeek::storage::serializer::json { +std::string JSON::versioned_name = "JSONv1"; + std::unique_ptr JSON::Instantiate() { return std::make_unique(); } JSON::JSON() : Serializer("JSON") {} std::optional JSON::Serialize(ValPtr val) { + static auto byte_converter = [](u_char c) { return std::byte(c); }; + byte_buffer buf; auto json = val->ToJSON(); - buf.reserve(json->Len()); + buf.reserve(json->Len() + versioned_name.size() + 1); - std::transform(json->Bytes(), json->Bytes() + json->Len(), std::back_inserter(buf), - [](u_char c) { return std::byte(c); }); + std::transform(versioned_name.begin(), versioned_name.end(), std::back_inserter(buf), byte_converter); + buf.push_back(static_cast(';')); + std::transform(json->Bytes(), json->Bytes() + json->Len(), std::back_inserter(buf), byte_converter); return buf; } zeek::expected JSON::Unserialize(byte_buffer_span buf, TypePtr type) { std::string_view text{reinterpret_cast(buf.data()), buf.size()}; - return zeek::detail::ValFromJSON(text, type, Func::nil); + + auto semicolon = text.find(';'); + if ( semicolon == std::string::npos ) + return zeek::unexpected("Version string missing"); + + std::string_view version = std::string_view(text).substr(0, semicolon); + if ( version != versioned_name ) + return zeek::unexpected( + util::fmt("Version doesn't match: %s vs %s", version.data(), versioned_name.c_str())); + + return zeek::detail::ValFromJSON(text.substr(semicolon + 1), type, Func::nil); } } // namespace zeek::storage::serializer::json diff --git a/src/storage/serializer/json/JSON.h b/src/storage/serializer/json/JSON.h index c9789d6b9f..d31d25acd5 100644 --- a/src/storage/serializer/json/JSON.h +++ b/src/storage/serializer/json/JSON.h @@ -15,6 +15,9 @@ public: std::optional Serialize(ValPtr val) override; zeek::expected Unserialize(byte_buffer_span buf, TypePtr type) override; + +private: + static std::string versioned_name; }; } // namespace zeek::storage::serializer::json From cb1ef47a3192327c3bee5c754def361486e02e23 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 11 Apr 2025 13:53:03 -0700 Subject: [PATCH 9/9] Add STORAGE_ prefixes for backends and serializers --- scripts/base/frameworks/storage/main.zeek | 2 +- src/storage/Component.cc | 4 ++-- testing/btest/Baseline/plugins.storage/output | 2 +- .../scripts.base.frameworks.storage.redis-disconnect/out | 4 ++-- .../Baseline/scripts.base.frameworks.storage.redis-sync/out | 4 ++-- .../scripts.base.frameworks.storage.sqlite-basic/out | 2 +- .../out | 2 +- testing/btest/plugins/storage.zeek | 6 +++--- .../scripts/base/frameworks/storage/compound-types.zeek | 2 +- testing/btest/scripts/base/frameworks/storage/erase.zeek | 2 +- .../btest/scripts/base/frameworks/storage/expiration.zeek | 2 +- .../btest/scripts/base/frameworks/storage/overwriting.zeek | 2 +- .../base/frameworks/storage/redis-async-reading-pcap.zeek | 2 +- .../btest/scripts/base/frameworks/storage/redis-async.zeek | 2 +- .../scripts/base/frameworks/storage/redis-cluster.zeek | 2 +- .../scripts/base/frameworks/storage/redis-disconnect.zeek | 4 ++-- .../btest/scripts/base/frameworks/storage/redis-erase.zeek | 2 +- .../scripts/base/frameworks/storage/redis-expiration.zeek | 2 +- .../base/frameworks/storage/redis-native-expiration.zeek | 2 +- .../btest/scripts/base/frameworks/storage/redis-sync.zeek | 4 ++-- .../base/frameworks/storage/sqlite-basic-reading-pcap.zeek | 2 +- .../base/frameworks/storage/sqlite-basic-sync-in-when.zeek | 2 +- .../btest/scripts/base/frameworks/storage/sqlite-basic.zeek | 4 ++-- .../base/frameworks/storage/sqlite-error-handling.zeek | 4 ++-- 24 files changed, 33 insertions(+), 33 deletions(-) diff --git a/scripts/base/frameworks/storage/main.zeek b/scripts/base/frameworks/storage/main.zeek index a1793dac37..75d72fe706 100644 --- a/scripts/base/frameworks/storage/main.zeek +++ b/scripts/base/frameworks/storage/main.zeek @@ -9,7 +9,7 @@ export { ## to add relevant fields to it. type BackendOptions: record { ## The serializer used for converting Zeek data. - serializer: Storage::Serializer &default=Storage::JSON; + serializer: Storage::Serializer &default=Storage::STORAGE_SERIALIZER_JSON; }; ## Record for passing arguments to :zeek:see:`Storage::Async::put` and diff --git a/src/storage/Component.cc b/src/storage/Component.cc index cc0ee4cc4c..1847843c62 100644 --- a/src/storage/Component.cc +++ b/src/storage/Component.cc @@ -14,7 +14,7 @@ BackendComponent::BackendComponent(const std::string& name, factory_callback arg void BackendComponent::Initialize() { InitializeTag(); - storage_mgr->BackendMgr().RegisterComponent(this); + storage_mgr->BackendMgr().RegisterComponent(this, "STORAGE_BACKEND_"); } void BackendComponent::DoDescribe(ODesc* d) const { @@ -29,7 +29,7 @@ SerializerComponent::SerializerComponent(const std::string& name, factory_callba void SerializerComponent::Initialize() { InitializeTag(); - storage_mgr->SerializerMgr().RegisterComponent(this); + storage_mgr->SerializerMgr().RegisterComponent(this, "STORAGE_SERIALIZER_"); } void SerializerComponent::DoDescribe(ODesc* d) const { diff --git a/testing/btest/Baseline/plugins.storage/output b/testing/btest/Baseline/plugins.storage/output index 89c4f9ab07..fbd1a0ecda 100644 --- a/testing/btest/Baseline/plugins.storage/output +++ b/testing/btest/Baseline/plugins.storage/output @@ -10,7 +10,7 @@ get result after erase, [code=Storage::KEY_NOT_FOUND, error_str=, close result, [code=Storage::SUCCESS, error_str=, value=] results of trying to use closed handle: get: Storage::NOT_CONNECTED, put: Storage::NOT_CONNECTED, erase: Storage::NOT_CONNECTED -open result 2, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error, value=] +open result 2, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::STORAGE_BACKEND_STORAGEDUMMY: open_fail was set to true, returning error, value=] close result on closed handle, [code=Storage::NOT_CONNECTED, error_str=Backend is closed, value=] open result 3, [code=Storage::SUCCESS, error_str=, value=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out index d87e524a8b..a62108de4f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. open_result, [code=Storage::SUCCESS, error_str=, value=] -Storage::backend_opened, Storage::REDIS, [serializer=Storage::JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]] -Storage::backend_lost, Storage::REDIS, [serializer=Storage::JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]], Server closed the connection +Storage::backend_opened, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]] +Storage::backend_lost, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]], Server closed the connection diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out index 52cc495f62..5bd50b0b4e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out @@ -9,5 +9,5 @@ get result same as originally inserted, T put result, [code=Storage::SUCCESS, error_str=, value=] get result, [code=Storage::SUCCESS, error_str=, value=value2345] get result same as overwritten, T -Storage::backend_opened, Storage::REDIS, [serializer=Storage::JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]] -Storage::backend_lost, Storage::REDIS, [serializer=Storage::JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]], Client disconnected +Storage::backend_opened, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]] +Storage::backend_lost, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=, key_prefix=testing]], Client disconnected diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out index 8ad2a12b65..7121683d22 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out @@ -1,5 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -Storage::backend_opened, Storage::SQLITE, [serializer=Storage::JSON, sqlite=[database_path=test.sqlite, table_name=testing, tuning_params={ +Storage::backend_opened, Storage::STORAGE_BACKEND_SQLITE, [serializer=Storage::STORAGE_SERIALIZER_JSON, sqlite=[database_path=test.sqlite, table_name=testing, tuning_params={ [synchronous] = normal, [temp_store] = memory, [journal_mode] = WAL diff --git a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out index 0f0dfe41cb..b295efbe65 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out +++ b/testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out @@ -1,5 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -Open result, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file, value=] +Open result, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::STORAGE_BACKEND_SQLITE: SQLite call failed: unable to open database file, value=] Open result 2, [code=Storage::SUCCESS, error_str=, value=] Put result with bad key type, [code=Storage::KEY_TYPE_MISMATCH, error_str=, value=] Put result on closed handle, [code=Storage::NOT_CONNECTED, error_str=Backend is closed, value=] diff --git a/testing/btest/plugins/storage.zeek b/testing/btest/plugins/storage.zeek index 2089497d9f..86a927f8ef 100644 --- a/testing/btest/plugins/storage.zeek +++ b/testing/btest/plugins/storage.zeek @@ -28,7 +28,7 @@ event zeek_init() { local value = "value5678"; # Basic operation. Open, put, and get the value back. - local res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, string, string); + local res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_STORAGEDUMMY, opts, string, string); print "open result", res; local b = res$value; @@ -63,7 +63,7 @@ event zeek_init() { # Test failing to open the handle and test closing an invalid handle. opts$dummy = [$open_fail = T, $timeout_put = F]; - res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, string, string); + res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_STORAGEDUMMY, opts, string, string); print "open result 2", res; res = Storage::Sync::close_backend(res$value); print "close result on closed handle", res; @@ -71,7 +71,7 @@ event zeek_init() { # Test timing out an async put request. opts$dummy = [$open_fail = F, $timeout_put = T]; - res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, string, string); + res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_STORAGEDUMMY, opts, string, string); print "open result 3", res; b = res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/compound-types.zeek b/testing/btest/scripts/base/frameworks/storage/compound-types.zeek index 419a1f3fc0..bb6af79c02 100644 --- a/testing/btest/scripts/base/frameworks/storage/compound-types.zeek +++ b/testing/btest/scripts/base/frameworks/storage/compound-types.zeek @@ -63,7 +63,7 @@ event zeek_init() { value[2] = "b"; value[3] = "c"; - local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, Rec, tbl); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, Rec, tbl); print "open result", open_res; local b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/erase.zeek b/testing/btest/scripts/base/frameworks/storage/erase.zeek index 772b910855..b1b38bd77c 100644 --- a/testing/btest/scripts/base/frameworks/storage/erase.zeek +++ b/testing/btest/scripts/base/frameworks/storage/erase.zeek @@ -17,7 +17,7 @@ event zeek_init() # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. - local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string); print "open result", open_res; local b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/expiration.zeek b/testing/btest/scripts/base/frameworks/storage/expiration.zeek index 857830233d..27b89e1f02 100644 --- a/testing/btest/scripts/base/frameworks/storage/expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/expiration.zeek @@ -33,7 +33,7 @@ event setup_test() local opts : Storage::BackendOptions; opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; - local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string); print "open result", open_res; b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek index 8e801a9d7c..75b0484cf8 100644 --- a/testing/btest/scripts/base/frameworks/storage/overwriting.zeek +++ b/testing/btest/scripts/base/frameworks/storage/overwriting.zeek @@ -18,7 +18,7 @@ event zeek_init() { local value = "value7890"; local value2 = "value2345"; - local res = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); + local res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, str, str); print "open result", res; local b = res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek index 85b20450f5..d401029baf 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek @@ -22,7 +22,7 @@ event zeek_init() local key = "key1234"; local value = "value5678"; - local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string); print "open result", open_res; local b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek index 5a2339fadc..881a5b9ae4 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-async.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-async.zeek @@ -25,7 +25,7 @@ event zeek_init() local value = "value5678"; when [opts, key, value] ( local open_res = Storage::Async::open_backend( - Storage::REDIS, opts, string, string) ) + Storage::STORAGE_BACKEND_REDIS, opts, string, string) ) { print "open result", open_res; local b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek index c2d437400f..6a0efc16d2 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek @@ -40,7 +40,7 @@ event zeek_init() opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( "REDIS_PORT")), $key_prefix="testing" ]; - local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string); backend = open_res$value; } diff --git a/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek b/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek index 55a8f2c320..5c44d27644 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek @@ -26,14 +26,14 @@ event Storage::backend_lost(tag: Storage::Backend, config: any, reason: string) event zeek_init() { local opts: Storage::BackendOptions; - opts$serializer = Storage::JSON; + opts$serializer = Storage::STORAGE_SERIALIZER_JSON; opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( "REDIS_PORT")), $key_prefix="testing" ]; local key = "key1234"; local value = "value1234"; - local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string); print "open_result", open_res; # Kill the redis server so the backend will disconnect and fire the backend_lost event. diff --git a/testing/btest/scripts/base/frameworks/storage/redis-erase.zeek b/testing/btest/scripts/base/frameworks/storage/redis-erase.zeek index bb6f09b477..65a64d6787 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-erase.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-erase.zeek @@ -21,7 +21,7 @@ event zeek_init() local key = "key1234"; local value = "value1234"; - local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string); print "open_result", open_res; local b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek index ed287aabeb..2951838c9c 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek @@ -40,7 +40,7 @@ event setup_test() opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( "REDIS_PORT")), $key_prefix="testing" ]; - local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string); print "open result", open_res; b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-native-expiration.zeek b/testing/btest/scripts/base/frameworks/storage/redis-native-expiration.zeek index 146f89d948..6931d1fe51 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-native-expiration.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-native-expiration.zeek @@ -40,7 +40,7 @@ event setup_test() opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( "REDIS_PORT")), $key_prefix="testing" ]; - local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string); print "open result", open_res; b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek index e19503df5d..2f09ddae44 100644 --- a/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek +++ b/testing/btest/scripts/base/frameworks/storage/redis-sync.zeek @@ -24,7 +24,7 @@ event Storage::backend_lost(tag: Storage::Backend, config: any, reason: string) event zeek_init() { local opts: Storage::BackendOptions; - opts$serializer = Storage::JSON; + opts$serializer = Storage::STORAGE_SERIALIZER_JSON; opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( "REDIS_PORT")), $key_prefix="testing" ]; @@ -32,7 +32,7 @@ event zeek_init() local value = "value1234"; local value2 = "value2345"; - local res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); + local res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string); print "open_result", res; local b = res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek index 764c9ea8cc..38ac20234b 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek @@ -20,7 +20,7 @@ event zeek_init() # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. - local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string); print "open result", open_res; local b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-sync-in-when.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-sync-in-when.zeek index b127c23919..3e4d12e421 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic-sync-in-when.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic-sync-in-when.zeek @@ -24,7 +24,7 @@ event zeek_init() # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. when [opts, key, value] ( local open_res = Storage::Sync::open_backend( - Storage::SQLITE, opts, string, string) ) + Storage::STORAGE_BACKEND_SQLITE, opts, string, string) ) { print "open result", open_res; local b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek index 2e6a815f97..977b31d71b 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek @@ -16,7 +16,7 @@ event zeek_init() { # Create a database file in the .tmp directory with a 'testing' table local opts: Storage::BackendOptions; - opts$serializer = Storage::JSON; + opts$serializer = Storage::STORAGE_SERIALIZER_JSON; opts$sqlite = [ $database_path="test.sqlite", $table_name="testing" ]; local key = "key1234"; @@ -25,7 +25,7 @@ event zeek_init() # Test inserting/retrieving a key/value pair that we know won't be in # the backend yet. when [opts, key, value] ( local open_res = Storage::Async::open_backend( - Storage::SQLITE, opts, string, string) ) + Storage::STORAGE_BACKEND_SQLITE, opts, string, string) ) { print "open result", open_res; local b = open_res$value; diff --git a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek index e85a179cf2..0c216405c1 100644 --- a/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek +++ b/testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek @@ -14,12 +14,12 @@ event zeek_init() { $table_name = "testing"]; # This should report an error in .stderr and reporter.log - local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string); + local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string); print "Open result", open_res; # Open a valid database file opts$sqlite$database_path = "test.sqlite"; - open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string); + open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string); print "Open result 2", open_res; local b = open_res$value;