Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/topic/timw/storage-serialization'

Browse source 
* origin/topic/timw/storage-serialization:
  Add STORAGE_ prefixes for backends and serializers
  Add versioning to JSON serializer
  Remove unnecessary includes in Val.h
  Move byte_buffer types from cluster and storage into util
  Remove unnecessary <array> and <memory> includes from util.h
  Mark storage classes as final where appropriate
  Add JSON storage serializer, use with existing backends/tests
  Make ValFromJSON return zeek::expected instead of a variant
  Ground work for pluggable storage serializers
This commit is contained in:
Tim Wojtulewicz 2025-04-14 10:11:59 -07:00
commit 6ecb8f0f5f
71 changed files with 563 additions and 258 deletions
Download patch file Download diff file Expand all files Collapse all files

20
CHANGES
View file

@ -1,3 +1,23 @@
7.2.0-dev.530 | 2025-04-14 10:11:59 -0700
* Add STORAGE_ prefixes for backends and serializers (Tim Wojtulewicz, Corelight)
* Add versioning to JSON serializer (Tim Wojtulewicz, Corelight)
* Remove unnecessary includes in Val.h (Tim Wojtulewicz, Corelight)
* Move byte_buffer types from cluster and storage into util (Tim Wojtulewicz, Corelight)
* Remove unnecessary <array> and <memory> includes from util.h (Tim Wojtulewicz, Corelight)
* Mark storage classes as final where appropriate (Tim Wojtulewicz, Corelight)
* Add JSON storage serializer, use with existing backends/tests (Tim Wojtulewicz, Corelight)
* Make ValFromJSON return zeek::expected instead of a variant (Tim Wojtulewicz, Corelight)
* Ground work for pluggable storage serializers (Tim Wojtulewicz, Corelight)
7.2.0-dev.520 | 2025-04-14 18:58:55 +0200 7.2.0-dev.520 | 2025-04-14 18:58:55 +0200
* Bump zeekjs to v0.17.0 (Arne Welzel, Corelight) * Bump zeekjs to v0.17.0 (Arne Welzel, Corelight)

2
VERSION
View file

@ -1 +1 @@
7.2.0-dev.520 7.2.0-dev.530

5
scripts/base/frameworks/storage/async.zeek
View file

@ -25,9 +25,8 @@ export {
## Returns: A record containing the status of the operation, and either an error ## Returns: A record containing the status of the operation, and either an error
## string on failure or a value on success. The value returned here will ## string on failure or a value on success. The value returned here will
## be an ``opaque of BackendHandle``. ## be an ``opaque of BackendHandle``.
global open_backend: function(btype: Storage::Backend, global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions,
options: Storage::BackendOptions, key_type: any, val_type: any) key_type: any, val_type: any): Storage::OperationResult;
: Storage::OperationResult;
## Closes an existing backend connection asynchronously. This method must be ## Closes an existing backend connection asynchronously. This method must be
## called via a :zeek:see:`when` condition or an error will be returned. ## called via a :zeek:see:`when` condition or an error will be returned.

17
scripts/base/frameworks/storage/main.zeek
View file

@ -7,23 +7,26 @@ export {
## :zeek:see:`Storage::Async::open_backend` and ## :zeek:see:`Storage::Async::open_backend` and
## :zeek:see:`Storage::Sync::open_backend`. Backend plugins can redef this record ## :zeek:see:`Storage::Sync::open_backend`. Backend plugins can redef this record
## to add relevant fields to it. ## to add relevant fields to it.
type BackendOptions: record { }; type BackendOptions: record {
## The serializer used for converting Zeek data.
serializer: Storage::Serializer &default=Storage::STORAGE_SERIALIZER_JSON;
};
## Record for passing arguments to :zeek:see:`Storage::Async::put` and ## Record for passing arguments to :zeek:see:`Storage::Async::put` and
## :zeek:see:`Storage::Sync::put`. ## :zeek:see:`Storage::Sync::put`.
type PutArgs: record { type PutArgs: record {
# The key to store the value under. ## The key to store the value under.
key: any; key: any;
# The value to store associated with the key. ## The value to store associated with the key.
value: any; value: any;
# Indicates whether this value should overwrite an existing entry for the ## Indicates whether this value should overwrite an existing entry for the
# key. ## key.
overwrite: bool &default=T; overwrite: bool &default=T;
# An interval of time until the entry is automatically removed from the ## An interval of time until the entry is automatically removed from the
# backend. ## backend.
expire_time: interval &default=0sec; expire_time: interval &default=0sec;
}; };
} }

5
scripts/base/frameworks/storage/sync.zeek
View file

@ -23,9 +23,8 @@ export {
## Returns: A record containing the status of the operation, and either an error ## Returns: A record containing the status of the operation, and either an error
## string on failure or a value on success. The value returned here will ## string on failure or a value on success. The value returned here will
## be an ``opaque of BackendHandle``. ## be an ``opaque of BackendHandle``.
global open_backend: function(btype: Storage::Backend, global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions,
options: Storage::BackendOptions, key_type: any, val_type: any) key_type: any, val_type: any): Storage::OperationResult;
: Storage::OperationResult;
## Closes an existing backend connection. ## Closes an existing backend connection.
## ##

90
src/Val.cc
View file

@ -891,8 +891,8 @@ unsigned int StringVal::ComputeFootprint(std::unordered_set<const Val*>* analyze
return 1 /* this object */ + static_cast<unsigned int>(Len()) / sizeof(Val); return 1 /* this object */ + static_cast<unsigned int>(Len()) / sizeof(Val);
} }
static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, const TypePtr& t, static zeek::expected<ValPtr, std::string> BuildVal(const rapidjson::Value& j, const TypePtr& t,
const FuncPtr& key_func) { const FuncPtr& key_func) {
auto mismatch_err = [t, &j]() { auto mismatch_err = [t, &j]() {
std::string json_type; std::string json_type;
switch ( j.GetType() ) { switch ( j.GetType() ) {
@ -906,7 +906,8 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
default: json_type = "unknown"; default: json_type = "unknown";
} }
return util::fmt("cannot convert JSON type '%s' to Zeek type '%s'", json_type.c_str(), type_name(t->Tag())); return zeek::unexpected<std::string>(
util::fmt("cannot convert JSON type '%s' to Zeek type '%s'", json_type.c_str(), type_name(t->Tag())));
}; };
if ( j.IsNull() ) if ( j.IsNull() )
@ -960,7 +961,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
parts.erase(std::remove_if(parts.begin(), parts.end(), [](auto x) { return x.empty(); }), parts.end()); parts.erase(std::remove_if(parts.begin(), parts.end(), [](auto x) { return x.empty(); }), parts.end());
if ( (parts.size() % 2) != 0 ) if ( (parts.size() % 2) != 0 )
return "wrong interval format, must be pairs of values with units"; return zeek::unexpected<std::string>("wrong interval format, must be pairs of values with units");
double interval_secs = 0.0; double interval_secs = 0.0;
for ( size_t i = 0; i < parts.size(); i += 2 ) { for ( size_t i = 0; i < parts.size(); i += 2 ) {
@ -980,7 +981,8 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
else if ( unit == "usec" || unit == "usecs" ) else if ( unit == "usec" || unit == "usecs" )
interval_secs += (value * Microseconds); interval_secs += (value * Microseconds);
else else
return util::fmt("wrong interval format, invalid unit type %s", unit.data()); return zeek::unexpected<std::string>(
util::fmt("wrong interval format, invalid unit type %s", unit.data()));
} }
return make_intrusive<IntervalVal>(interval_secs, Seconds); return make_intrusive<IntervalVal>(interval_secs, Seconds);
@ -991,11 +993,10 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
case TYPE_PORT: { case TYPE_PORT: {
if ( j.IsString() ) { if ( j.IsString() ) {
int port = 0;
if ( j.GetStringLength() > 0 && j.GetStringLength() < 10 ) { if ( j.GetStringLength() > 0 && j.GetStringLength() < 10 ) {
char* slash; char* slash;
errno = 0; errno = 0;
port = strtol(j.GetString(), &slash, 10); auto port = strtol(j.GetString(), &slash, 10);
if ( ! errno ) { if ( ! errno ) {
++slash; ++slash;
if ( util::streq(slash, "tcp") ) if ( util::streq(slash, "tcp") )
@ -1009,15 +1010,17 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
} }
} }
return "wrong port format, string must be /[0-9]{1,5}\\/(tcp|udp|icmp|unknown)/"; return zeek::unexpected<std::string>(
"wrong port format, string must be /[0-9]{1,5}\\/(tcp|udp|icmp|unknown)/");
} }
else if ( j.IsObject() ) { else if ( j.IsObject() ) {
if ( ! j.HasMember("port") || ! j.HasMember("proto") ) if ( ! j.HasMember("port") || ! j.HasMember("proto") )
return "wrong port format, object must have 'port' and 'proto' members"; return zeek::unexpected<std::string>(
"wrong port format, object must have 'port' and 'proto' members");
if ( ! j["port"].IsNumber() ) if ( ! j["port"].IsNumber() )
return "wrong port format, port must be a number"; return zeek::unexpected<std::string>("wrong port format, port must be a number");
if ( ! j["proto"].IsString() ) if ( ! j["proto"].IsString() )
return "wrong port format, protocol must be a string"; return zeek::unexpected<std::string>("wrong port format, protocol must be a string");
std::string proto{j["proto"].GetString()}; std::string proto{j["proto"].GetString()};
@ -1030,10 +1033,10 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
if ( proto == "unknown" ) if ( proto == "unknown" )
return val_mgr->Port(j["port"].GetInt(), TRANSPORT_UNKNOWN); return val_mgr->Port(j["port"].GetInt(), TRANSPORT_UNKNOWN);
return "wrong port format, invalid protocol string"; return zeek::unexpected<std::string>("wrong port format, invalid protocol string");
} }
else else
return "wrong port format, must be string or object"; return zeek::unexpected<std::string>("wrong port format, must be string or object");
} }
case TYPE_PATTERN: { case TYPE_PATTERN: {
@ -1055,7 +1058,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
auto re = std::make_unique<RE_Matcher>(candidate.c_str()); auto re = std::make_unique<RE_Matcher>(candidate.c_str());
if ( ! re->Compile() ) if ( ! re->Compile() )
return "error compiling pattern"; return zeek::unexpected<std::string>("error compiling pattern");
return make_intrusive<PatternVal>(re.release()); return make_intrusive<PatternVal>(re.release());
} }
@ -1074,7 +1077,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
std::string_view subnet_sv(j.GetString(), j.GetStringLength()); std::string_view subnet_sv(j.GetString(), j.GetStringLength());
auto pos = subnet_sv.find('/'); auto pos = subnet_sv.find('/');
if ( pos == subnet_sv.npos ) if ( pos == subnet_sv.npos )
return util::fmt("invalid value for subnet: '%s'", j.GetString()); return zeek::unexpected<std::string>(util::fmt("invalid value for subnet: '%s'", j.GetString()));
candidate = std::string(j.GetString(), pos); candidate = std::string(j.GetString(), pos);
@ -1082,7 +1085,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
char* end; char* end;
width = strtol(subnet_sv.data() + pos + 1, &end, 10); width = strtol(subnet_sv.data() + pos + 1, &end, 10);
if ( subnet_sv.data() + pos + 1 == end || errno ) if ( subnet_sv.data() + pos + 1 == end || errno )
return util::fmt("invalid value for subnet: '%s'", j.GetString()); return zeek::unexpected<std::string>(util::fmt("invalid value for subnet: '%s'", j.GetString()));
} }
if ( candidate.front() == '[' ) if ( candidate.front() == '[' )
@ -1104,7 +1107,8 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
auto intval = et->Lookup({j.GetString(), j.GetStringLength()}); auto intval = et->Lookup({j.GetString(), j.GetStringLength()});
if ( intval < 0 ) if ( intval < 0 )
return util::fmt("'%s' is not a valid enum for '%s'.", j.GetString(), et->GetName().c_str()); return zeek::unexpected<std::string>(
util::fmt("'%s' is not a valid enum for '%s'.", j.GetString(), et->GetName().c_str()));
return et->GetEnumVal(intval); return et->GetEnumVal(intval);
} }
@ -1126,19 +1130,19 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
return mismatch_err(); return mismatch_err();
for ( const auto& item : j.GetArray() ) { for ( const auto& item : j.GetArray() ) {
std::variant<ValPtr, std::string> v; zeek::expected<ValPtr, std::string> v;
if ( tl->GetTypes().size() == 1 ) if ( tl->GetTypes().size() == 1 )
v = BuildVal(item, tl->GetPureType(), key_func); v = BuildVal(item, tl->GetPureType(), key_func);
else else
v = BuildVal(item, tl, key_func); v = BuildVal(item, tl, key_func);
if ( ! get_if<ValPtr>(&v) ) if ( ! v )
return v; return v;
if ( ! std::get<ValPtr>(v) ) if ( v.value() == nullptr )
continue; continue;
tv->Assign(std::move(std::get<ValPtr>(v)), nullptr); tv->Assign(v.value(), nullptr);
} }
return tv; return tv;
@ -1151,7 +1155,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
rapidjson::Document idxstr; rapidjson::Document idxstr;
idxstr.Parse(it->name.GetString(), it->name.GetStringLength()); idxstr.Parse(it->name.GetString(), it->name.GetStringLength());
std::variant<ValPtr, std::string> idx; zeek::expected<ValPtr, std::string> idx;
if ( tl->GetTypes().size() > 1 ) if ( tl->GetTypes().size() > 1 )
idx = BuildVal(idxstr, tl, key_func); idx = BuildVal(idxstr, tl, key_func);
@ -1163,19 +1167,19 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
// Parse the string's content, not the full JSON string. // Parse the string's content, not the full JSON string.
idx = BuildVal(idxstr, tl->GetPureType(), key_func); idx = BuildVal(idxstr, tl->GetPureType(), key_func);
if ( ! get_if<ValPtr>(&idx) ) if ( ! idx )
return idx; return idx;
if ( ! std::get<ValPtr>(idx) ) if ( idx.value() == nullptr )
continue; continue;
auto v = BuildVal(it->value, tt->Yield(), key_func); auto v = BuildVal(it->value, tt->Yield(), key_func);
if ( ! get_if<ValPtr>(&v) ) if ( ! v )
return v; return v;
if ( ! std::get<ValPtr>(v) ) if ( v.value() == nullptr )
continue; continue;
tv->Assign(std::move(std::get<ValPtr>(idx)), std::move(std::get<ValPtr>(v))); tv->Assign(idx.value(), v.value());
} }
return tv; return tv;
@ -1202,7 +1206,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
} }
if ( ! result ) if ( ! result )
return "key function error"; return zeek::unexpected<std::string>("key function error");
normalized_keys[result->AsStringVal()->CheckString()] = &it->value; normalized_keys[result->AsStringVal()->CheckString()] = &it->value;
} }
@ -1226,17 +1230,18 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
if ( ! td_i->GetAttr(detail::ATTR_OPTIONAL) && ! td_i->GetAttr(detail::ATTR_DEFAULT) ) if ( ! td_i->GetAttr(detail::ATTR_OPTIONAL) && ! td_i->GetAttr(detail::ATTR_DEFAULT) )
// jval being set means it is a null JSON value else // jval being set means it is a null JSON value else
// it wasn't even there. // it wasn't even there.
return util::fmt("required field %s$%s is %s in JSON", t->GetName().c_str(), td_i->id, return zeek::unexpected<std::string>(util::fmt("required field %s$%s is %s in JSON",
jval ? "null" : "missing"); t->GetName().c_str(), td_i->id,
jval ? "null" : "missing"));
continue; continue;
} }
auto v = BuildVal(*jval, td_i->type, key_func); auto v = BuildVal(*jval, td_i->type, key_func);
if ( ! get_if<ValPtr>(&v) ) if ( ! v )
return v; return v;
rv->Assign(i, std::move(std::get<ValPtr>(v))); rv->Assign(i, v.value());
} }
return rv; return rv;
@ -1249,16 +1254,16 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
auto lt = t->AsTypeList(); auto lt = t->AsTypeList();
if ( j.GetArray().Size() < lt->GetTypes().size() ) if ( j.GetArray().Size() < lt->GetTypes().size() )
return "index type doesn't match"; return zeek::unexpected<std::string>("index type doesn't match");
auto lv = make_intrusive<ListVal>(TYPE_ANY); auto lv = make_intrusive<ListVal>(TYPE_ANY);
for ( size_t i = 0; i < lt->GetTypes().size(); i++ ) { for ( size_t i = 0; i < lt->GetTypes().size(); i++ ) {
auto v = BuildVal(j.GetArray()[i], lt->GetTypes()[i], key_func); auto v = BuildVal(j.GetArray()[i], lt->GetTypes()[i], key_func);
if ( ! get_if<ValPtr>(&v) ) if ( ! v )
return v; return v;
lv->Append(std::move(std::get<ValPtr>(v))); lv->Append(v.value());
} }
return lv; return lv;
@ -1272,29 +1277,30 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
auto vv = make_intrusive<VectorVal>(IntrusivePtr{NewRef{}, vt}); auto vv = make_intrusive<VectorVal>(IntrusivePtr{NewRef{}, vt});
for ( const auto& item : j.GetArray() ) { for ( const auto& item : j.GetArray() ) {
auto v = BuildVal(item, vt->Yield(), key_func); auto v = BuildVal(item, vt->Yield(), key_func);
if ( ! get_if<ValPtr>(&v) ) if ( ! v )
return v; return v;
if ( ! std::get<ValPtr>(v) ) if ( v.value() == nullptr )
continue; continue;
vv->Assign(vv->Size(), std::move(std::get<ValPtr>(v))); vv->Assign(vv->Size(), v.value());
} }
return vv; return vv;
} }
default: return util::fmt("type '%s' unsupported", type_name(t->Tag())); default: return zeek::unexpected<std::string>(util::fmt("type '%s' unsupported", type_name(t->Tag())));
} }
} }
std::variant<ValPtr, std::string> detail::ValFromJSON(std::string_view json_str, const TypePtr& t, zeek::expected<ValPtr, std::string> detail::ValFromJSON(std::string_view json_str, const TypePtr& t,
const FuncPtr& key_func) { const FuncPtr& key_func) {
rapidjson::Document doc; rapidjson::Document doc;
rapidjson::ParseResult ok = doc.Parse(json_str.data(), json_str.length()); rapidjson::ParseResult ok = doc.Parse(json_str.data(), json_str.length());
if ( ! ok ) if ( ! ok )
return util::fmt("JSON parse error: %s Offset: %lu", rapidjson::GetParseError_En(ok.Code()), ok.Offset()); return zeek::unexpected<std::string>(
util::fmt("JSON parse error: %s Offset: %lu", rapidjson::GetParseError_En(ok.Code()), ok.Offset()));
return BuildVal(doc, t, key_func); return BuildVal(doc, t, key_func);
} }

6
src/Val.h
View file

@ -4,9 +4,7 @@
#include <sys/types.h> // for u_char #include <sys/types.h> // for u_char
#include <array> #include <array>
#include <list>
#include <unordered_map> #include <unordered_map>
#include <variant>
#include <vector> #include <vector>
#include "zeek/IntrusivePtr.h" #include "zeek/IntrusivePtr.h"
@ -1774,8 +1772,8 @@ namespace detail {
// //
// The *key_func* parameter is a Zeek script function called for every JSON key // The *key_func* parameter is a Zeek script function called for every JSON key
// for normalization. If Func::nil is passed, no normalization happens. // for normalization. If Func::nil is passed, no normalization happens.
extern std::variant<ValPtr, std::string> ValFromJSON(std::string_view json_str, const TypePtr& t, extern zeek::expected<ValPtr, std::string> ValFromJSON(std::string_view json_str, const TypePtr& t,
const FuncPtr& key_func); const FuncPtr& key_func);
// If the given vector is an empty vector-of-any ("unspecified"), // If the given vector is an empty vector-of-any ("unspecified"),
// concretizes it to the given type. *v* gives the vector and *t* the // concretizes it to the given type. *v* gives the vector and *t* the

5
src/broker/Manager.h
View file

@ -400,8 +400,7 @@ private:
// This should never be reached, broker itself doesn't call this and overrides // This should never be reached, broker itself doesn't call this and overrides
// the generic DoPublishEvent() method that would call this. // the generic DoPublishEvent() method that would call this.
bool DoPublishEvent(const std::string& topic, const std::string& format, bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) override {
const cluster::detail::byte_buffer& buf) override {
throw std::logic_error("not implemented"); throw std::logic_error("not implemented");
} }
@ -416,7 +415,7 @@ private:
} }
bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format,
cluster::detail::byte_buffer& buf) override { byte_buffer& buf) override {
// Not implemented by broker. // Not implemented by broker.
throw std::logic_error("not implemented"); throw std::logic_error("not implemented");
} }

11
src/cluster/Backend.cc
View file

@ -100,7 +100,7 @@ std::optional<detail::Event> Backend::MakeClusterEvent(FuncValPtr handler, ArgsS
// Default implementation doing the serialization. // Default implementation doing the serialization.
bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& event) { bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& event) {
cluster::detail::byte_buffer buf; byte_buffer buf;
if ( ! event_serializer->SerializeEvent(buf, event) ) if ( ! event_serializer->SerializeEvent(buf, event) )
return false; return false;
@ -111,7 +111,7 @@ bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& e
// Default implementation doing log record serialization. // Default implementation doing log record serialization.
bool Backend::DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, bool Backend::DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header,
zeek::Span<zeek::logging::detail::LogRecord> records) { zeek::Span<zeek::logging::detail::LogRecord> records) {
cluster::detail::byte_buffer buf; byte_buffer buf;
if ( ! log_serializer->SerializeLogWrite(buf, header, records) ) if ( ! log_serializer->SerializeLogWrite(buf, header, records) )
return false; return false;
@ -123,8 +123,7 @@ void Backend::EnqueueEvent(EventHandlerPtr h, zeek::Args args) {
event_handling_strategy->EnqueueLocalEvent(h, std::move(args)); event_handling_strategy->EnqueueLocalEvent(h, std::move(args));
} }
bool Backend::ProcessEventMessage(std::string_view topic, std::string_view format, bool Backend::ProcessEventMessage(std::string_view topic, std::string_view format, const byte_buffer_span payload) {
const detail::byte_buffer_span payload) {
if ( format != event_serializer->Name() ) { if ( format != event_serializer->Name() ) {
zeek::reporter->Error("ProcessEventMessage: Wrong format: %s vs %s", std::string{format}.c_str(), zeek::reporter->Error("ProcessEventMessage: Wrong format: %s vs %s", std::string{format}.c_str(),
event_serializer->Name().c_str()); event_serializer->Name().c_str());
@ -143,7 +142,7 @@ bool Backend::ProcessEventMessage(std::string_view topic, std::string_view forma
return event_handling_strategy->HandleRemoteEvent(topic, std::move(*r)); return event_handling_strategy->HandleRemoteEvent(topic, std::move(*r));
} }
bool Backend::ProcessLogMessage(std::string_view format, detail::byte_buffer_span payload) { bool Backend::ProcessLogMessage(std::string_view format, byte_buffer_span payload) {
// We could also dynamically lookup the right de-serializer, but // We could also dynamically lookup the right de-serializer, but
// for now assume we just receive what is configured. // for now assume we just receive what is configured.
if ( format != log_serializer->Name() ) { if ( format != log_serializer->Name() ) {
@ -162,7 +161,7 @@ bool Backend::ProcessLogMessage(std::string_view format, detail::byte_buffer_spa
return zeek::log_mgr->WriteBatchFromRemote(result->header, std::move(result->records)); return zeek::log_mgr->WriteBatchFromRemote(result->header, std::move(result->records));
} }
bool ThreadedBackend::ProcessBackendMessage(int tag, detail::byte_buffer_span payload) { bool ThreadedBackend::ProcessBackendMessage(int tag, byte_buffer_span payload) {
return DoProcessBackendMessage(tag, payload); return DoProcessBackendMessage(tag, payload);
} }

23
src/cluster/Backend.h
View file

@ -311,12 +311,12 @@ protected:
/** /**
* Process an incoming event message. * Process an incoming event message.
*/ */
bool ProcessEventMessage(std::string_view topic, std::string_view format, detail::byte_buffer_span payload); bool ProcessEventMessage(std::string_view topic, std::string_view format, byte_buffer_span payload);
/** /**
* Process an incoming log message. * Process an incoming log message.
*/ */
bool ProcessLogMessage(std::string_view format, detail::byte_buffer_span payload); bool ProcessLogMessage(std::string_view format, byte_buffer_span payload);
private: private:
/** /**
@ -350,7 +350,7 @@ private:
/** /**
* Publish a cluster::detail::Event to the given topic. * Publish a cluster::detail::Event to the given topic.
* *
* The default implementation serializes to a detail::byte_buffer and * The default implementation serializes to a byte_buffer and
* calls DoPublishEvent() with the resulting buffer. * calls DoPublishEvent() with the resulting buffer.
* *
* This hook method only exists for the existing Broker implementation that * This hook method only exists for the existing Broker implementation that
@ -373,8 +373,7 @@ private:
* @param buf the serialized Event. * @param buf the serialized Event.
* @return true if the message has been published successfully. * @return true if the message has been published successfully.
*/ */
virtual bool DoPublishEvent(const std::string& topic, const std::string& format, virtual bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) = 0;
const detail::byte_buffer& buf) = 0;
/** /**
* Register interest in messages that use a certain topic prefix. * Register interest in messages that use a certain topic prefix.
@ -405,7 +404,7 @@ private:
/** /**
* Serialize a log batch, then forward it to DoPublishLogWrites() below. * Serialize a log batch, then forward it to DoPublishLogWrites() below.
* The default implementation serializes to a detail::byte_buffer and * The default implementation serializes to a byte_buffer and
* calls DoPublishLogWrites() with the resulting buffer. * calls DoPublishLogWrites() with the resulting buffer.
* *
* This hook method only exists for the existing Broker implementation that * This hook method only exists for the existing Broker implementation that
@ -440,7 +439,7 @@ private:
* @return true if the message has been published successfully. * @return true if the message has been published successfully.
*/ */
virtual bool DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, const std::string& format, virtual bool DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, const std::string& format,
detail::byte_buffer& buf) = 0; byte_buffer& buf) = 0;
std::string name; std::string name;
zeek::Tag tag; zeek::Tag tag;
@ -471,7 +470,7 @@ private:
struct EventMessage { struct EventMessage {
std::string topic; std::string topic;
std::string format; std::string format;
detail::byte_buffer payload; byte_buffer payload;
auto payload_span() const { return Span(payload.data(), payload.size()); }; auto payload_span() const { return Span(payload.data(), payload.size()); };
}; };
@ -481,7 +480,7 @@ struct EventMessage {
*/ */
struct LogMessage { struct LogMessage {
std::string format; std::string format;
detail::byte_buffer payload; byte_buffer payload;
auto payload_span() const { return Span(payload.data(), payload.size()); }; auto payload_span() const { return Span(payload.data(), payload.size()); };
}; };
@ -494,7 +493,7 @@ struct LogMessage {
*/ */
struct BackendMessage { struct BackendMessage {
int tag; int tag;
detail::byte_buffer payload; byte_buffer payload;
auto payload_span() const { return Span(payload.data(), payload.size()); }; auto payload_span() const { return Span(payload.data(), payload.size()); };
}; };
@ -546,13 +545,13 @@ private:
/** /**
* Process a backend specific message queued as BackendMessage. * Process a backend specific message queued as BackendMessage.
*/ */
bool ProcessBackendMessage(int tag, detail::byte_buffer_span payload); bool ProcessBackendMessage(int tag, byte_buffer_span payload);
/** /**
* If a cluster backend produces messages of type BackendMessage, * If a cluster backend produces messages of type BackendMessage,
* this method will be invoked by the main thread to process it. * this method will be invoked by the main thread to process it.
*/ */
virtual bool DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) { return false; }; virtual bool DoProcessBackendMessage(int tag, byte_buffer_span payload) { return false; };
/** /**
* Hook method for OnLooProcess. * Hook method for OnLooProcess.

14
src/cluster/Serializer.h
View file

@ -6,7 +6,6 @@
#include <optional> #include <optional>
#include <string> #include <string>
#include <vector>
#include "zeek/Span.h" #include "zeek/Span.h"
#include "zeek/logging/Types.h" #include "zeek/logging/Types.h"
@ -15,13 +14,8 @@ namespace zeek::cluster {
namespace detail { namespace detail {
class Event; class Event;
using byte_buffer = std::vector<std::byte>;
using byte_buffer_span = Span<const std::byte>;
} // namespace detail } // namespace detail
/** /**
* This class handles encoding of events into byte buffers and back. * This class handles encoding of events into byte buffers and back.
* *
@ -40,7 +34,7 @@ public:
* *
* @returns True on success, false in exceptional cases (e.g. unsupported serialization). * @returns True on success, false in exceptional cases (e.g. unsupported serialization).
*/ */
virtual bool SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) = 0; virtual bool SerializeEvent(byte_buffer& buf, const detail::Event& event) = 0;
/** /**
* Unserialize an event from a given byte buffer. * Unserialize an event from a given byte buffer.
@ -49,7 +43,7 @@ public:
* *
* @returns The event, or std::nullopt on error. * @returns The event, or std::nullopt on error.
*/ */
virtual std::optional<cluster::detail::Event> UnserializeEvent(detail::byte_buffer_span buf) = 0; virtual std::optional<cluster::detail::Event> UnserializeEvent(byte_buffer_span buf) = 0;
/** /**
* @returns The name of this event serializer instance. * @returns The name of this event serializer instance.
@ -85,7 +79,7 @@ public:
* @param header The log batch header. * @param header The log batch header.
* @param records The actual log writes. * @param records The actual log writes.
*/ */
virtual bool SerializeLogWrite(detail::byte_buffer& buf, const logging::detail::LogWriteHeader& header, virtual bool SerializeLogWrite(byte_buffer& buf, const logging::detail::LogWriteHeader& header,
zeek::Span<logging::detail::LogRecord> records) = 0; zeek::Span<logging::detail::LogRecord> records) = 0;
/** /**
@ -93,7 +87,7 @@ public:
* *
* @param buf The span representing received log writes. * @param buf The span representing received log writes.
*/ */
virtual std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(detail::byte_buffer_span buf) = 0; virtual std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(byte_buffer_span buf) = 0;
/** /**
* @returns The name of this log serializer instance. * @returns The name of this log serializer instance.

13
src/cluster/backend/zeromq/ZeroMQ.cc
View file

@ -253,8 +253,7 @@ bool ZeroMQBackend::SpawnZmqProxyThread() {
return proxy_thread->Start(); return proxy_thread->Start();
} }
bool ZeroMQBackend::DoPublishEvent(const std::string& topic, const std::string& format, bool ZeroMQBackend::DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) {
const cluster::detail::byte_buffer& buf) {
// Publishing an event happens as a multipart message with 4 parts: // Publishing an event happens as a multipart message with 4 parts:
// //
// * The topic to publish to - this is required by XPUB/XSUB // * The topic to publish to - this is required by XPUB/XSUB
@ -336,7 +335,7 @@ bool ZeroMQBackend::DoUnsubscribe(const std::string& topic_prefix) {
} }
bool ZeroMQBackend::DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, bool ZeroMQBackend::DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format,
cluster::detail::byte_buffer& buf) { byte_buffer& buf) {
ZEROMQ_DEBUG("Publishing %zu bytes of log writes (path %s)", buf.size(), header.path.c_str()); ZEROMQ_DEBUG("Publishing %zu bytes of log writes (path %s)", buf.size(), header.path.c_str());
static std::string message_type = "log-write"; static std::string message_type = "log-write";
@ -405,7 +404,7 @@ void ZeroMQBackend::Run() {
continue; continue;
} }
detail::byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()}; byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()};
LogMessage lm{.format = std::string(msg[2].data<const char>(), msg[2].size()), LogMessage lm{.format = std::string(msg[2].data<const char>(), msg[2].size()),
.payload = std::move(payload)}; .payload = std::move(payload)};
@ -487,7 +486,7 @@ void ZeroMQBackend::Run() {
QueueMessage qm; QueueMessage qm;
auto* start = msg[0].data<std::byte>() + 1; auto* start = msg[0].data<std::byte>() + 1;
auto* end = msg[0].data<std::byte>() + msg[0].size(); auto* end = msg[0].data<std::byte>() + msg[0].size();
detail::byte_buffer topic(start, end); byte_buffer topic(start, end);
if ( first == 1 ) { if ( first == 1 ) {
qm = BackendMessage{1, std::move(topic)}; qm = BackendMessage{1, std::move(topic)};
} }
@ -516,7 +515,7 @@ void ZeroMQBackend::Run() {
if ( sender == NodeId() ) if ( sender == NodeId() )
continue; continue;
detail::byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()}; byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()};
EventMessage em{.topic = std::string(msg[0].data<const char>(), msg[0].size()), EventMessage em{.topic = std::string(msg[0].data<const char>(), msg[0].size()),
.format = std::string(msg[2].data<const char>(), msg[2].size()), .format = std::string(msg[2].data<const char>(), msg[2].size()),
.payload = std::move(payload)}; .payload = std::move(payload)};
@ -644,7 +643,7 @@ void ZeroMQBackend::Run() {
} }
} }
bool ZeroMQBackend::DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) { bool ZeroMQBackend::DoProcessBackendMessage(int tag, byte_buffer_span payload) {
if ( tag == 0 || tag == 1 ) { if ( tag == 0 || tag == 1 ) {
std::string topic{reinterpret_cast<const char*>(payload.data()), payload.size()}; std::string topic{reinterpret_cast<const char*>(payload.data()), payload.size()};
zeek::EventHandlerPtr eh; zeek::EventHandlerPtr eh;

7
src/cluster/backend/zeromq/ZeroMQ.h
View file

@ -60,17 +60,16 @@ private:
void DoTerminate() override; void DoTerminate() override;
bool DoPublishEvent(const std::string& topic, const std::string& format, bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) override;
const cluster::detail::byte_buffer& buf) override;
bool DoSubscribe(const std::string& topic_prefix, SubscribeCallback cb) override; bool DoSubscribe(const std::string& topic_prefix, SubscribeCallback cb) override;
bool DoUnsubscribe(const std::string& topic_prefix) override; bool DoUnsubscribe(const std::string& topic_prefix) override;
bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format,
cluster::detail::byte_buffer& buf) override; byte_buffer& buf) override;
bool DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) override; bool DoProcessBackendMessage(int tag, byte_buffer_span payload) override;
// Script level variables. // Script level variables.
std::string connect_xsub_endpoint; std::string connect_xsub_endpoint;

6
src/cluster/serializer/binary-serialization-format/Serializer.cc
View file

@ -77,7 +77,7 @@ bool detail::BinarySerializationFormatLogSerializer::SerializeLogWrite(byte_buff
} }
std::optional<zeek::logging::detail::LogWriteBatch> detail::BinarySerializationFormatLogSerializer::UnserializeLogWrite( std::optional<zeek::logging::detail::LogWriteBatch> detail::BinarySerializationFormatLogSerializer::UnserializeLogWrite(
detail::byte_buffer_span buf) { byte_buffer_span buf) {
zeek::detail::BinarySerializationFormat fmt; zeek::detail::BinarySerializationFormat fmt;
fmt.StartRead(reinterpret_cast<const char*>(buf.data()), buf.size()); fmt.StartRead(reinterpret_cast<const char*>(buf.data()), buf.size());
@ -145,7 +145,7 @@ std::optional<zeek::logging::detail::LogWriteBatch> detail::BinarySerializationF
TEST_SUITE_BEGIN("cluster serializer binary-serialization-format"); TEST_SUITE_BEGIN("cluster serializer binary-serialization-format");
TEST_CASE("roundtrip") { TEST_CASE("roundtrip") {
detail::byte_buffer buf; zeek::byte_buffer buf;
detail::BinarySerializationFormatLogSerializer serializer; detail::BinarySerializationFormatLogSerializer serializer;
static const auto& stream_id_type = zeek::id::find_type<zeek::EnumType>("Log::ID"); static const auto& stream_id_type = zeek::id::find_type<zeek::EnumType>("Log::ID");
@ -161,7 +161,7 @@ TEST_CASE("roundtrip") {
0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00,
0x00, 0x00, 0x16, 0x01, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; 0x00, 0x00, 0x16, 0x01, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
std::byte* p = reinterpret_cast<std::byte*>(&expected_bytes[0]); std::byte* p = reinterpret_cast<std::byte*>(&expected_bytes[0]);
detail::byte_buffer expected{p, p + sizeof(expected_bytes)}; zeek::byte_buffer expected{p, p + sizeof(expected_bytes)};
auto s = stream_id_type->Lookup("Log::UNKNOWN"); auto s = stream_id_type->Lookup("Log::UNKNOWN");
REQUIRE_GE(s, 0); REQUIRE_GE(s, 0);

4
src/cluster/serializer/binary-serialization-format/Serializer.h
View file

@ -13,10 +13,10 @@ class BinarySerializationFormatLogSerializer : public cluster::LogSerializer {
public: public:
BinarySerializationFormatLogSerializer() : LogSerializer("zeek-bin-serializer") {} BinarySerializationFormatLogSerializer() : LogSerializer("zeek-bin-serializer") {}
bool SerializeLogWrite(cluster::detail::byte_buffer& buf, const logging::detail::LogWriteHeader& header, bool SerializeLogWrite(byte_buffer& buf, const logging::detail::LogWriteHeader& header,
zeek::Span<logging::detail::LogRecord> records) override; zeek::Span<logging::detail::LogRecord> records) override;
std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(detail::byte_buffer_span buf) override; std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(byte_buffer_span buf) override;
}; };
} // namespace zeek::cluster::detail } // namespace zeek::cluster::detail

10
src/cluster/serializer/broker/Serializer.cc
View file

@ -101,7 +101,7 @@ std::optional<detail::Event> detail::to_zeek_event(const broker::zeek::Event& ev
return detail::Event{handler, std::move(vl), ts}; return detail::Event{handler, std::move(vl), ts};
} }
bool detail::BrokerBinV1_Serializer::SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) { bool detail::BrokerBinV1_Serializer::SerializeEvent(byte_buffer& buf, const detail::Event& event) {
auto ev = to_broker_event(event); auto ev = to_broker_event(event);
if ( ! ev ) if ( ! ev )
return false; return false;
@ -117,7 +117,7 @@ bool detail::BrokerBinV1_Serializer::SerializeEvent(detail::byte_buffer& buf, co
return true; return true;
} }
std::optional<detail::Event> detail::BrokerBinV1_Serializer::UnserializeEvent(detail::byte_buffer_span buf) { std::optional<detail::Event> detail::BrokerBinV1_Serializer::UnserializeEvent(byte_buffer_span buf) {
auto r = broker::data_envelope::deserialize(broker::endpoint_id::nil(), broker::endpoint_id::nil(), 0, "", auto r = broker::data_envelope::deserialize(broker::endpoint_id::nil(), broker::endpoint_id::nil(), 0, "",
buf.data(), buf.size()); buf.data(), buf.size());
if ( ! r ) if ( ! r )
@ -152,7 +152,7 @@ bool detail::BrokerJsonV1_Serializer::SerializeEvent(byte_buffer& buf, const det
return true; return true;
} }
std::optional<detail::Event> detail::BrokerJsonV1_Serializer::UnserializeEvent(detail::byte_buffer_span buf) { std::optional<detail::Event> detail::BrokerJsonV1_Serializer::UnserializeEvent(byte_buffer_span buf) {
broker::variant res; broker::variant res;
auto err = auto err =
broker::format::json::v1::decode(std::string_view{reinterpret_cast<const char*>(buf.data()), buf.size()}, res); broker::format::json::v1::decode(std::string_view{reinterpret_cast<const char*>(buf.data()), buf.size()}, res);
@ -173,7 +173,7 @@ TEST_SUITE_BEGIN("cluster serializer broker");
TEST_CASE("roundtrip") { TEST_CASE("roundtrip") {
auto* handler = zeek::event_registry->Lookup("Supervisor::node_status"); auto* handler = zeek::event_registry->Lookup("Supervisor::node_status");
detail::Event e{handler, zeek::Args{zeek::make_intrusive<zeek::StringVal>("TEST"), zeek::val_mgr->Count(42)}}; detail::Event e{handler, zeek::Args{zeek::make_intrusive<zeek::StringVal>("TEST"), zeek::val_mgr->Count(42)}};
detail::byte_buffer buf; zeek::byte_buffer buf;
SUBCASE("json") { SUBCASE("json") {
detail::BrokerJsonV1_Serializer serializer; detail::BrokerJsonV1_Serializer serializer;
@ -201,7 +201,7 @@ TEST_CASE("roundtrip") {
0x01, 0x0e, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x09, 0x01, 0x0e, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x09,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
std::byte* p = reinterpret_cast<std::byte*>(&expected_bytes[0]); std::byte* p = reinterpret_cast<std::byte*>(&expected_bytes[0]);
detail::byte_buffer expected{p, p + sizeof(expected_bytes)}; zeek::byte_buffer expected{p, p + sizeof(expected_bytes)};
serializer.SerializeEvent(buf, e); serializer.SerializeEvent(buf, e);

8
src/cluster/serializer/broker/Serializer.h
View file

@ -34,9 +34,9 @@ class BrokerBinV1_Serializer : public EventSerializer {
public: public:
BrokerBinV1_Serializer() : EventSerializer("broker-bin-v1") {} BrokerBinV1_Serializer() : EventSerializer("broker-bin-v1") {}
bool SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) override; bool SerializeEvent(byte_buffer& buf, const detail::Event& event) override;
std::optional<detail::Event> UnserializeEvent(detail::byte_buffer_span buf) override; std::optional<detail::Event> UnserializeEvent(byte_buffer_span buf) override;
}; };
// Implementation of the EventSerializer that uses the existing broker::detail::val_to_data() // Implementation of the EventSerializer that uses the existing broker::detail::val_to_data()
@ -45,9 +45,9 @@ class BrokerJsonV1_Serializer : public EventSerializer {
public: public:
BrokerJsonV1_Serializer() : EventSerializer("broker-json-v1") {} BrokerJsonV1_Serializer() : EventSerializer("broker-json-v1") {}
bool SerializeEvent(zeek::cluster::detail::byte_buffer& buf, const detail::Event& event) override; bool SerializeEvent(byte_buffer& buf, const detail::Event& event) override;
std::optional<detail::Event> UnserializeEvent(detail::byte_buffer_span buf) override; std::optional<detail::Event> UnserializeEvent(byte_buffer_span buf) override;
}; };
} // namespace zeek::cluster::detail } // namespace zeek::cluster::detail

2
src/plugin/Component.cc
View file

@ -47,6 +47,8 @@ void Component::Describe(ODesc* d) const {
case component::STORAGE_BACKEND: d->Add("Storage Backend"); break; case component::STORAGE_BACKEND: d->Add("Storage Backend"); break;
case component::STORAGE_SERIALIZER: d->Add("Storage Serializer"); break;
default: default:
reporter->InternalWarning("unknown component type in plugin::Component::Describe"); reporter->InternalWarning("unknown component type in plugin::Component::Describe");
d->Add("<unknown component type>"); d->Add("<unknown component type>");

27
src/plugin/Component.h
View file

@ -21,19 +21,20 @@ namespace component {
* Component types. * Component types.
*/ */
enum Type { enum Type {
READER, /// An input reader (not currently used). READER, /// An input reader (not currently used).
WRITER, /// A logging writer (not currently used). WRITER, /// A logging writer (not currently used).
ANALYZER, /// A protocol analyzer. ANALYZER, /// A protocol analyzer.
PACKET_ANALYZER, /// A packet analyzer. PACKET_ANALYZER, /// A packet analyzer.
FILE_ANALYZER, /// A file analyzer. FILE_ANALYZER, /// A file analyzer.
IOSOURCE, /// An I/O source, excluding packet sources. IOSOURCE, /// An I/O source, excluding packet sources.
PKTSRC, /// A packet source. PKTSRC, /// A packet source.
PKTDUMPER, /// A packet dumper. PKTDUMPER, /// A packet dumper.
SESSION_ADAPTER, /// A session adapter analyzer. SESSION_ADAPTER, /// A session adapter analyzer.
CLUSTER_BACKEND, /// A cluster backend. CLUSTER_BACKEND, /// A cluster backend.
EVENT_SERIALIZER, /// A serializer for events, used by cluster backends. EVENT_SERIALIZER, /// A serializer for events, used by cluster backends.
LOG_SERIALIZER, /// A serializer for log batches, used by cluster backends. LOG_SERIALIZER, /// A serializer for log batches, used by cluster backends.
STORAGE_BACKEND, /// A backend for the storage framework. STORAGE_BACKEND, /// A backend for the storage framework.
STORAGE_SERIALIZER, /// A serializer for the storage framework.
}; };
} // namespace component } // namespace component

11
src/storage/Backend.cc
View file

@ -66,7 +66,7 @@ void OpenResultCallback::Complete(OperationResult res) {
} }
Backend::Backend(uint8_t modes, std::string_view tag_name) : modes(modes) { Backend::Backend(uint8_t modes, std::string_view tag_name) : modes(modes) {
tag = storage_mgr->GetComponentTag(std::string{tag_name}); tag = storage_mgr->BackendMgr().GetComponentTag(std::string{tag_name});
tag_str = zeek::obj_desc_short(tag.AsVal().get()); tag_str = zeek::obj_desc_short(tag.AsVal().get());
} }
@ -75,6 +75,15 @@ OperationResult Backend::Open(OpenResultCallback* cb, RecordValPtr options, Type
val_type = std::move(vt); val_type = std::move(vt);
backend_options = options; backend_options = options;
auto stype = options->GetField<EnumVal>("serializer");
zeek::Tag stag{stype};
auto s = storage_mgr->InstantiateSerializer(stag);
if ( ! s )
return {ReturnCode::INITIALIZATION_FAILED, s.error()};
serializer = std::move(s.value());
auto ret = DoOpen(cb, std::move(options)); auto ret = DoOpen(cb, std::move(options));
if ( ! ret.value ) if ( ! ret.value )
ret.value = cb->Backend(); ret.value = cb->Backend();

4
src/storage/Backend.h
View file

@ -5,6 +5,7 @@
#include "zeek/OpaqueVal.h" #include "zeek/OpaqueVal.h"
#include "zeek/Tag.h" #include "zeek/Tag.h"
#include "zeek/Val.h" #include "zeek/Val.h"
#include "zeek/storage/Serializer.h"
namespace zeek::detail::trigger { namespace zeek::detail::trigger {
class Trigger; class Trigger;
@ -58,7 +59,7 @@ struct OperationResult {
class ResultCallback { class ResultCallback {
public: public:
ResultCallback() = default; ResultCallback() = default;
ResultCallback(detail::trigger::TriggerPtr trigger, const void* assoc); ResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc);
virtual ~ResultCallback() = default; virtual ~ResultCallback() = default;
/** /**
@ -239,6 +240,7 @@ protected:
zeek::Tag tag; zeek::Tag tag;
std::string tag_str; std::string tag_str;
std::unique_ptr<Serializer> serializer;
private: private:
/** /**

1
src/storage/CMakeLists.txt
View file

@ -11,3 +11,4 @@ zeek_add_subdir_library(
storage-sync.bif) storage-sync.bif)
add_subdirectory(backend) add_subdirectory(backend)
add_subdirectory(serializer)

25
src/storage/Component.cc
View file

@ -7,19 +7,34 @@
namespace zeek::storage { namespace zeek::storage {
Component::Component(const std::string& name, factory_callback arg_factory) BackendComponent::BackendComponent(const std::string& name, factory_callback arg_factory)
: plugin::Component(plugin::component::STORAGE_BACKEND, name, 0, storage_mgr->GetTagType()) { : plugin::Component(plugin::component::STORAGE_BACKEND, name, 0, storage_mgr->BackendMgr().GetTagType()) {
factory = arg_factory; factory = arg_factory;
} }
void Component::Initialize() { void BackendComponent::Initialize() {
InitializeTag(); InitializeTag();
storage_mgr->RegisterComponent(this); storage_mgr->BackendMgr().RegisterComponent(this, "STORAGE_BACKEND_");
} }
void Component::DoDescribe(ODesc* d) const { void BackendComponent::DoDescribe(ODesc* d) const {
d->Add("Storage::STORAGE_BACKEND_"); d->Add("Storage::STORAGE_BACKEND_");
d->Add(CanonicalName()); d->Add(CanonicalName());
} }
SerializerComponent::SerializerComponent(const std::string& name, factory_callback arg_factory)
: plugin::Component(plugin::component::STORAGE_SERIALIZER, name, 0, storage_mgr->SerializerMgr().GetTagType()) {
factory = arg_factory;
}
void SerializerComponent::Initialize() {
InitializeTag();
storage_mgr->SerializerMgr().RegisterComponent(this, "STORAGE_SERIALIZER_");
}
void SerializerComponent::DoDescribe(ODesc* d) const {
d->Add("Storage::STORAGE_SERIALIZER_");
d->Add(CanonicalName());
}
} // namespace zeek::storage } // namespace zeek::storage

55
src/storage/Component.h
View file

@ -7,11 +7,12 @@
namespace zeek::storage { namespace zeek::storage {
class Backend; class Backend;
class Serializer;
/** /**
* Component description for plugins providing storage backends. * Component description for plugins providing storage backends.
*/ */
class Component : public plugin::Component { class BackendComponent : public plugin::Component {
public: public:
using factory_callback = IntrusivePtr<Backend> (*)(); using factory_callback = IntrusivePtr<Backend> (*)();
@ -27,12 +28,60 @@ public:
* method inside the class that just allocates and returns a new * method inside the class that just allocates and returns a new
* instance. * instance.
*/ */
Component(const std::string& name, factory_callback factory); BackendComponent(const std::string& name, factory_callback factory);
/** /**
* Destructor. * Destructor.
*/ */
~Component() override = default; ~BackendComponent() override = default;
/**
* Initialization function. This function has to be called before any
* plugin component functionality is used; it is used to add the
* plugin component to the list of components and to initialize tags
*/
void Initialize() override;
/**
* Returns the backend's factory function.
*/
factory_callback Factory() const { return factory; }
protected:
/**
* Overridden from plugin::Component.
*/
void DoDescribe(ODesc* d) const override;
private:
factory_callback factory;
};
/**
* Component description for plugins providing serialization for storage data..
*/
class SerializerComponent : public plugin::Component {
public:
using factory_callback = std::unique_ptr<Serializer> (*)();
/**
* Constructor.
*
* @param name The name of the provided backend. This name is used
* across the system to identify the backend.
*
* @param factory A factory function to instantiate instances of the
* backend's class, which must be derived directly or indirectly from
* storage::Backend. This is typically a static \c Instantiate()
* method inside the class that just allocates and returns a new
* instance.
*/
SerializerComponent(const std::string& name, factory_callback factory);
/**
* Destructor.
*/
~SerializerComponent() override = default;
/** /**
* Initialization function. This function has to be called before any * Initialization function. This function has to be called before any

44
src/storage/Manager.cc
View file

@ -28,7 +28,9 @@ void detail::ExpirationTimer::Dispatch(double t, bool is_expire) {
storage_mgr->StartExpirationTimer(); storage_mgr->StartExpirationTimer();
} }
Manager::Manager() : plugin::ComponentManager<storage::Component>("Storage", "Backend") {} Manager::Manager()
: backend_mgr(plugin::ComponentManager<storage::BackendComponent>("Storage", "Backend")),
serializer_mgr(plugin::ComponentManager<storage::SerializerComponent>("Storage", "Serializer")) {}
Manager::~Manager() { Manager::~Manager() {
// TODO: should we shut down any existing backends? force-poll until all of their existing // TODO: should we shut down any existing backends? force-poll until all of their existing
@ -48,24 +50,40 @@ void Manager::InitPostScript() {
StartExpirationTimer(); StartExpirationTimer();
} }
zeek::expected<BackendPtr, std::string> Manager::Instantiate(const Tag& type) { zeek::expected<BackendPtr, std::string> Manager::InstantiateBackend(const Tag& type) {
Component* c = Lookup(type); BackendComponent* c = backend_mgr.Lookup(type);
if ( ! c ) { if ( ! c )
return zeek::unexpected<std::string>( return zeek::unexpected<std::string>(
util::fmt("Request to open unknown backend (%d:%d)", type.Type(), type.Subtype())); util::fmt("Request to instantiate unknown backend type (%d:%d)", type.Type(), type.Subtype()));
}
if ( ! c->Factory() ) { if ( ! c->Factory() )
return zeek::unexpected<std::string>( return zeek::unexpected<std::string>(
util::fmt("Factory invalid for backend %s", GetComponentName(type).c_str())); util::fmt("Factory invalid for backend %s", backend_mgr.GetComponentName(type).c_str()));
}
BackendPtr bp = c->Factory()(); auto bp = c->Factory()();
if ( ! bp ) { if ( ! bp )
return zeek::unexpected<std::string>( return zeek::unexpected<std::string>(
util::fmt("Failed to instantiate backend %s", GetComponentName(type).c_str())); util::fmt("Failed to instantiate backend %s", backend_mgr.GetComponentName(type).c_str()));
}
return bp;
}
zeek::expected<std::unique_ptr<Serializer>, std::string> Manager::InstantiateSerializer(const Tag& type) {
SerializerComponent* c = serializer_mgr.Lookup(type);
if ( ! c )
return zeek::unexpected<std::string>(
util::fmt("Request to instantiate unknown serializer type (%d:%d)", type.Type(), type.Subtype()));
if ( ! c->Factory() )
return zeek::unexpected<std::string>(
util::fmt("Factory invalid for serializer %s", serializer_mgr.GetComponentName(type).c_str()));
auto bp = c->Factory()();
if ( ! bp )
return zeek::unexpected<std::string>(
util::fmt("Failed to instantiate serializer %s", serializer_mgr.GetComponentName(type).c_str()));
return bp; return bp;
} }

20
src/storage/Manager.h
View file

@ -10,6 +10,7 @@
#include "zeek/plugin/ComponentManager.h" #include "zeek/plugin/ComponentManager.h"
#include "zeek/storage/Backend.h" #include "zeek/storage/Backend.h"
#include "zeek/storage/Component.h" #include "zeek/storage/Component.h"
#include "zeek/storage/Serializer.h"
namespace zeek::storage { namespace zeek::storage {
@ -24,7 +25,7 @@ public:
} // namespace detail } // namespace detail
class Manager final : public plugin::ComponentManager<Component> { class Manager final {
public: public:
Manager(); Manager();
~Manager(); ~Manager();
@ -43,7 +44,16 @@ public:
* @return A std::expected containing either a valid BackendPtr with the result of the * @return A std::expected containing either a valid BackendPtr with the result of the
* operation or a string containing an error message for failure. * operation or a string containing an error message for failure.
*/ */
zeek::expected<BackendPtr, std::string> Instantiate(const Tag& type); zeek::expected<BackendPtr, std::string> InstantiateBackend(const Tag& type);
/**
* Instantiates a new serializer object.
*
* @param type The tag for the type of backend being opened.
* @return A std::expected containing either a valid BackendPtr with the result of the
* operation or a string containing an error message for failure.
*/
zeek::expected<std::unique_ptr<Serializer>, std::string> InstantiateSerializer(const Tag& type);
/** /**
* Opens a new storage backend. * Opens a new storage backend.
@ -82,6 +92,9 @@ public:
*/ */
void Expire(double t); void Expire(double t);
plugin::ComponentManager<BackendComponent>& BackendMgr() { return backend_mgr; }
plugin::ComponentManager<SerializerComponent>& SerializerMgr() { return serializer_mgr; }
protected: protected:
friend class storage::detail::ExpirationTimer; friend class storage::detail::ExpirationTimer;
void RunExpireThread(); void RunExpireThread();
@ -94,6 +107,9 @@ protected:
private: private:
std::vector<BackendPtr> backends; std::vector<BackendPtr> backends;
std::mutex backends_mtx; std::mutex backends_mtx;
plugin::ComponentManager<BackendComponent> backend_mgr;
plugin::ComponentManager<SerializerComponent> serializer_mgr;
}; };
} // namespace zeek::storage } // namespace zeek::storage

10
src/storage/ReturnCode.cc
View file

@ -19,6 +19,8 @@ EnumValPtr ReturnCode::CONNECTION_FAILED;
EnumValPtr ReturnCode::DISCONNECTION_FAILED; EnumValPtr ReturnCode::DISCONNECTION_FAILED;
EnumValPtr ReturnCode::INITIALIZATION_FAILED; EnumValPtr ReturnCode::INITIALIZATION_FAILED;
EnumValPtr ReturnCode::IN_PROGRESS; EnumValPtr ReturnCode::IN_PROGRESS;
EnumValPtr ReturnCode::SERIALIZATION_FAILED;
EnumValPtr ReturnCode::UNSERIALIZATION_FAILED;
void ReturnCode::Initialize() { void ReturnCode::Initialize() {
static const auto& return_code_type = zeek::id::find_type<zeek::EnumType>("Storage::ReturnCode"); static const auto& return_code_type = zeek::id::find_type<zeek::EnumType>("Storage::ReturnCode");
@ -61,6 +63,12 @@ void ReturnCode::Initialize() {
tmp = return_code_type->Lookup("Storage::IN_PROGRESS"); tmp = return_code_type->Lookup("Storage::IN_PROGRESS");
IN_PROGRESS = return_code_type->GetEnumVal(tmp); IN_PROGRESS = return_code_type->GetEnumVal(tmp);
tmp = return_code_type->Lookup("Storage::SERIALIZATION_FAILED");
SERIALIZATION_FAILED = return_code_type->GetEnumVal(tmp);
tmp = return_code_type->Lookup("Storage::UNSERIALIZATION_FAILED");
UNSERIALIZATION_FAILED = return_code_type->GetEnumVal(tmp);
} }
void ReturnCode::Cleanup() { void ReturnCode::Cleanup() {
@ -77,6 +85,8 @@ void ReturnCode::Cleanup() {
DISCONNECTION_FAILED.reset(); DISCONNECTION_FAILED.reset();
INITIALIZATION_FAILED.reset(); INITIALIZATION_FAILED.reset();
IN_PROGRESS.reset(); IN_PROGRESS.reset();
SERIALIZATION_FAILED.reset();
UNSERIALIZATION_FAILED.reset();
} }
} // namespace zeek::storage } // namespace zeek::storage

4
src/storage/ReturnCode.h
View file

@ -14,7 +14,7 @@ namespace storage {
* A collection of EnumValPtrs for the default set of result codes in the storage framework. * A collection of EnumValPtrs for the default set of result codes in the storage framework.
* should be kept up-to-date with the Storage::ReturnCodes script-level enum. * should be kept up-to-date with the Storage::ReturnCodes script-level enum.
*/ */
class ReturnCode { class ReturnCode final {
public: public:
static void Initialize(); static void Initialize();
static void Cleanup(); static void Cleanup();
@ -32,6 +32,8 @@ public:
static EnumValPtr DISCONNECTION_FAILED; static EnumValPtr DISCONNECTION_FAILED;
static EnumValPtr INITIALIZATION_FAILED; static EnumValPtr INITIALIZATION_FAILED;
static EnumValPtr IN_PROGRESS; static EnumValPtr IN_PROGRESS;
static EnumValPtr SERIALIZATION_FAILED;
static EnumValPtr UNSERIALIZATION_FAILED;
}; };
} // namespace storage } // namespace storage

42
src/storage/Serializer.h Normal file
View file

@ -0,0 +1,42 @@
// See the file "COPYING" in the main distribution directory for copyright.
#pragma once
#include "zeek/Val.h"
namespace zeek::storage {
/**
* Base class for a serializer used by storage backends.
*/
class Serializer {
public:
virtual ~Serializer() = default;
/**
* Serializes Zeek Val data into another format.
*
* @param val The data to serialize.
*
* @return On success, a byte buffer containing the serialized data. std::nullopt will
* be returned on failure.
*/
virtual std::optional<byte_buffer> Serialize(ValPtr val) = 0;
/**
* Unserializes a byte buffer into Zeek Val objects of a specific type.
*
* @param buf The byte data to unserialize.
* @param type The Zeek script-level type to unserialize the data into.
*
* @return A zeek::expected containing either the unserialized Val data on success, or
* a string containing an error message on failure.
*/
virtual zeek::expected<ValPtr, std::string> Unserialize(byte_buffer_span buf, TypePtr type) = 0;
protected:
Serializer(std::string name) : name(std::move(name)) {}
std::string name;
};
} // namespace zeek::storage

4
src/storage/backend/redis/Plugin.cc
View file

@ -7,10 +7,10 @@
namespace zeek::storage::backend::redis { namespace zeek::storage::backend::redis {
class Plugin : public plugin::Plugin { class Plugin final : public plugin::Plugin {
public: public:
plugin::Configuration Configure() override { plugin::Configuration Configure() override {
AddComponent(new storage::Component("REDIS", backend::redis::Redis::Instantiate)); AddComponent(new storage::BackendComponent("REDIS", backend::redis::Redis::Instantiate));
plugin::Configuration config; plugin::Configuration config;
config.name = "Zeek::Storage_Backend_Redis"; config.name = "Zeek::Storage_Backend_Redis";

48
src/storage/backend/redis/Redis.cc
View file

@ -254,24 +254,30 @@ OperationResult Redis::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool
auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex);
std::string format = "SET %s:%s %s"; std::string format = "SET %s:%b %b";
if ( ! overwrite ) if ( ! overwrite )
format.append(" NX"); format.append(" NX");
auto json_key = key->ToJSON()->ToStdString(); auto key_data = serializer->Serialize(key);
auto json_value = value->ToJSON()->ToStdString(); if ( ! key_data )
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
auto val_data = serializer->Serialize(value);
if ( ! val_data )
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize value"};
int status; int status;
// Use built-in expiration if reading live data, since time will move // Use built-in expiration if reading live data, since time will move
// forward consistently. If reading pcaps, we'll do something else. // forward consistently. If reading pcaps, we'll do something else.
if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) { if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) {
format.append(" PXAT %" PRIu64); format.append(" PXAT %" PRIu64);
status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), key_data->data(),
json_value.data(), static_cast<uint64_t>(expiration_time * 1e3)); key_data->size(), val_data->data(), val_data->size(),
static_cast<uint64_t>(expiration_time * 1e3));
} }
else else
status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(), status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), key_data->data(),
json_value.data()); key_data->size(), val_data->data(), val_data->size());
if ( connected && status == REDIS_ERR ) if ( connected && status == REDIS_ERR )
return {ReturnCode::OPERATION_FAILED, util::fmt("Failed to queue put operation: %s", async_ctx->errstr)}; return {ReturnCode::OPERATION_FAILED, util::fmt("Failed to queue put operation: %s", async_ctx->errstr)};
@ -284,10 +290,10 @@ OperationResult Redis::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool
format = "ZADD %s_expire"; format = "ZADD %s_expire";
if ( ! overwrite ) if ( ! overwrite )
format.append(" NX"); format.append(" NX");
format += " %f %s"; format += " %f %b";
status = redisAsyncCommand(async_ctx, redisZADD, NULL, format.c_str(), key_prefix.data(), expiration_time, status = redisAsyncCommand(async_ctx, redisZADD, NULL, format.c_str(), key_prefix.data(), expiration_time,
json_key.data()); key_data->data(), key_data->size());
if ( connected && status == REDIS_ERR ) if ( connected && status == REDIS_ERR )
return {ReturnCode::OPERATION_FAILED, util::fmt("ZADD operation failed: %s", async_ctx->errstr)}; return {ReturnCode::OPERATION_FAILED, util::fmt("ZADD operation failed: %s", async_ctx->errstr)};
@ -307,8 +313,12 @@ OperationResult Redis::DoGet(ResultCallback* cb, ValPtr key) {
auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex);
int status = redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%s", key_prefix.data(), auto key_data = serializer->Serialize(key);
key->ToJSON()->ToStdStringView().data()); if ( ! key_data )
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
int status =
redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%b", key_prefix.data(), key_data->data(), key_data->size());
if ( connected && status == REDIS_ERR ) if ( connected && status == REDIS_ERR )
return {ReturnCode::OPERATION_FAILED, util::fmt("Failed to queue get operation: %s", async_ctx->errstr)}; return {ReturnCode::OPERATION_FAILED, util::fmt("Failed to queue get operation: %s", async_ctx->errstr)};
@ -330,8 +340,12 @@ OperationResult Redis::DoErase(ResultCallback* cb, ValPtr key) {
auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex); auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex);
int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%s", key_prefix.data(), auto key_data = serializer->Serialize(key);
key->ToJSON()->ToStdStringView().data()); if ( ! key_data )
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%b", key_prefix.data(), key_data->data(),
key_data->size());
if ( connected && status == REDIS_ERR ) if ( connected && status == REDIS_ERR )
return {ReturnCode::OPERATION_FAILED, async_ctx->errstr}; return {ReturnCode::OPERATION_FAILED, async_ctx->errstr};
@ -439,11 +453,11 @@ void Redis::HandleGetResult(redisReply* reply, ResultCallback* callback) {
else if ( reply->type == REDIS_REPLY_ERROR ) else if ( reply->type == REDIS_REPLY_ERROR )
res = ParseReplyError("get", reply->str); res = ParseReplyError("get", reply->str);
else { else {
auto val = zeek::detail::ValFromJSON(reply->str, val_type, Func::nil); auto val = serializer->Unserialize({(std::byte*)reply->str, reply->len}, val_type);
if ( std::holds_alternative<ValPtr>(val) ) if ( val )
res = {ReturnCode::SUCCESS, "", std::get<ValPtr>(val)}; res = {ReturnCode::SUCCESS, "", val.value()};
else else
res = {ReturnCode::OPERATION_FAILED, std::get<std::string>(val)}; res = {ReturnCode::OPERATION_FAILED, val.error()};
} }
freeReplyObject(reply); freeReplyObject(reply);

2
src/storage/backend/redis/Redis.h
View file

@ -13,7 +13,7 @@ struct redisReply;
struct redisPollEvents; struct redisPollEvents;
namespace zeek::storage::backend::redis { namespace zeek::storage::backend::redis {
class Redis : public Backend, public iosource::IOSource { class Redis final : public Backend, public iosource::IOSource {
public: public:
Redis() : Backend(SupportedModes::ASYNC, "REDIS"), IOSource(true) {} Redis() : Backend(SupportedModes::ASYNC, "REDIS"), IOSource(true) {}
~Redis() override = default; ~Redis() override = default;

4
src/storage/backend/sqlite/Plugin.cc
View file

@ -7,10 +7,10 @@
namespace zeek::storage::backend::sqlite { namespace zeek::storage::backend::sqlite {
class Plugin : public plugin::Plugin { class Plugin final : public plugin::Plugin {
public: public:
plugin::Configuration Configure() override { plugin::Configuration Configure() override {
AddComponent(new storage::Component("SQLITE", backend::sqlite::SQLite::Instantiate)); AddComponent(new storage::BackendComponent("SQLITE", backend::sqlite::SQLite::Instantiate));
plugin::Configuration config; plugin::Configuration config;
config.name = "Zeek::Storage_Backend_SQLite"; config.name = "Zeek::Storage_Backend_SQLite";

54
src/storage/backend/sqlite/SQLite.cc
View file

@ -44,7 +44,7 @@ OperationResult SQLite::DoOpen(OpenResultCallback* cb, RecordValPtr options) {
} }
std::string create = "create table if not exists " + table_name + " ("; std::string create = "create table if not exists " + table_name + " (";
create.append("key_str text primary key, value_str text not null, expire_time real);"); create.append("key_str blob primary key, value_str blob not null, expire_time real);");
char* errorMsg = nullptr; char* errorMsg = nullptr;
if ( int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); res != SQLITE_OK ) { if ( int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); res != SQLITE_OK ) {
@ -151,8 +151,9 @@ OperationResult SQLite::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool
if ( ! db ) if ( ! db )
return {ReturnCode::NOT_CONNECTED}; return {ReturnCode::NOT_CONNECTED};
auto json_key = key->ToJSON(); auto key_data = serializer->Serialize(key);
auto json_value = value->ToJSON(); if ( ! key_data )
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
sqlite3_stmt* stmt; sqlite3_stmt* stmt;
if ( ! overwrite ) if ( ! overwrite )
@ -160,15 +161,17 @@ OperationResult SQLite::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool
else else
stmt = put_update_stmt.get(); stmt = put_update_stmt.get();
auto key_str = json_key->ToStdStringView(); if ( auto res = CheckError(sqlite3_bind_blob(stmt, 1, key_data->data(), key_data->size(), SQLITE_STATIC));
if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC));
res.code != ReturnCode::SUCCESS ) { res.code != ReturnCode::SUCCESS ) {
sqlite3_reset(stmt); sqlite3_reset(stmt);
return res; return res;
} }
auto value_str = json_value->ToStdStringView(); auto val_data = serializer->Serialize(value);
if ( auto res = CheckError(sqlite3_bind_text(stmt, 2, value_str.data(), value_str.size(), SQLITE_STATIC)); if ( ! val_data )
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize value"};
if ( auto res = CheckError(sqlite3_bind_blob(stmt, 2, val_data->data(), val_data->size(), SQLITE_STATIC));
res.code != ReturnCode::SUCCESS ) { res.code != ReturnCode::SUCCESS ) {
sqlite3_reset(stmt); sqlite3_reset(stmt);
return res; return res;
@ -180,7 +183,7 @@ OperationResult SQLite::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool
} }
if ( overwrite ) { if ( overwrite ) {
if ( auto res = CheckError(sqlite3_bind_text(stmt, 4, value_str.data(), value_str.size(), SQLITE_STATIC)); if ( auto res = CheckError(sqlite3_bind_blob(stmt, 4, val_data->data(), val_data->size(), SQLITE_STATIC));
res.code != ReturnCode::SUCCESS ) { res.code != ReturnCode::SUCCESS ) {
sqlite3_reset(stmt); sqlite3_reset(stmt);
return res; return res;
@ -197,11 +200,13 @@ OperationResult SQLite::DoGet(ResultCallback* cb, ValPtr key) {
if ( ! db ) if ( ! db )
return {ReturnCode::NOT_CONNECTED}; return {ReturnCode::NOT_CONNECTED};
auto json_key = key->ToJSON(); auto key_data = serializer->Serialize(key);
if ( ! key_data )
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
auto stmt = get_stmt.get(); auto stmt = get_stmt.get();
auto key_str = json_key->ToStdStringView(); if ( auto res = CheckError(sqlite3_bind_blob(stmt, 1, key_data->data(), key_data->size(), SQLITE_STATIC));
if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC));
res.code != ReturnCode::SUCCESS ) { res.code != ReturnCode::SUCCESS ) {
sqlite3_reset(stmt); sqlite3_reset(stmt);
return res; return res;
@ -217,11 +222,13 @@ OperationResult SQLite::DoErase(ResultCallback* cb, ValPtr key) {
if ( ! db ) if ( ! db )
return {ReturnCode::NOT_CONNECTED}; return {ReturnCode::NOT_CONNECTED};
auto json_key = key->ToJSON(); auto key_data = serializer->Serialize(key);
if ( ! key_data )
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
auto stmt = erase_stmt.get(); auto stmt = erase_stmt.get();
auto key_str = json_key->ToStdStringView(); if ( auto res = CheckError(sqlite3_bind_blob(stmt, 1, key_data->data(), key_data->size(), SQLITE_STATIC));
if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC));
res.code != ReturnCode::SUCCESS ) { res.code != ReturnCode::SUCCESS ) {
sqlite3_reset(stmt); sqlite3_reset(stmt);
return res; return res;
@ -266,17 +273,16 @@ OperationResult SQLite::Step(sqlite3_stmt* stmt, bool parse_value) {
int step_status = sqlite3_step(stmt); int step_status = sqlite3_step(stmt);
if ( step_status == SQLITE_ROW ) { if ( step_status == SQLITE_ROW ) {
if ( parse_value ) { if ( parse_value ) {
// Column 1 is the value auto blob = static_cast<const std::byte*>(sqlite3_column_blob(stmt, 0));
const char* text = (const char*)sqlite3_column_text(stmt, 0); size_t blob_size = sqlite3_column_bytes(stmt, 0);
auto val = zeek::detail::ValFromJSON(text, val_type, Func::nil);
auto val = serializer->Unserialize({blob, blob_size}, val_type);
sqlite3_reset(stmt); sqlite3_reset(stmt);
if ( std::holds_alternative<ValPtr>(val) ) {
ValPtr val_v = std::get<ValPtr>(val); if ( val )
ret = {ReturnCode::SUCCESS, "", val_v}; ret = {ReturnCode::SUCCESS, "", val.value()};
} else
else { ret = {ReturnCode::OPERATION_FAILED, val.error()};
ret = {ReturnCode::OPERATION_FAILED, std::get<std::string>(val)};
}
} }
else { else {
ret = {ReturnCode::OPERATION_FAILED, "sqlite3_step should not have returned a value"}; ret = {ReturnCode::OPERATION_FAILED, "sqlite3_step should not have returned a value"};

2
src/storage/backend/sqlite/SQLite.h
View file

@ -10,7 +10,7 @@ struct sqlite3_stmt;
namespace zeek::storage::backend::sqlite { namespace zeek::storage::backend::sqlite {
class SQLite : public Backend { class SQLite final : public Backend {
public: public:
SQLite() : Backend(SupportedModes::SYNC, "SQLITE") {} SQLite() : Backend(SupportedModes::SYNC, "SQLITE") {}
~SQLite() override = default; ~SQLite() override = default;

1
src/storage/serializer/CMakeLists.txt Normal file
View file

@ -0,0 +1 @@
add_subdirectory(json)

3
src/storage/serializer/json/CMakeLists.txt Normal file
View file

@ -0,0 +1,3 @@
zeek_add_plugin(
Zeek Storage_Serializer_JSON
SOURCES JSON.cc Plugin.cc)

44
src/storage/serializer/json/JSON.cc Normal file
View file

@ -0,0 +1,44 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "zeek/storage/serializer/json/JSON.h"
#include "zeek/Func.h"
namespace zeek::storage::serializer::json {
std::string JSON::versioned_name = "JSONv1";
std::unique_ptr<Serializer> JSON::Instantiate() { return std::make_unique<JSON>(); }
JSON::JSON() : Serializer("JSON") {}
std::optional<byte_buffer> JSON::Serialize(ValPtr val) {
static auto byte_converter = [](u_char c) { return std::byte(c); };
byte_buffer buf;
auto json = val->ToJSON();
buf.reserve(json->Len() + versioned_name.size() + 1);
std::transform(versioned_name.begin(), versioned_name.end(), std::back_inserter(buf), byte_converter);
buf.push_back(static_cast<std::byte>(';'));
std::transform(json->Bytes(), json->Bytes() + json->Len(), std::back_inserter(buf), byte_converter);
return buf;
}
zeek::expected<ValPtr, std::string> JSON::Unserialize(byte_buffer_span buf, TypePtr type) {
std::string_view text{reinterpret_cast<std::string::const_pointer>(buf.data()), buf.size()};
auto semicolon = text.find(';');
if ( semicolon == std::string::npos )
return zeek::unexpected<std::string>("Version string missing");
std::string_view version = std::string_view(text).substr(0, semicolon);
if ( version != versioned_name )
return zeek::unexpected<std::string>(
util::fmt("Version doesn't match: %s vs %s", version.data(), versioned_name.c_str()));
return zeek::detail::ValFromJSON(text.substr(semicolon + 1), type, Func::nil);
}
} // namespace zeek::storage::serializer::json

23
src/storage/serializer/json/JSON.h Normal file
View file

@ -0,0 +1,23 @@
// See the file "COPYING" in the main distribution directory for copyright.
#pragma once
#include "zeek/storage/Serializer.h"
namespace zeek::storage::serializer::json {
class JSON final : public Serializer {
public:
static std::unique_ptr<Serializer> Instantiate();
JSON();
~JSON() override = default;
std::optional<byte_buffer> Serialize(ValPtr val) override;
zeek::expected<ValPtr, std::string> Unserialize(byte_buffer_span buf, TypePtr type) override;
private:
static std::string versioned_name;
};
} // namespace zeek::storage::serializer::json

22
src/storage/serializer/json/Plugin.cc Normal file
View file

@ -0,0 +1,22 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "zeek/plugin/Plugin.h"
#include "zeek/storage/Component.h"
#include "zeek/storage/serializer/json/JSON.h"
namespace zeek::storage::serializer::json {
class Plugin final : public plugin::Plugin {
public:
plugin::Configuration Configure() override {
AddComponent(new storage::SerializerComponent("JSON", serializer::json::JSON::Instantiate));
plugin::Configuration config;
config.name = "Zeek::Storage_Serializer_JSON";
config.description = "JSON serializer for storage framework";
return config;
}
} plugin;
} // namespace zeek::storage::serializer::json

2
src/storage/storage-async.bif
View file

@ -82,7 +82,7 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any,
auto btype_val = IntrusivePtr<EnumVal>{NewRef{}, btype->AsEnumVal()}; auto btype_val = IntrusivePtr<EnumVal>{NewRef{}, btype->AsEnumVal()};
Tag tag{btype_val}; Tag tag{btype_val};
auto b = storage_mgr->Instantiate(tag); auto b = storage_mgr->InstantiateBackend(tag);
if ( ! b.has_value() ) { if ( ! b.has_value() ) {
trigger->Cache( trigger->Cache(

2
src/storage/storage-sync.bif
View file

@ -33,7 +33,7 @@ function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, k
auto btype_val = IntrusivePtr<EnumVal>{NewRef{}, btype->AsEnumVal()}; auto btype_val = IntrusivePtr<EnumVal>{NewRef{}, btype->AsEnumVal()};
Tag tag{btype_val}; Tag tag{btype_val};
auto b = storage_mgr->Instantiate(tag); auto b = storage_mgr->InstantiateBackend(tag);
if ( ! b.has_value() ) { if ( ! b.has_value() ) {
emit_builtin_error(b.error().c_str()); emit_builtin_error(b.error().c_str());

6
src/supervisor/Supervisor.cc
View file

@ -1363,8 +1363,8 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const {
auto tt = rt->GetFieldType<TableType>("cluster"); auto tt = rt->GetFieldType<TableType>("cluster");
auto json_res = detail::ValFromJSON(cluster, tt, Func::nil); auto json_res = detail::ValFromJSON(cluster, tt, Func::nil);
if ( auto val = std::get_if<ValPtr>(&json_res) ) { if ( json_res ) {
rval->AssignField("cluster", *val); rval->AssignField("cluster", json_res.value());
} }
else { else {
// This should never happen: the JSON data comes from a table[string] of // This should never happen: the JSON data comes from a table[string] of
@ -1372,7 +1372,7 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const {
// here can be hard to debug. Other JSON code (see FromJSON()) fails // here can be hard to debug. Other JSON code (see FromJSON()) fails
// silently when the JSON is misformatted. We just warn: // silently when the JSON is misformatted. We just warn:
fprintf(stderr, "Could not parse %s's cluster table from '%s': %s\n", name.c_str(), cluster.c_str(), fprintf(stderr, "Could not parse %s's cluster table from '%s': %s\n", name.c_str(), cluster.c_str(),
std::get<std::string>(json_res).c_str()); json_res.error().c_str());
rval->AssignField("cluster", make_intrusive<TableVal>(std::move(tt))); rval->AssignField("cluster", make_intrusive<TableVal>(std::move(tt)));
} }

8
src/util.h
View file

@ -16,14 +16,12 @@
#include <libgen.h> #include <libgen.h>
#include <unistd.h> #include <unistd.h>
#include <array>
#include <cinttypes> #include <cinttypes>
#include <cstdarg> #include <cstdarg>
#include <cstdint> #include <cstdint>
#include <cstdio> #include <cstdio>
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
#include <memory> // std::unique_ptr
#include <string> #include <string>
#include <string_view> #include <string_view>
#include <vector> #include <vector>
@ -106,6 +104,8 @@ template<typename E>
using unexpected = nonstd::unexpected<E>; using unexpected = nonstd::unexpected<E>;
} // namespace zeek } // namespace zeek
#include "zeek/Span.h"
using zeek_int_t = int64_t; using zeek_int_t = int64_t;
using zeek_uint_t = uint64_t; using zeek_uint_t = uint64_t;
@ -121,6 +121,10 @@ namespace zeek {
class ODesc; class ODesc;
class RecordVal; class RecordVal;
// Byte buffer types used by serialization code in storage and cluster.
using byte_buffer = std::vector<std::byte>;
using byte_buffer_span = Span<const std::byte>;
namespace util { namespace util {
namespace detail { namespace detail {

6
src/zeek.bif
View file

@ -5268,15 +5268,15 @@ function from_json%(s: string, t: any, key_func: string_mapper &default=from_jso
auto res = zeek::detail::ValFromJSON(s->ToStdStringView(), t->AsType()->AsTypeType()->GetType(), auto res = zeek::detail::ValFromJSON(s->ToStdStringView(), t->AsType()->AsTypeType()->GetType(),
key_func_ptr); key_func_ptr);
if ( auto val = std::get_if<zeek::ValPtr>(&res) ) if ( res )
{ {
rval->Assign(v_idx, *val); rval->Assign(v_idx, res.value());
rval->Assign(valid_idx, true); rval->Assign(valid_idx, true);
} }
else else
{ {
rval->Assign(valid_idx, false); rval->Assign(valid_idx, false);
zeek::emit_builtin_error(std::get<std::string>(res).c_str()); zeek::emit_builtin_error(res.error().c_str());
} }
return std::move(rval); return std::move(rval);

2
src/zeekygen/ScriptInfo.cc
View file

@ -324,6 +324,8 @@ void ScriptInfo::DoInitPostScript() {
else if ( name == "base/frameworks/storage/main.zeek" ) { else if ( name == "base/frameworks/storage/main.zeek" ) {
const auto& backend_id = zeek::detail::global_scope()->Find("Storage::Backend"); const auto& backend_id = zeek::detail::global_scope()->Find("Storage::Backend");
types.push_back(new IdentifierInfo(backend_id, this)); types.push_back(new IdentifierInfo(backend_id, this));
const auto& serializer_id = zeek::detail::global_scope()->Find("Storage::Serializer");
types.push_back(new IdentifierInfo(serializer_id, this));
} }
} }

2
testing/btest/Baseline/plugins.storage/output
View file

@ -10,7 +10,7 @@ get result after erase, [code=Storage::KEY_NOT_FOUND, error_str=<uninitialized>,
close result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<uninitialized>] close result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<uninitialized>]
results of trying to use closed handle: get: Storage::NOT_CONNECTED, put: Storage::NOT_CONNECTED, erase: Storage::NOT_CONNECTED results of trying to use closed handle: get: Storage::NOT_CONNECTED, put: Storage::NOT_CONNECTED, erase: Storage::NOT_CONNECTED
open result 2, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error, value=<opaque of BackendHandleVal>] open result 2, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::STORAGE_BACKEND_STORAGEDUMMY: open_fail was set to true, returning error, value=<opaque of BackendHandleVal>]
close result on closed handle, [code=Storage::NOT_CONNECTED, error_str=Backend is closed, value=<uninitialized>] close result on closed handle, [code=Storage::NOT_CONNECTED, error_str=Backend is closed, value=<uninitialized>]
open result 3, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<opaque of BackendHandleVal>] open result 3, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<opaque of BackendHandleVal>]

4
testing/btest/Baseline/scripts.base.frameworks.storage.redis-disconnect/out
View file

@ -1,4 +1,4 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
open_result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<opaque of BackendHandleVal>] open_result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<opaque of BackendHandleVal>]
Storage::backend_opened, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]] Storage::backend_opened, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]]
Storage::backend_lost, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]], Server closed the connection Storage::backend_lost, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]], Server closed the connection

4
testing/btest/Baseline/scripts.base.frameworks.storage.redis-sync/out
View file

@ -9,5 +9,5 @@ get result same as originally inserted, T
put result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<uninitialized>] put result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<uninitialized>]
get result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=value2345] get result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=value2345]
get result same as overwritten, T get result same as overwritten, T
Storage::backend_opened, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]] Storage::backend_opened, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]]
Storage::backend_lost, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]], Client disconnected Storage::backend_lost, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]], Client disconnected

2
testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-basic/out
View file

@ -1,5 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Storage::backend_opened, Storage::SQLITE, [sqlite=[database_path=test.sqlite, table_name=testing, tuning_params={ Storage::backend_opened, Storage::STORAGE_BACKEND_SQLITE, [serializer=Storage::STORAGE_SERIALIZER_JSON, sqlite=[database_path=test.sqlite, table_name=testing, tuning_params={
[synchronous] = normal, [synchronous] = normal,
[temp_store] = memory, [temp_store] = memory,
[journal_mode] = WAL [journal_mode] = WAL

2
testing/btest/Baseline/scripts.base.frameworks.storage.sqlite-error-handling/out
View file

@ -1,5 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Open result, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file, value=<opaque of BackendHandleVal>] Open result, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::STORAGE_BACKEND_SQLITE: SQLite call failed: unable to open database file, value=<opaque of BackendHandleVal>]
Open result 2, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<opaque of BackendHandleVal>] Open result 2, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<opaque of BackendHandleVal>]
Put result with bad key type, [code=Storage::KEY_TYPE_MISMATCH, error_str=<uninitialized>, value=<uninitialized>] Put result with bad key type, [code=Storage::KEY_TYPE_MISMATCH, error_str=<uninitialized>, value=<uninitialized>]
Put result on closed handle, [code=Storage::NOT_CONNECTED, error_str=Backend is closed, value=<uninitialized>] Put result on closed handle, [code=Storage::NOT_CONNECTED, error_str=Backend is closed, value=<uninitialized>]

3
testing/btest/plugins/storage-plugin/src/Plugin.cc
View file

@ -11,7 +11,8 @@ Plugin plugin;
using namespace btest::plugin::Testing_StorageDummy; using namespace btest::plugin::Testing_StorageDummy;
zeek::plugin::Configuration Plugin::Configure() { zeek::plugin::Configuration Plugin::Configure() {
AddComponent(new zeek::storage::Component("StorageDummy", btest::storage::backend::StorageDummy::Instantiate)); AddComponent(
new zeek::storage::BackendComponent("StorageDummy", btest::storage::backend::StorageDummy::Instantiate));
zeek::plugin::Configuration config; zeek::plugin::Configuration config;
config.name = "Testing::StorageDummy"; config.name = "Testing::StorageDummy";

35
testing/btest/plugins/storage-plugin/src/StorageDummy.cc
View file

@ -49,9 +49,10 @@ OperationResult StorageDummy::DoPut(ResultCallback* cb, ValPtr key, ValPtr value
if ( timeout_put ) if ( timeout_put )
return {ReturnCode::TIMEOUT}; return {ReturnCode::TIMEOUT};
auto json_key = key->ToJSON()->ToStdString(); auto key_data = serializer->Serialize(key);
auto json_value = value->ToJSON()->ToStdString(); auto val_data = serializer->Serialize(value);
data[json_key] = json_value;
data[*key_data] = *val_data;
return {ReturnCode::SUCCESS}; return {ReturnCode::SUCCESS};
} }
@ -59,31 +60,31 @@ OperationResult StorageDummy::DoPut(ResultCallback* cb, ValPtr key, ValPtr value
* The workhorse method for Get(). This must be implemented for plugins. * The workhorse method for Get(). This must be implemented for plugins.
*/ */
OperationResult StorageDummy::DoGet(ResultCallback* cb, ValPtr key) { OperationResult StorageDummy::DoGet(ResultCallback* cb, ValPtr key) {
auto json_key = key->ToJSON(); auto key_data = serializer->Serialize(key);
auto it = data.find(json_key->ToStdString());
auto it = data.find(*key_data);
if ( it == data.end() ) if ( it == data.end() )
return {ReturnCode::KEY_NOT_FOUND}; return {ReturnCode::KEY_NOT_FOUND};
auto val = zeek::detail::ValFromJSON(it->second.c_str(), val_type, Func::nil); auto val = serializer->Unserialize(it->second, val_type);
if ( std::holds_alternative<ValPtr>(val) ) { if ( val )
ValPtr val_v = std::get<ValPtr>(val); return {ReturnCode::SUCCESS, "", val.value()};
return {ReturnCode::SUCCESS, "", val_v};
}
return {ReturnCode::OPERATION_FAILED, std::get<std::string>(val)}; return {ReturnCode::UNSERIALIZATION_FAILED, val.error()};
} }
/** /**
* The workhorse method for Erase(). This must be implemented for plugins. * The workhorse method for Erase(). This must be implemented for plugins.
*/ */
OperationResult StorageDummy::DoErase(ResultCallback* cb, ValPtr key) { OperationResult StorageDummy::DoErase(ResultCallback* cb, ValPtr key) {
auto json_key = key->ToJSON(); auto key_data = serializer->Serialize(key);
auto it = data.find(json_key->ToStdString());
if ( it == data.end() )
return {ReturnCode::KEY_NOT_FOUND};
data.erase(it); if ( auto it = data.find(*key_data); it != data.end() ) {
return {ReturnCode::SUCCESS}; data.erase(it);
return {ReturnCode::SUCCESS};
}
return {ReturnCode::KEY_NOT_FOUND};
} }
} // namespace btest::storage::backend } // namespace btest::storage::backend

2
testing/btest/plugins/storage-plugin/src/StorageDummy.h
View file

@ -50,7 +50,7 @@ public:
zeek::storage::OperationResult DoErase(zeek::storage::ResultCallback* cb, zeek::ValPtr key) override; zeek::storage::OperationResult DoErase(zeek::storage::ResultCallback* cb, zeek::ValPtr key) override;
private: private:
std::map<std::string, std::string> data; std::map<zeek::byte_buffer, zeek::byte_buffer> data;
bool open = false; bool open = false;
}; };

6
testing/btest/plugins/storage.zeek
View file

@ -28,7 +28,7 @@ event zeek_init() {
local value = "value5678"; local value = "value5678";
# Basic operation. Open, put, and get the value back. # Basic operation. Open, put, and get the value back.
local res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, string, string); local res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_STORAGEDUMMY, opts, string, string);
print "open result", res; print "open result", res;
local b = res$value; local b = res$value;
@ -63,7 +63,7 @@ event zeek_init() {
# Test failing to open the handle and test closing an invalid handle. # Test failing to open the handle and test closing an invalid handle.
opts$dummy = [$open_fail = T, $timeout_put = F]; opts$dummy = [$open_fail = T, $timeout_put = F];
res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, string, string); res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_STORAGEDUMMY, opts, string, string);
print "open result 2", res; print "open result 2", res;
res = Storage::Sync::close_backend(res$value); res = Storage::Sync::close_backend(res$value);
print "close result on closed handle", res; print "close result on closed handle", res;
@ -71,7 +71,7 @@ event zeek_init() {
# Test timing out an async put request. # Test timing out an async put request.
opts$dummy = [$open_fail = F, $timeout_put = T]; opts$dummy = [$open_fail = F, $timeout_put = T];
res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, string, string); res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_STORAGEDUMMY, opts, string, string);
print "open result 3", res; print "open result 3", res;
b = res$value; b = res$value;

2
testing/btest/scripts/base/frameworks/storage/compound-types.zeek
View file

@ -63,7 +63,7 @@ event zeek_init() {
value[2] = "b"; value[2] = "b";
value[3] = "c"; value[3] = "c";
local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, Rec, tbl); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, Rec, tbl);
print "open result", open_res; print "open result", open_res;
local b = open_res$value; local b = open_res$value;

2
testing/btest/scripts/base/frameworks/storage/erase.zeek
View file

@ -17,7 +17,7 @@ event zeek_init()
# Test inserting/retrieving a key/value pair that we know won't be in # Test inserting/retrieving a key/value pair that we know won't be in
# the backend yet. # the backend yet.
local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string);
print "open result", open_res; print "open result", open_res;
local b = open_res$value; local b = open_res$value;

2
testing/btest/scripts/base/frameworks/storage/expiration.zeek
View file

@ -33,7 +33,7 @@ event setup_test()
local opts : Storage::BackendOptions; local opts : Storage::BackendOptions;
opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"]; opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"];
local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string);
print "open result", open_res; print "open result", open_res;
b = open_res$value; b = open_res$value;

2
testing/btest/scripts/base/frameworks/storage/overwriting.zeek
View file

@ -18,7 +18,7 @@ event zeek_init() {
local value = "value7890"; local value = "value7890";
local value2 = "value2345"; local value2 = "value2345";
local res = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str); local res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, str, str);
print "open result", res; print "open result", res;
local b = res$value; local b = res$value;

2
testing/btest/scripts/base/frameworks/storage/redis-async-reading-pcap.zeek
View file

@ -22,7 +22,7 @@ event zeek_init()
local key = "key1234"; local key = "key1234";
local value = "value5678"; local value = "value5678";
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
print "open result", open_res; print "open result", open_res;
local b = open_res$value; local b = open_res$value;

2
testing/btest/scripts/base/frameworks/storage/redis-async.zeek
View file

@ -25,7 +25,7 @@ event zeek_init()
local value = "value5678"; local value = "value5678";
when [opts, key, value] ( local open_res = Storage::Async::open_backend( when [opts, key, value] ( local open_res = Storage::Async::open_backend(
Storage::REDIS, opts, string, string) ) Storage::STORAGE_BACKEND_REDIS, opts, string, string) )
{ {
print "open result", open_res; print "open result", open_res;
local b = open_res$value; local b = open_res$value;

2
testing/btest/scripts/base/frameworks/storage/redis-cluster.zeek
View file

@ -40,7 +40,7 @@ event zeek_init()
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
"REDIS_PORT")), $key_prefix="testing" ]; "REDIS_PORT")), $key_prefix="testing" ];
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
backend = open_res$value; backend = open_res$value;
} }

3
testing/btest/scripts/base/frameworks/storage/redis-disconnect.zeek
View file

@ -26,13 +26,14 @@ event Storage::backend_lost(tag: Storage::Backend, config: any, reason: string)
event zeek_init() event zeek_init()
{ {
local opts: Storage::BackendOptions; local opts: Storage::BackendOptions;
opts$serializer = Storage::STORAGE_SERIALIZER_JSON;
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
"REDIS_PORT")), $key_prefix="testing" ]; "REDIS_PORT")), $key_prefix="testing" ];
local key = "key1234"; local key = "key1234";
local value = "value1234"; local value = "value1234";
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
print "open_result", open_res; print "open_result", open_res;
# Kill the redis server so the backend will disconnect and fire the backend_lost event. # Kill the redis server so the backend will disconnect and fire the backend_lost event.

2
testing/btest/scripts/base/frameworks/storage/redis-erase.zeek
View file

@ -21,7 +21,7 @@ event zeek_init()
local key = "key1234"; local key = "key1234";
local value = "value1234"; local value = "value1234";
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
print "open_result", open_res; print "open_result", open_res;
local b = open_res$value; local b = open_res$value;

2
testing/btest/scripts/base/frameworks/storage/redis-expiration.zeek
View file

@ -40,7 +40,7 @@ event setup_test()
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
"REDIS_PORT")), $key_prefix="testing" ]; "REDIS_PORT")), $key_prefix="testing" ];
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
print "open result", open_res; print "open result", open_res;
b = open_res$value; b = open_res$value;

2
testing/btest/scripts/base/frameworks/storage/redis-native-expiration.zeek
View file

@ -40,7 +40,7 @@ event setup_test()
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
"REDIS_PORT")), $key_prefix="testing" ]; "REDIS_PORT")), $key_prefix="testing" ];
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
print "open result", open_res; print "open result", open_res;
b = open_res$value; b = open_res$value;

3
testing/btest/scripts/base/frameworks/storage/redis-sync.zeek
View file

@ -24,6 +24,7 @@ event Storage::backend_lost(tag: Storage::Backend, config: any, reason: string)
event zeek_init() event zeek_init()
{ {
local opts: Storage::BackendOptions; local opts: Storage::BackendOptions;
opts$serializer = Storage::STORAGE_SERIALIZER_JSON;
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv( opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
"REDIS_PORT")), $key_prefix="testing" ]; "REDIS_PORT")), $key_prefix="testing" ];
@ -31,7 +32,7 @@ event zeek_init()
local value = "value1234"; local value = "value1234";
local value2 = "value2345"; local value2 = "value2345";
local res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string); local res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
print "open_result", res; print "open_result", res;
local b = res$value; local b = res$value;

2
testing/btest/scripts/base/frameworks/storage/sqlite-basic-reading-pcap.zeek
View file

@ -20,7 +20,7 @@ event zeek_init()
# Test inserting/retrieving a key/value pair that we know won't be in # Test inserting/retrieving a key/value pair that we know won't be in
# the backend yet. # the backend yet.
local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string);
print "open result", open_res; print "open result", open_res;
local b = open_res$value; local b = open_res$value;

2
testing/btest/scripts/base/frameworks/storage/sqlite-basic-sync-in-when.zeek
View file

@ -24,7 +24,7 @@ event zeek_init()
# Test inserting/retrieving a key/value pair that we know won't be in # Test inserting/retrieving a key/value pair that we know won't be in
# the backend yet. # the backend yet.
when [opts, key, value] ( local open_res = Storage::Sync::open_backend( when [opts, key, value] ( local open_res = Storage::Sync::open_backend(
Storage::SQLITE, opts, string, string) ) Storage::STORAGE_BACKEND_SQLITE, opts, string, string) )
{ {
print "open result", open_res; print "open result", open_res;
local b = open_res$value; local b = open_res$value;

3
testing/btest/scripts/base/frameworks/storage/sqlite-basic.zeek
View file

@ -16,6 +16,7 @@ event zeek_init()
{ {
# Create a database file in the .tmp directory with a 'testing' table # Create a database file in the .tmp directory with a 'testing' table
local opts: Storage::BackendOptions; local opts: Storage::BackendOptions;
opts$serializer = Storage::STORAGE_SERIALIZER_JSON;
opts$sqlite = [ $database_path="test.sqlite", $table_name="testing" ]; opts$sqlite = [ $database_path="test.sqlite", $table_name="testing" ];
local key = "key1234"; local key = "key1234";
@ -24,7 +25,7 @@ event zeek_init()
# Test inserting/retrieving a key/value pair that we know won't be in # Test inserting/retrieving a key/value pair that we know won't be in
# the backend yet. # the backend yet.
when [opts, key, value] ( local open_res = Storage::Async::open_backend( when [opts, key, value] ( local open_res = Storage::Async::open_backend(
Storage::SQLITE, opts, string, string) ) Storage::STORAGE_BACKEND_SQLITE, opts, string, string) )
{ {
print "open result", open_res; print "open result", open_res;
local b = open_res$value; local b = open_res$value;

4
testing/btest/scripts/base/frameworks/storage/sqlite-error-handling.zeek
View file

@ -14,12 +14,12 @@ event zeek_init() {
$table_name = "testing"]; $table_name = "testing"];
# This should report an error in .stderr and reporter.log # This should report an error in .stderr and reporter.log
local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string); local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string);
print "Open result", open_res; print "Open result", open_res;
# Open a valid database file # Open a valid database file
opts$sqlite$database_path = "test.sqlite"; opts$sqlite$database_path = "test.sqlite";
open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string); open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string);
print "Open result 2", open_res; print "Open result 2", open_res;
local b = open_res$value; local b = open_res$value;