mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
Merge remote-tracking branch 'origin/topic/timw/storage-serialization'
* origin/topic/timw/storage-serialization: Add STORAGE_ prefixes for backends and serializers Add versioning to JSON serializer Remove unnecessary includes in Val.h Move byte_buffer types from cluster and storage into util Remove unnecessary <array> and <memory> includes from util.h Mark storage classes as final where appropriate Add JSON storage serializer, use with existing backends/tests Make ValFromJSON return zeek::expected instead of a variant Ground work for pluggable storage serializers
This commit is contained in:
Tim Wojtulewicz
commit
6ecb8f0f5f
71 changed files with 563 additions and 258 deletions
20
CHANGES
20
CHANGES
|
|
@ -1,3 +1,23 @@
|
|||
7.2.0-dev.530 | 2025-04-14 10:11:59 -0700
|
||||
|
||||
* Add STORAGE_ prefixes for backends and serializers (Tim Wojtulewicz, Corelight)
|
||||
|
||||
* Add versioning to JSON serializer (Tim Wojtulewicz, Corelight)
|
||||
|
||||
* Remove unnecessary includes in Val.h (Tim Wojtulewicz, Corelight)
|
||||
|
||||
* Move byte_buffer types from cluster and storage into util (Tim Wojtulewicz, Corelight)
|
||||
|
||||
* Remove unnecessary <array> and <memory> includes from util.h (Tim Wojtulewicz, Corelight)
|
||||
|
||||
* Mark storage classes as final where appropriate (Tim Wojtulewicz, Corelight)
|
||||
|
||||
* Add JSON storage serializer, use with existing backends/tests (Tim Wojtulewicz, Corelight)
|
||||
|
||||
* Make ValFromJSON return zeek::expected instead of a variant (Tim Wojtulewicz, Corelight)
|
||||
|
||||
* Ground work for pluggable storage serializers (Tim Wojtulewicz, Corelight)
|
||||
|
||||
7.2.0-dev.520 | 2025-04-14 18:58:55 +0200
|
||||
|
||||
* Bump zeekjs to v0.17.0 (Arne Welzel, Corelight)
|
||||
|
|
|
|||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
7.2.0-dev.520
|
||||
7.2.0-dev.530
|
||||
|
|
|
|||
|
|
@ -25,9 +25,8 @@ export {
|
|||
## Returns: A record containing the status of the operation, and either an error
|
||||
## string on failure or a value on success. The value returned here will
|
||||
## be an ``opaque of BackendHandle``.
|
||||
global open_backend: function(btype: Storage::Backend,
|
||||
options: Storage::BackendOptions, key_type: any, val_type: any)
|
||||
: Storage::OperationResult;
|
||||
global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions,
|
||||
key_type: any, val_type: any): Storage::OperationResult;
|
||||
|
||||
## Closes an existing backend connection asynchronously. This method must be
|
||||
## called via a :zeek:see:`when` condition or an error will be returned.
|
||||
|
|
|
|||
|
|
@ -7,23 +7,26 @@ export {
|
|||
## :zeek:see:`Storage::Async::open_backend` and
|
||||
## :zeek:see:`Storage::Sync::open_backend`. Backend plugins can redef this record
|
||||
## to add relevant fields to it.
|
||||
type BackendOptions: record { };
|
||||
type BackendOptions: record {
|
||||
## The serializer used for converting Zeek data.
|
||||
serializer: Storage::Serializer &default=Storage::STORAGE_SERIALIZER_JSON;
|
||||
};
|
||||
|
||||
## Record for passing arguments to :zeek:see:`Storage::Async::put` and
|
||||
## :zeek:see:`Storage::Sync::put`.
|
||||
type PutArgs: record {
|
||||
# The key to store the value under.
|
||||
## The key to store the value under.
|
||||
key: any;
|
||||
|
||||
# The value to store associated with the key.
|
||||
## The value to store associated with the key.
|
||||
value: any;
|
||||
|
||||
# Indicates whether this value should overwrite an existing entry for the
|
||||
# key.
|
||||
## Indicates whether this value should overwrite an existing entry for the
|
||||
## key.
|
||||
overwrite: bool &default=T;
|
||||
|
||||
# An interval of time until the entry is automatically removed from the
|
||||
# backend.
|
||||
## An interval of time until the entry is automatically removed from the
|
||||
## backend.
|
||||
expire_time: interval &default=0sec;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,9 +23,8 @@ export {
|
|||
## Returns: A record containing the status of the operation, and either an error
|
||||
## string on failure or a value on success. The value returned here will
|
||||
## be an ``opaque of BackendHandle``.
|
||||
global open_backend: function(btype: Storage::Backend,
|
||||
options: Storage::BackendOptions, key_type: any, val_type: any)
|
||||
: Storage::OperationResult;
|
||||
global open_backend: function(btype: Storage::Backend, options: Storage::BackendOptions,
|
||||
key_type: any, val_type: any): Storage::OperationResult;
|
||||
|
||||
## Closes an existing backend connection.
|
||||
##
|
||||
|
|
|
|||
86
src/Val.cc
86
src/Val.cc
|
|
@ -891,7 +891,7 @@ unsigned int StringVal::ComputeFootprint(std::unordered_set<const Val*>* analyze
|
|||
return 1 /* this object */ + static_cast<unsigned int>(Len()) / sizeof(Val);
|
||||
}
|
||||
|
||||
static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, const TypePtr& t,
|
||||
static zeek::expected<ValPtr, std::string> BuildVal(const rapidjson::Value& j, const TypePtr& t,
|
||||
const FuncPtr& key_func) {
|
||||
auto mismatch_err = [t, &j]() {
|
||||
std::string json_type;
|
||||
|
|
@ -906,7 +906,8 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
default: json_type = "unknown";
|
||||
}
|
||||
|
||||
return util::fmt("cannot convert JSON type '%s' to Zeek type '%s'", json_type.c_str(), type_name(t->Tag()));
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("cannot convert JSON type '%s' to Zeek type '%s'", json_type.c_str(), type_name(t->Tag())));
|
||||
};
|
||||
|
||||
if ( j.IsNull() )
|
||||
|
|
@ -960,7 +961,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
parts.erase(std::remove_if(parts.begin(), parts.end(), [](auto x) { return x.empty(); }), parts.end());
|
||||
|
||||
if ( (parts.size() % 2) != 0 )
|
||||
return "wrong interval format, must be pairs of values with units";
|
||||
return zeek::unexpected<std::string>("wrong interval format, must be pairs of values with units");
|
||||
|
||||
double interval_secs = 0.0;
|
||||
for ( size_t i = 0; i < parts.size(); i += 2 ) {
|
||||
|
|
@ -980,7 +981,8 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
else if ( unit == "usec" || unit == "usecs" )
|
||||
interval_secs += (value * Microseconds);
|
||||
else
|
||||
return util::fmt("wrong interval format, invalid unit type %s", unit.data());
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("wrong interval format, invalid unit type %s", unit.data()));
|
||||
}
|
||||
|
||||
return make_intrusive<IntervalVal>(interval_secs, Seconds);
|
||||
|
|
@ -991,11 +993,10 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
|
||||
case TYPE_PORT: {
|
||||
if ( j.IsString() ) {
|
||||
int port = 0;
|
||||
if ( j.GetStringLength() > 0 && j.GetStringLength() < 10 ) {
|
||||
char* slash;
|
||||
errno = 0;
|
||||
port = strtol(j.GetString(), &slash, 10);
|
||||
auto port = strtol(j.GetString(), &slash, 10);
|
||||
if ( ! errno ) {
|
||||
++slash;
|
||||
if ( util::streq(slash, "tcp") )
|
||||
|
|
@ -1009,15 +1010,17 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
}
|
||||
}
|
||||
|
||||
return "wrong port format, string must be /[0-9]{1,5}\\/(tcp|udp|icmp|unknown)/";
|
||||
return zeek::unexpected<std::string>(
|
||||
"wrong port format, string must be /[0-9]{1,5}\\/(tcp|udp|icmp|unknown)/");
|
||||
}
|
||||
else if ( j.IsObject() ) {
|
||||
if ( ! j.HasMember("port") || ! j.HasMember("proto") )
|
||||
return "wrong port format, object must have 'port' and 'proto' members";
|
||||
return zeek::unexpected<std::string>(
|
||||
"wrong port format, object must have 'port' and 'proto' members");
|
||||
if ( ! j["port"].IsNumber() )
|
||||
return "wrong port format, port must be a number";
|
||||
return zeek::unexpected<std::string>("wrong port format, port must be a number");
|
||||
if ( ! j["proto"].IsString() )
|
||||
return "wrong port format, protocol must be a string";
|
||||
return zeek::unexpected<std::string>("wrong port format, protocol must be a string");
|
||||
|
||||
std::string proto{j["proto"].GetString()};
|
||||
|
||||
|
|
@ -1030,10 +1033,10 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
if ( proto == "unknown" )
|
||||
return val_mgr->Port(j["port"].GetInt(), TRANSPORT_UNKNOWN);
|
||||
|
||||
return "wrong port format, invalid protocol string";
|
||||
return zeek::unexpected<std::string>("wrong port format, invalid protocol string");
|
||||
}
|
||||
else
|
||||
return "wrong port format, must be string or object";
|
||||
return zeek::unexpected<std::string>("wrong port format, must be string or object");
|
||||
}
|
||||
|
||||
case TYPE_PATTERN: {
|
||||
|
|
@ -1055,7 +1058,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
|
||||
auto re = std::make_unique<RE_Matcher>(candidate.c_str());
|
||||
if ( ! re->Compile() )
|
||||
return "error compiling pattern";
|
||||
return zeek::unexpected<std::string>("error compiling pattern");
|
||||
|
||||
return make_intrusive<PatternVal>(re.release());
|
||||
}
|
||||
|
|
@ -1074,7 +1077,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
std::string_view subnet_sv(j.GetString(), j.GetStringLength());
|
||||
auto pos = subnet_sv.find('/');
|
||||
if ( pos == subnet_sv.npos )
|
||||
return util::fmt("invalid value for subnet: '%s'", j.GetString());
|
||||
return zeek::unexpected<std::string>(util::fmt("invalid value for subnet: '%s'", j.GetString()));
|
||||
|
||||
candidate = std::string(j.GetString(), pos);
|
||||
|
||||
|
|
@ -1082,7 +1085,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
char* end;
|
||||
width = strtol(subnet_sv.data() + pos + 1, &end, 10);
|
||||
if ( subnet_sv.data() + pos + 1 == end || errno )
|
||||
return util::fmt("invalid value for subnet: '%s'", j.GetString());
|
||||
return zeek::unexpected<std::string>(util::fmt("invalid value for subnet: '%s'", j.GetString()));
|
||||
}
|
||||
|
||||
if ( candidate.front() == '[' )
|
||||
|
|
@ -1104,7 +1107,8 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
auto intval = et->Lookup({j.GetString(), j.GetStringLength()});
|
||||
|
||||
if ( intval < 0 )
|
||||
return util::fmt("'%s' is not a valid enum for '%s'.", j.GetString(), et->GetName().c_str());
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("'%s' is not a valid enum for '%s'.", j.GetString(), et->GetName().c_str()));
|
||||
|
||||
return et->GetEnumVal(intval);
|
||||
}
|
||||
|
|
@ -1126,19 +1130,19 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
return mismatch_err();
|
||||
|
||||
for ( const auto& item : j.GetArray() ) {
|
||||
std::variant<ValPtr, std::string> v;
|
||||
zeek::expected<ValPtr, std::string> v;
|
||||
|
||||
if ( tl->GetTypes().size() == 1 )
|
||||
v = BuildVal(item, tl->GetPureType(), key_func);
|
||||
else
|
||||
v = BuildVal(item, tl, key_func);
|
||||
|
||||
if ( ! get_if<ValPtr>(&v) )
|
||||
if ( ! v )
|
||||
return v;
|
||||
if ( ! std::get<ValPtr>(v) )
|
||||
if ( v.value() == nullptr )
|
||||
continue;
|
||||
|
||||
tv->Assign(std::move(std::get<ValPtr>(v)), nullptr);
|
||||
tv->Assign(v.value(), nullptr);
|
||||
}
|
||||
|
||||
return tv;
|
||||
|
|
@ -1151,7 +1155,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
rapidjson::Document idxstr;
|
||||
idxstr.Parse(it->name.GetString(), it->name.GetStringLength());
|
||||
|
||||
std::variant<ValPtr, std::string> idx;
|
||||
zeek::expected<ValPtr, std::string> idx;
|
||||
|
||||
if ( tl->GetTypes().size() > 1 )
|
||||
idx = BuildVal(idxstr, tl, key_func);
|
||||
|
|
@ -1163,19 +1167,19 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
// Parse the string's content, not the full JSON string.
|
||||
idx = BuildVal(idxstr, tl->GetPureType(), key_func);
|
||||
|
||||
if ( ! get_if<ValPtr>(&idx) )
|
||||
if ( ! idx )
|
||||
return idx;
|
||||
if ( ! std::get<ValPtr>(idx) )
|
||||
if ( idx.value() == nullptr )
|
||||
continue;
|
||||
|
||||
auto v = BuildVal(it->value, tt->Yield(), key_func);
|
||||
|
||||
if ( ! get_if<ValPtr>(&v) )
|
||||
if ( ! v )
|
||||
return v;
|
||||
if ( ! std::get<ValPtr>(v) )
|
||||
if ( v.value() == nullptr )
|
||||
continue;
|
||||
|
||||
tv->Assign(std::move(std::get<ValPtr>(idx)), std::move(std::get<ValPtr>(v)));
|
||||
tv->Assign(idx.value(), v.value());
|
||||
}
|
||||
|
||||
return tv;
|
||||
|
|
@ -1202,7 +1206,7 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
}
|
||||
|
||||
if ( ! result )
|
||||
return "key function error";
|
||||
return zeek::unexpected<std::string>("key function error");
|
||||
|
||||
normalized_keys[result->AsStringVal()->CheckString()] = &it->value;
|
||||
}
|
||||
|
|
@ -1226,17 +1230,18 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
if ( ! td_i->GetAttr(detail::ATTR_OPTIONAL) && ! td_i->GetAttr(detail::ATTR_DEFAULT) )
|
||||
// jval being set means it is a null JSON value else
|
||||
// it wasn't even there.
|
||||
return util::fmt("required field %s$%s is %s in JSON", t->GetName().c_str(), td_i->id,
|
||||
jval ? "null" : "missing");
|
||||
return zeek::unexpected<std::string>(util::fmt("required field %s$%s is %s in JSON",
|
||||
t->GetName().c_str(), td_i->id,
|
||||
jval ? "null" : "missing"));
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
auto v = BuildVal(*jval, td_i->type, key_func);
|
||||
if ( ! get_if<ValPtr>(&v) )
|
||||
if ( ! v )
|
||||
return v;
|
||||
|
||||
rv->Assign(i, std::move(std::get<ValPtr>(v)));
|
||||
rv->Assign(i, v.value());
|
||||
}
|
||||
|
||||
return rv;
|
||||
|
|
@ -1249,16 +1254,16 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
auto lt = t->AsTypeList();
|
||||
|
||||
if ( j.GetArray().Size() < lt->GetTypes().size() )
|
||||
return "index type doesn't match";
|
||||
return zeek::unexpected<std::string>("index type doesn't match");
|
||||
|
||||
auto lv = make_intrusive<ListVal>(TYPE_ANY);
|
||||
|
||||
for ( size_t i = 0; i < lt->GetTypes().size(); i++ ) {
|
||||
auto v = BuildVal(j.GetArray()[i], lt->GetTypes()[i], key_func);
|
||||
if ( ! get_if<ValPtr>(&v) )
|
||||
if ( ! v )
|
||||
return v;
|
||||
|
||||
lv->Append(std::move(std::get<ValPtr>(v)));
|
||||
lv->Append(v.value());
|
||||
}
|
||||
|
||||
return lv;
|
||||
|
|
@ -1272,29 +1277,30 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
auto vv = make_intrusive<VectorVal>(IntrusivePtr{NewRef{}, vt});
|
||||
for ( const auto& item : j.GetArray() ) {
|
||||
auto v = BuildVal(item, vt->Yield(), key_func);
|
||||
if ( ! get_if<ValPtr>(&v) )
|
||||
if ( ! v )
|
||||
return v;
|
||||
|
||||
if ( ! std::get<ValPtr>(v) )
|
||||
if ( v.value() == nullptr )
|
||||
continue;
|
||||
|
||||
vv->Assign(vv->Size(), std::move(std::get<ValPtr>(v)));
|
||||
vv->Assign(vv->Size(), v.value());
|
||||
}
|
||||
|
||||
return vv;
|
||||
}
|
||||
|
||||
default: return util::fmt("type '%s' unsupported", type_name(t->Tag()));
|
||||
default: return zeek::unexpected<std::string>(util::fmt("type '%s' unsupported", type_name(t->Tag())));
|
||||
}
|
||||
}
|
||||
|
||||
std::variant<ValPtr, std::string> detail::ValFromJSON(std::string_view json_str, const TypePtr& t,
|
||||
zeek::expected<ValPtr, std::string> detail::ValFromJSON(std::string_view json_str, const TypePtr& t,
|
||||
const FuncPtr& key_func) {
|
||||
rapidjson::Document doc;
|
||||
rapidjson::ParseResult ok = doc.Parse(json_str.data(), json_str.length());
|
||||
|
||||
if ( ! ok )
|
||||
return util::fmt("JSON parse error: %s Offset: %lu", rapidjson::GetParseError_En(ok.Code()), ok.Offset());
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("JSON parse error: %s Offset: %lu", rapidjson::GetParseError_En(ok.Code()), ok.Offset()));
|
||||
|
||||
return BuildVal(doc, t, key_func);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,9 +4,7 @@
|
|||
|
||||
#include <sys/types.h> // for u_char
|
||||
#include <array>
|
||||
#include <list>
|
||||
#include <unordered_map>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include "zeek/IntrusivePtr.h"
|
||||
|
|
@ -1774,7 +1772,7 @@ namespace detail {
|
|||
//
|
||||
// The *key_func* parameter is a Zeek script function called for every JSON key
|
||||
// for normalization. If Func::nil is passed, no normalization happens.
|
||||
extern std::variant<ValPtr, std::string> ValFromJSON(std::string_view json_str, const TypePtr& t,
|
||||
extern zeek::expected<ValPtr, std::string> ValFromJSON(std::string_view json_str, const TypePtr& t,
|
||||
const FuncPtr& key_func);
|
||||
|
||||
// If the given vector is an empty vector-of-any ("unspecified"),
|
||||
|
|
|
|||
|
|
@ -400,8 +400,7 @@ private:
|
|||
|
||||
// This should never be reached, broker itself doesn't call this and overrides
|
||||
// the generic DoPublishEvent() method that would call this.
|
||||
bool DoPublishEvent(const std::string& topic, const std::string& format,
|
||||
const cluster::detail::byte_buffer& buf) override {
|
||||
bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) override {
|
||||
throw std::logic_error("not implemented");
|
||||
}
|
||||
|
||||
|
|
@ -416,7 +415,7 @@ private:
|
|||
}
|
||||
|
||||
bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format,
|
||||
cluster::detail::byte_buffer& buf) override {
|
||||
byte_buffer& buf) override {
|
||||
// Not implemented by broker.
|
||||
throw std::logic_error("not implemented");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ std::optional<detail::Event> Backend::MakeClusterEvent(FuncValPtr handler, ArgsS
|
|||
|
||||
// Default implementation doing the serialization.
|
||||
bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& event) {
|
||||
cluster::detail::byte_buffer buf;
|
||||
byte_buffer buf;
|
||||
|
||||
if ( ! event_serializer->SerializeEvent(buf, event) )
|
||||
return false;
|
||||
|
|
@ -111,7 +111,7 @@ bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& e
|
|||
// Default implementation doing log record serialization.
|
||||
bool Backend::DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header,
|
||||
zeek::Span<zeek::logging::detail::LogRecord> records) {
|
||||
cluster::detail::byte_buffer buf;
|
||||
byte_buffer buf;
|
||||
|
||||
if ( ! log_serializer->SerializeLogWrite(buf, header, records) )
|
||||
return false;
|
||||
|
|
@ -123,8 +123,7 @@ void Backend::EnqueueEvent(EventHandlerPtr h, zeek::Args args) {
|
|||
event_handling_strategy->EnqueueLocalEvent(h, std::move(args));
|
||||
}
|
||||
|
||||
bool Backend::ProcessEventMessage(std::string_view topic, std::string_view format,
|
||||
const detail::byte_buffer_span payload) {
|
||||
bool Backend::ProcessEventMessage(std::string_view topic, std::string_view format, const byte_buffer_span payload) {
|
||||
if ( format != event_serializer->Name() ) {
|
||||
zeek::reporter->Error("ProcessEventMessage: Wrong format: %s vs %s", std::string{format}.c_str(),
|
||||
event_serializer->Name().c_str());
|
||||
|
|
@ -143,7 +142,7 @@ bool Backend::ProcessEventMessage(std::string_view topic, std::string_view forma
|
|||
return event_handling_strategy->HandleRemoteEvent(topic, std::move(*r));
|
||||
}
|
||||
|
||||
bool Backend::ProcessLogMessage(std::string_view format, detail::byte_buffer_span payload) {
|
||||
bool Backend::ProcessLogMessage(std::string_view format, byte_buffer_span payload) {
|
||||
// We could also dynamically lookup the right de-serializer, but
|
||||
// for now assume we just receive what is configured.
|
||||
if ( format != log_serializer->Name() ) {
|
||||
|
|
@ -162,7 +161,7 @@ bool Backend::ProcessLogMessage(std::string_view format, detail::byte_buffer_spa
|
|||
return zeek::log_mgr->WriteBatchFromRemote(result->header, std::move(result->records));
|
||||
}
|
||||
|
||||
bool ThreadedBackend::ProcessBackendMessage(int tag, detail::byte_buffer_span payload) {
|
||||
bool ThreadedBackend::ProcessBackendMessage(int tag, byte_buffer_span payload) {
|
||||
return DoProcessBackendMessage(tag, payload);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -311,12 +311,12 @@ protected:
|
|||
/**
|
||||
* Process an incoming event message.
|
||||
*/
|
||||
bool ProcessEventMessage(std::string_view topic, std::string_view format, detail::byte_buffer_span payload);
|
||||
bool ProcessEventMessage(std::string_view topic, std::string_view format, byte_buffer_span payload);
|
||||
|
||||
/**
|
||||
* Process an incoming log message.
|
||||
*/
|
||||
bool ProcessLogMessage(std::string_view format, detail::byte_buffer_span payload);
|
||||
bool ProcessLogMessage(std::string_view format, byte_buffer_span payload);
|
||||
|
||||
private:
|
||||
/**
|
||||
|
|
@ -350,7 +350,7 @@ private:
|
|||
/**
|
||||
* Publish a cluster::detail::Event to the given topic.
|
||||
*
|
||||
* The default implementation serializes to a detail::byte_buffer and
|
||||
* The default implementation serializes to a byte_buffer and
|
||||
* calls DoPublishEvent() with the resulting buffer.
|
||||
*
|
||||
* This hook method only exists for the existing Broker implementation that
|
||||
|
|
@ -373,8 +373,7 @@ private:
|
|||
* @param buf the serialized Event.
|
||||
* @return true if the message has been published successfully.
|
||||
*/
|
||||
virtual bool DoPublishEvent(const std::string& topic, const std::string& format,
|
||||
const detail::byte_buffer& buf) = 0;
|
||||
virtual bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) = 0;
|
||||
|
||||
/**
|
||||
* Register interest in messages that use a certain topic prefix.
|
||||
|
|
@ -405,7 +404,7 @@ private:
|
|||
/**
|
||||
* Serialize a log batch, then forward it to DoPublishLogWrites() below.
|
||||
|
||||
* The default implementation serializes to a detail::byte_buffer and
|
||||
* The default implementation serializes to a byte_buffer and
|
||||
* calls DoPublishLogWrites() with the resulting buffer.
|
||||
*
|
||||
* This hook method only exists for the existing Broker implementation that
|
||||
|
|
@ -440,7 +439,7 @@ private:
|
|||
* @return true if the message has been published successfully.
|
||||
*/
|
||||
virtual bool DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, const std::string& format,
|
||||
detail::byte_buffer& buf) = 0;
|
||||
byte_buffer& buf) = 0;
|
||||
|
||||
std::string name;
|
||||
zeek::Tag tag;
|
||||
|
|
@ -471,7 +470,7 @@ private:
|
|||
struct EventMessage {
|
||||
std::string topic;
|
||||
std::string format;
|
||||
detail::byte_buffer payload;
|
||||
byte_buffer payload;
|
||||
|
||||
auto payload_span() const { return Span(payload.data(), payload.size()); };
|
||||
};
|
||||
|
|
@ -481,7 +480,7 @@ struct EventMessage {
|
|||
*/
|
||||
struct LogMessage {
|
||||
std::string format;
|
||||
detail::byte_buffer payload;
|
||||
byte_buffer payload;
|
||||
|
||||
auto payload_span() const { return Span(payload.data(), payload.size()); };
|
||||
};
|
||||
|
|
@ -494,7 +493,7 @@ struct LogMessage {
|
|||
*/
|
||||
struct BackendMessage {
|
||||
int tag;
|
||||
detail::byte_buffer payload;
|
||||
byte_buffer payload;
|
||||
|
||||
auto payload_span() const { return Span(payload.data(), payload.size()); };
|
||||
};
|
||||
|
|
@ -546,13 +545,13 @@ private:
|
|||
/**
|
||||
* Process a backend specific message queued as BackendMessage.
|
||||
*/
|
||||
bool ProcessBackendMessage(int tag, detail::byte_buffer_span payload);
|
||||
bool ProcessBackendMessage(int tag, byte_buffer_span payload);
|
||||
|
||||
/**
|
||||
* If a cluster backend produces messages of type BackendMessage,
|
||||
* this method will be invoked by the main thread to process it.
|
||||
*/
|
||||
virtual bool DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) { return false; };
|
||||
virtual bool DoProcessBackendMessage(int tag, byte_buffer_span payload) { return false; };
|
||||
|
||||
/**
|
||||
* Hook method for OnLooProcess.
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@
|
|||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "zeek/Span.h"
|
||||
#include "zeek/logging/Types.h"
|
||||
|
|
@ -15,13 +14,8 @@ namespace zeek::cluster {
|
|||
|
||||
namespace detail {
|
||||
class Event;
|
||||
|
||||
using byte_buffer = std::vector<std::byte>;
|
||||
using byte_buffer_span = Span<const std::byte>;
|
||||
|
||||
} // namespace detail
|
||||
|
||||
|
||||
/**
|
||||
* This class handles encoding of events into byte buffers and back.
|
||||
*
|
||||
|
|
@ -40,7 +34,7 @@ public:
|
|||
*
|
||||
* @returns True on success, false in exceptional cases (e.g. unsupported serialization).
|
||||
*/
|
||||
virtual bool SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) = 0;
|
||||
virtual bool SerializeEvent(byte_buffer& buf, const detail::Event& event) = 0;
|
||||
|
||||
/**
|
||||
* Unserialize an event from a given byte buffer.
|
||||
|
|
@ -49,7 +43,7 @@ public:
|
|||
*
|
||||
* @returns The event, or std::nullopt on error.
|
||||
*/
|
||||
virtual std::optional<cluster::detail::Event> UnserializeEvent(detail::byte_buffer_span buf) = 0;
|
||||
virtual std::optional<cluster::detail::Event> UnserializeEvent(byte_buffer_span buf) = 0;
|
||||
|
||||
/**
|
||||
* @returns The name of this event serializer instance.
|
||||
|
|
@ -85,7 +79,7 @@ public:
|
|||
* @param header The log batch header.
|
||||
* @param records The actual log writes.
|
||||
*/
|
||||
virtual bool SerializeLogWrite(detail::byte_buffer& buf, const logging::detail::LogWriteHeader& header,
|
||||
virtual bool SerializeLogWrite(byte_buffer& buf, const logging::detail::LogWriteHeader& header,
|
||||
zeek::Span<logging::detail::LogRecord> records) = 0;
|
||||
|
||||
/**
|
||||
|
|
@ -93,7 +87,7 @@ public:
|
|||
*
|
||||
* @param buf The span representing received log writes.
|
||||
*/
|
||||
virtual std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(detail::byte_buffer_span buf) = 0;
|
||||
virtual std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(byte_buffer_span buf) = 0;
|
||||
|
||||
/**
|
||||
* @returns The name of this log serializer instance.
|
||||
|
|
|
|||
|
|
@ -253,8 +253,7 @@ bool ZeroMQBackend::SpawnZmqProxyThread() {
|
|||
return proxy_thread->Start();
|
||||
}
|
||||
|
||||
bool ZeroMQBackend::DoPublishEvent(const std::string& topic, const std::string& format,
|
||||
const cluster::detail::byte_buffer& buf) {
|
||||
bool ZeroMQBackend::DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) {
|
||||
// Publishing an event happens as a multipart message with 4 parts:
|
||||
//
|
||||
// * The topic to publish to - this is required by XPUB/XSUB
|
||||
|
|
@ -336,7 +335,7 @@ bool ZeroMQBackend::DoUnsubscribe(const std::string& topic_prefix) {
|
|||
}
|
||||
|
||||
bool ZeroMQBackend::DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format,
|
||||
cluster::detail::byte_buffer& buf) {
|
||||
byte_buffer& buf) {
|
||||
ZEROMQ_DEBUG("Publishing %zu bytes of log writes (path %s)", buf.size(), header.path.c_str());
|
||||
static std::string message_type = "log-write";
|
||||
|
||||
|
|
@ -405,7 +404,7 @@ void ZeroMQBackend::Run() {
|
|||
continue;
|
||||
}
|
||||
|
||||
detail::byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()};
|
||||
byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()};
|
||||
LogMessage lm{.format = std::string(msg[2].data<const char>(), msg[2].size()),
|
||||
.payload = std::move(payload)};
|
||||
|
||||
|
|
@ -487,7 +486,7 @@ void ZeroMQBackend::Run() {
|
|||
QueueMessage qm;
|
||||
auto* start = msg[0].data<std::byte>() + 1;
|
||||
auto* end = msg[0].data<std::byte>() + msg[0].size();
|
||||
detail::byte_buffer topic(start, end);
|
||||
byte_buffer topic(start, end);
|
||||
if ( first == 1 ) {
|
||||
qm = BackendMessage{1, std::move(topic)};
|
||||
}
|
||||
|
|
@ -516,7 +515,7 @@ void ZeroMQBackend::Run() {
|
|||
if ( sender == NodeId() )
|
||||
continue;
|
||||
|
||||
detail::byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()};
|
||||
byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()};
|
||||
EventMessage em{.topic = std::string(msg[0].data<const char>(), msg[0].size()),
|
||||
.format = std::string(msg[2].data<const char>(), msg[2].size()),
|
||||
.payload = std::move(payload)};
|
||||
|
|
@ -644,7 +643,7 @@ void ZeroMQBackend::Run() {
|
|||
}
|
||||
}
|
||||
|
||||
bool ZeroMQBackend::DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) {
|
||||
bool ZeroMQBackend::DoProcessBackendMessage(int tag, byte_buffer_span payload) {
|
||||
if ( tag == 0 || tag == 1 ) {
|
||||
std::string topic{reinterpret_cast<const char*>(payload.data()), payload.size()};
|
||||
zeek::EventHandlerPtr eh;
|
||||
|
|
|
|||
|
|
@ -60,17 +60,16 @@ private:
|
|||
|
||||
void DoTerminate() override;
|
||||
|
||||
bool DoPublishEvent(const std::string& topic, const std::string& format,
|
||||
const cluster::detail::byte_buffer& buf) override;
|
||||
bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) override;
|
||||
|
||||
bool DoSubscribe(const std::string& topic_prefix, SubscribeCallback cb) override;
|
||||
|
||||
bool DoUnsubscribe(const std::string& topic_prefix) override;
|
||||
|
||||
bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format,
|
||||
cluster::detail::byte_buffer& buf) override;
|
||||
byte_buffer& buf) override;
|
||||
|
||||
bool DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) override;
|
||||
bool DoProcessBackendMessage(int tag, byte_buffer_span payload) override;
|
||||
|
||||
// Script level variables.
|
||||
std::string connect_xsub_endpoint;
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ bool detail::BinarySerializationFormatLogSerializer::SerializeLogWrite(byte_buff
|
|||
}
|
||||
|
||||
std::optional<zeek::logging::detail::LogWriteBatch> detail::BinarySerializationFormatLogSerializer::UnserializeLogWrite(
|
||||
detail::byte_buffer_span buf) {
|
||||
byte_buffer_span buf) {
|
||||
zeek::detail::BinarySerializationFormat fmt;
|
||||
fmt.StartRead(reinterpret_cast<const char*>(buf.data()), buf.size());
|
||||
|
||||
|
|
@ -145,7 +145,7 @@ std::optional<zeek::logging::detail::LogWriteBatch> detail::BinarySerializationF
|
|||
TEST_SUITE_BEGIN("cluster serializer binary-serialization-format");
|
||||
|
||||
TEST_CASE("roundtrip") {
|
||||
detail::byte_buffer buf;
|
||||
zeek::byte_buffer buf;
|
||||
detail::BinarySerializationFormatLogSerializer serializer;
|
||||
|
||||
static const auto& stream_id_type = zeek::id::find_type<zeek::EnumType>("Log::ID");
|
||||
|
|
@ -161,7 +161,7 @@ TEST_CASE("roundtrip") {
|
|||
0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00,
|
||||
0x00, 0x00, 0x16, 0x01, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
|
||||
std::byte* p = reinterpret_cast<std::byte*>(&expected_bytes[0]);
|
||||
detail::byte_buffer expected{p, p + sizeof(expected_bytes)};
|
||||
zeek::byte_buffer expected{p, p + sizeof(expected_bytes)};
|
||||
|
||||
auto s = stream_id_type->Lookup("Log::UNKNOWN");
|
||||
REQUIRE_GE(s, 0);
|
||||
|
|
|
|||
|
|
@ -13,10 +13,10 @@ class BinarySerializationFormatLogSerializer : public cluster::LogSerializer {
|
|||
public:
|
||||
BinarySerializationFormatLogSerializer() : LogSerializer("zeek-bin-serializer") {}
|
||||
|
||||
bool SerializeLogWrite(cluster::detail::byte_buffer& buf, const logging::detail::LogWriteHeader& header,
|
||||
bool SerializeLogWrite(byte_buffer& buf, const logging::detail::LogWriteHeader& header,
|
||||
zeek::Span<logging::detail::LogRecord> records) override;
|
||||
|
||||
std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(detail::byte_buffer_span buf) override;
|
||||
std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(byte_buffer_span buf) override;
|
||||
};
|
||||
|
||||
} // namespace zeek::cluster::detail
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ std::optional<detail::Event> detail::to_zeek_event(const broker::zeek::Event& ev
|
|||
return detail::Event{handler, std::move(vl), ts};
|
||||
}
|
||||
|
||||
bool detail::BrokerBinV1_Serializer::SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) {
|
||||
bool detail::BrokerBinV1_Serializer::SerializeEvent(byte_buffer& buf, const detail::Event& event) {
|
||||
auto ev = to_broker_event(event);
|
||||
if ( ! ev )
|
||||
return false;
|
||||
|
|
@ -117,7 +117,7 @@ bool detail::BrokerBinV1_Serializer::SerializeEvent(detail::byte_buffer& buf, co
|
|||
return true;
|
||||
}
|
||||
|
||||
std::optional<detail::Event> detail::BrokerBinV1_Serializer::UnserializeEvent(detail::byte_buffer_span buf) {
|
||||
std::optional<detail::Event> detail::BrokerBinV1_Serializer::UnserializeEvent(byte_buffer_span buf) {
|
||||
auto r = broker::data_envelope::deserialize(broker::endpoint_id::nil(), broker::endpoint_id::nil(), 0, "",
|
||||
buf.data(), buf.size());
|
||||
if ( ! r )
|
||||
|
|
@ -152,7 +152,7 @@ bool detail::BrokerJsonV1_Serializer::SerializeEvent(byte_buffer& buf, const det
|
|||
return true;
|
||||
}
|
||||
|
||||
std::optional<detail::Event> detail::BrokerJsonV1_Serializer::UnserializeEvent(detail::byte_buffer_span buf) {
|
||||
std::optional<detail::Event> detail::BrokerJsonV1_Serializer::UnserializeEvent(byte_buffer_span buf) {
|
||||
broker::variant res;
|
||||
auto err =
|
||||
broker::format::json::v1::decode(std::string_view{reinterpret_cast<const char*>(buf.data()), buf.size()}, res);
|
||||
|
|
@ -173,7 +173,7 @@ TEST_SUITE_BEGIN("cluster serializer broker");
|
|||
TEST_CASE("roundtrip") {
|
||||
auto* handler = zeek::event_registry->Lookup("Supervisor::node_status");
|
||||
detail::Event e{handler, zeek::Args{zeek::make_intrusive<zeek::StringVal>("TEST"), zeek::val_mgr->Count(42)}};
|
||||
detail::byte_buffer buf;
|
||||
zeek::byte_buffer buf;
|
||||
|
||||
SUBCASE("json") {
|
||||
detail::BrokerJsonV1_Serializer serializer;
|
||||
|
|
@ -201,7 +201,7 @@ TEST_CASE("roundtrip") {
|
|||
0x01, 0x0e, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x09,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
|
||||
std::byte* p = reinterpret_cast<std::byte*>(&expected_bytes[0]);
|
||||
detail::byte_buffer expected{p, p + sizeof(expected_bytes)};
|
||||
zeek::byte_buffer expected{p, p + sizeof(expected_bytes)};
|
||||
|
||||
serializer.SerializeEvent(buf, e);
|
||||
|
||||
|
|
|
|||
|
|
@ -34,9 +34,9 @@ class BrokerBinV1_Serializer : public EventSerializer {
|
|||
public:
|
||||
BrokerBinV1_Serializer() : EventSerializer("broker-bin-v1") {}
|
||||
|
||||
bool SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) override;
|
||||
bool SerializeEvent(byte_buffer& buf, const detail::Event& event) override;
|
||||
|
||||
std::optional<detail::Event> UnserializeEvent(detail::byte_buffer_span buf) override;
|
||||
std::optional<detail::Event> UnserializeEvent(byte_buffer_span buf) override;
|
||||
};
|
||||
|
||||
// Implementation of the EventSerializer that uses the existing broker::detail::val_to_data()
|
||||
|
|
@ -45,9 +45,9 @@ class BrokerJsonV1_Serializer : public EventSerializer {
|
|||
public:
|
||||
BrokerJsonV1_Serializer() : EventSerializer("broker-json-v1") {}
|
||||
|
||||
bool SerializeEvent(zeek::cluster::detail::byte_buffer& buf, const detail::Event& event) override;
|
||||
bool SerializeEvent(byte_buffer& buf, const detail::Event& event) override;
|
||||
|
||||
std::optional<detail::Event> UnserializeEvent(detail::byte_buffer_span buf) override;
|
||||
std::optional<detail::Event> UnserializeEvent(byte_buffer_span buf) override;
|
||||
};
|
||||
|
||||
} // namespace zeek::cluster::detail
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@ void Component::Describe(ODesc* d) const {
|
|||
|
||||
case component::STORAGE_BACKEND: d->Add("Storage Backend"); break;
|
||||
|
||||
case component::STORAGE_SERIALIZER: d->Add("Storage Serializer"); break;
|
||||
|
||||
default:
|
||||
reporter->InternalWarning("unknown component type in plugin::Component::Describe");
|
||||
d->Add("<unknown component type>");
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ enum Type {
|
|||
EVENT_SERIALIZER, /// A serializer for events, used by cluster backends.
|
||||
LOG_SERIALIZER, /// A serializer for log batches, used by cluster backends.
|
||||
STORAGE_BACKEND, /// A backend for the storage framework.
|
||||
STORAGE_SERIALIZER, /// A serializer for the storage framework.
|
||||
};
|
||||
|
||||
} // namespace component
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ void OpenResultCallback::Complete(OperationResult res) {
|
|||
}
|
||||
|
||||
Backend::Backend(uint8_t modes, std::string_view tag_name) : modes(modes) {
|
||||
tag = storage_mgr->GetComponentTag(std::string{tag_name});
|
||||
tag = storage_mgr->BackendMgr().GetComponentTag(std::string{tag_name});
|
||||
tag_str = zeek::obj_desc_short(tag.AsVal().get());
|
||||
}
|
||||
|
||||
|
|
@ -75,6 +75,15 @@ OperationResult Backend::Open(OpenResultCallback* cb, RecordValPtr options, Type
|
|||
val_type = std::move(vt);
|
||||
backend_options = options;
|
||||
|
||||
auto stype = options->GetField<EnumVal>("serializer");
|
||||
zeek::Tag stag{stype};
|
||||
|
||||
auto s = storage_mgr->InstantiateSerializer(stag);
|
||||
if ( ! s )
|
||||
return {ReturnCode::INITIALIZATION_FAILED, s.error()};
|
||||
|
||||
serializer = std::move(s.value());
|
||||
|
||||
auto ret = DoOpen(cb, std::move(options));
|
||||
if ( ! ret.value )
|
||||
ret.value = cb->Backend();
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
#include "zeek/OpaqueVal.h"
|
||||
#include "zeek/Tag.h"
|
||||
#include "zeek/Val.h"
|
||||
#include "zeek/storage/Serializer.h"
|
||||
|
||||
namespace zeek::detail::trigger {
|
||||
class Trigger;
|
||||
|
|
@ -58,7 +59,7 @@ struct OperationResult {
|
|||
class ResultCallback {
|
||||
public:
|
||||
ResultCallback() = default;
|
||||
ResultCallback(detail::trigger::TriggerPtr trigger, const void* assoc);
|
||||
ResultCallback(zeek::detail::trigger::TriggerPtr trigger, const void* assoc);
|
||||
virtual ~ResultCallback() = default;
|
||||
|
||||
/**
|
||||
|
|
@ -239,6 +240,7 @@ protected:
|
|||
|
||||
zeek::Tag tag;
|
||||
std::string tag_str;
|
||||
std::unique_ptr<Serializer> serializer;
|
||||
|
||||
private:
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -11,3 +11,4 @@ zeek_add_subdir_library(
|
|||
storage-sync.bif)
|
||||
|
||||
add_subdirectory(backend)
|
||||
add_subdirectory(serializer)
|
||||
|
|
|
|||
|
|
@ -7,19 +7,34 @@
|
|||
|
||||
namespace zeek::storage {
|
||||
|
||||
Component::Component(const std::string& name, factory_callback arg_factory)
|
||||
: plugin::Component(plugin::component::STORAGE_BACKEND, name, 0, storage_mgr->GetTagType()) {
|
||||
BackendComponent::BackendComponent(const std::string& name, factory_callback arg_factory)
|
||||
: plugin::Component(plugin::component::STORAGE_BACKEND, name, 0, storage_mgr->BackendMgr().GetTagType()) {
|
||||
factory = arg_factory;
|
||||
}
|
||||
|
||||
void Component::Initialize() {
|
||||
void BackendComponent::Initialize() {
|
||||
InitializeTag();
|
||||
storage_mgr->RegisterComponent(this);
|
||||
storage_mgr->BackendMgr().RegisterComponent(this, "STORAGE_BACKEND_");
|
||||
}
|
||||
|
||||
void Component::DoDescribe(ODesc* d) const {
|
||||
void BackendComponent::DoDescribe(ODesc* d) const {
|
||||
d->Add("Storage::STORAGE_BACKEND_");
|
||||
d->Add(CanonicalName());
|
||||
}
|
||||
|
||||
SerializerComponent::SerializerComponent(const std::string& name, factory_callback arg_factory)
|
||||
: plugin::Component(plugin::component::STORAGE_SERIALIZER, name, 0, storage_mgr->SerializerMgr().GetTagType()) {
|
||||
factory = arg_factory;
|
||||
}
|
||||
|
||||
void SerializerComponent::Initialize() {
|
||||
InitializeTag();
|
||||
storage_mgr->SerializerMgr().RegisterComponent(this, "STORAGE_SERIALIZER_");
|
||||
}
|
||||
|
||||
void SerializerComponent::DoDescribe(ODesc* d) const {
|
||||
d->Add("Storage::STORAGE_SERIALIZER_");
|
||||
d->Add(CanonicalName());
|
||||
}
|
||||
|
||||
} // namespace zeek::storage
|
||||
|
|
|
|||
|
|
@ -7,11 +7,12 @@
|
|||
namespace zeek::storage {
|
||||
|
||||
class Backend;
|
||||
class Serializer;
|
||||
|
||||
/**
|
||||
* Component description for plugins providing storage backends.
|
||||
*/
|
||||
class Component : public plugin::Component {
|
||||
class BackendComponent : public plugin::Component {
|
||||
public:
|
||||
using factory_callback = IntrusivePtr<Backend> (*)();
|
||||
|
||||
|
|
@ -27,12 +28,60 @@ public:
|
|||
* method inside the class that just allocates and returns a new
|
||||
* instance.
|
||||
*/
|
||||
Component(const std::string& name, factory_callback factory);
|
||||
BackendComponent(const std::string& name, factory_callback factory);
|
||||
|
||||
/**
|
||||
* Destructor.
|
||||
*/
|
||||
~Component() override = default;
|
||||
~BackendComponent() override = default;
|
||||
|
||||
/**
|
||||
* Initialization function. This function has to be called before any
|
||||
* plugin component functionality is used; it is used to add the
|
||||
* plugin component to the list of components and to initialize tags
|
||||
*/
|
||||
void Initialize() override;
|
||||
|
||||
/**
|
||||
* Returns the backend's factory function.
|
||||
*/
|
||||
factory_callback Factory() const { return factory; }
|
||||
|
||||
protected:
|
||||
/**
|
||||
* Overridden from plugin::Component.
|
||||
*/
|
||||
void DoDescribe(ODesc* d) const override;
|
||||
|
||||
private:
|
||||
factory_callback factory;
|
||||
};
|
||||
|
||||
/**
|
||||
* Component description for plugins providing serialization for storage data..
|
||||
*/
|
||||
class SerializerComponent : public plugin::Component {
|
||||
public:
|
||||
using factory_callback = std::unique_ptr<Serializer> (*)();
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
*
|
||||
* @param name The name of the provided backend. This name is used
|
||||
* across the system to identify the backend.
|
||||
*
|
||||
* @param factory A factory function to instantiate instances of the
|
||||
* backend's class, which must be derived directly or indirectly from
|
||||
* storage::Backend. This is typically a static \c Instantiate()
|
||||
* method inside the class that just allocates and returns a new
|
||||
* instance.
|
||||
*/
|
||||
SerializerComponent(const std::string& name, factory_callback factory);
|
||||
|
||||
/**
|
||||
* Destructor.
|
||||
*/
|
||||
~SerializerComponent() override = default;
|
||||
|
||||
/**
|
||||
* Initialization function. This function has to be called before any
|
||||
|
|
|
|||
|
|
@ -28,7 +28,9 @@ void detail::ExpirationTimer::Dispatch(double t, bool is_expire) {
|
|||
storage_mgr->StartExpirationTimer();
|
||||
}
|
||||
|
||||
Manager::Manager() : plugin::ComponentManager<storage::Component>("Storage", "Backend") {}
|
||||
Manager::Manager()
|
||||
: backend_mgr(plugin::ComponentManager<storage::BackendComponent>("Storage", "Backend")),
|
||||
serializer_mgr(plugin::ComponentManager<storage::SerializerComponent>("Storage", "Serializer")) {}
|
||||
|
||||
Manager::~Manager() {
|
||||
// TODO: should we shut down any existing backends? force-poll until all of their existing
|
||||
|
|
@ -48,24 +50,40 @@ void Manager::InitPostScript() {
|
|||
StartExpirationTimer();
|
||||
}
|
||||
|
||||
zeek::expected<BackendPtr, std::string> Manager::Instantiate(const Tag& type) {
|
||||
Component* c = Lookup(type);
|
||||
if ( ! c ) {
|
||||
zeek::expected<BackendPtr, std::string> Manager::InstantiateBackend(const Tag& type) {
|
||||
BackendComponent* c = backend_mgr.Lookup(type);
|
||||
if ( ! c )
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("Request to open unknown backend (%d:%d)", type.Type(), type.Subtype()));
|
||||
}
|
||||
util::fmt("Request to instantiate unknown backend type (%d:%d)", type.Type(), type.Subtype()));
|
||||
|
||||
if ( ! c->Factory() ) {
|
||||
if ( ! c->Factory() )
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("Factory invalid for backend %s", GetComponentName(type).c_str()));
|
||||
}
|
||||
util::fmt("Factory invalid for backend %s", backend_mgr.GetComponentName(type).c_str()));
|
||||
|
||||
BackendPtr bp = c->Factory()();
|
||||
auto bp = c->Factory()();
|
||||
|
||||
if ( ! bp ) {
|
||||
if ( ! bp )
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("Failed to instantiate backend %s", GetComponentName(type).c_str()));
|
||||
}
|
||||
util::fmt("Failed to instantiate backend %s", backend_mgr.GetComponentName(type).c_str()));
|
||||
|
||||
return bp;
|
||||
}
|
||||
|
||||
zeek::expected<std::unique_ptr<Serializer>, std::string> Manager::InstantiateSerializer(const Tag& type) {
|
||||
SerializerComponent* c = serializer_mgr.Lookup(type);
|
||||
if ( ! c )
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("Request to instantiate unknown serializer type (%d:%d)", type.Type(), type.Subtype()));
|
||||
|
||||
if ( ! c->Factory() )
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("Factory invalid for serializer %s", serializer_mgr.GetComponentName(type).c_str()));
|
||||
|
||||
auto bp = c->Factory()();
|
||||
|
||||
if ( ! bp )
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("Failed to instantiate serializer %s", serializer_mgr.GetComponentName(type).c_str()));
|
||||
|
||||
return bp;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include "zeek/plugin/ComponentManager.h"
|
||||
#include "zeek/storage/Backend.h"
|
||||
#include "zeek/storage/Component.h"
|
||||
#include "zeek/storage/Serializer.h"
|
||||
|
||||
namespace zeek::storage {
|
||||
|
||||
|
|
@ -24,7 +25,7 @@ public:
|
|||
|
||||
} // namespace detail
|
||||
|
||||
class Manager final : public plugin::ComponentManager<Component> {
|
||||
class Manager final {
|
||||
public:
|
||||
Manager();
|
||||
~Manager();
|
||||
|
|
@ -43,7 +44,16 @@ public:
|
|||
* @return A std::expected containing either a valid BackendPtr with the result of the
|
||||
* operation or a string containing an error message for failure.
|
||||
*/
|
||||
zeek::expected<BackendPtr, std::string> Instantiate(const Tag& type);
|
||||
zeek::expected<BackendPtr, std::string> InstantiateBackend(const Tag& type);
|
||||
|
||||
/**
|
||||
* Instantiates a new serializer object.
|
||||
*
|
||||
* @param type The tag for the type of backend being opened.
|
||||
* @return A std::expected containing either a valid BackendPtr with the result of the
|
||||
* operation or a string containing an error message for failure.
|
||||
*/
|
||||
zeek::expected<std::unique_ptr<Serializer>, std::string> InstantiateSerializer(const Tag& type);
|
||||
|
||||
/**
|
||||
* Opens a new storage backend.
|
||||
|
|
@ -82,6 +92,9 @@ public:
|
|||
*/
|
||||
void Expire(double t);
|
||||
|
||||
plugin::ComponentManager<BackendComponent>& BackendMgr() { return backend_mgr; }
|
||||
plugin::ComponentManager<SerializerComponent>& SerializerMgr() { return serializer_mgr; }
|
||||
|
||||
protected:
|
||||
friend class storage::detail::ExpirationTimer;
|
||||
void RunExpireThread();
|
||||
|
|
@ -94,6 +107,9 @@ protected:
|
|||
private:
|
||||
std::vector<BackendPtr> backends;
|
||||
std::mutex backends_mtx;
|
||||
|
||||
plugin::ComponentManager<BackendComponent> backend_mgr;
|
||||
plugin::ComponentManager<SerializerComponent> serializer_mgr;
|
||||
};
|
||||
|
||||
} // namespace zeek::storage
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ EnumValPtr ReturnCode::CONNECTION_FAILED;
|
|||
EnumValPtr ReturnCode::DISCONNECTION_FAILED;
|
||||
EnumValPtr ReturnCode::INITIALIZATION_FAILED;
|
||||
EnumValPtr ReturnCode::IN_PROGRESS;
|
||||
EnumValPtr ReturnCode::SERIALIZATION_FAILED;
|
||||
EnumValPtr ReturnCode::UNSERIALIZATION_FAILED;
|
||||
|
||||
void ReturnCode::Initialize() {
|
||||
static const auto& return_code_type = zeek::id::find_type<zeek::EnumType>("Storage::ReturnCode");
|
||||
|
|
@ -61,6 +63,12 @@ void ReturnCode::Initialize() {
|
|||
|
||||
tmp = return_code_type->Lookup("Storage::IN_PROGRESS");
|
||||
IN_PROGRESS = return_code_type->GetEnumVal(tmp);
|
||||
|
||||
tmp = return_code_type->Lookup("Storage::SERIALIZATION_FAILED");
|
||||
SERIALIZATION_FAILED = return_code_type->GetEnumVal(tmp);
|
||||
|
||||
tmp = return_code_type->Lookup("Storage::UNSERIALIZATION_FAILED");
|
||||
UNSERIALIZATION_FAILED = return_code_type->GetEnumVal(tmp);
|
||||
}
|
||||
|
||||
void ReturnCode::Cleanup() {
|
||||
|
|
@ -77,6 +85,8 @@ void ReturnCode::Cleanup() {
|
|||
DISCONNECTION_FAILED.reset();
|
||||
INITIALIZATION_FAILED.reset();
|
||||
IN_PROGRESS.reset();
|
||||
SERIALIZATION_FAILED.reset();
|
||||
UNSERIALIZATION_FAILED.reset();
|
||||
}
|
||||
|
||||
} // namespace zeek::storage
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ namespace storage {
|
|||
* A collection of EnumValPtrs for the default set of result codes in the storage framework.
|
||||
* should be kept up-to-date with the Storage::ReturnCodes script-level enum.
|
||||
*/
|
||||
class ReturnCode {
|
||||
class ReturnCode final {
|
||||
public:
|
||||
static void Initialize();
|
||||
static void Cleanup();
|
||||
|
|
@ -32,6 +32,8 @@ public:
|
|||
static EnumValPtr DISCONNECTION_FAILED;
|
||||
static EnumValPtr INITIALIZATION_FAILED;
|
||||
static EnumValPtr IN_PROGRESS;
|
||||
static EnumValPtr SERIALIZATION_FAILED;
|
||||
static EnumValPtr UNSERIALIZATION_FAILED;
|
||||
};
|
||||
|
||||
} // namespace storage
|
||||
|
|
|
|||
42
src/storage/Serializer.h
Normal file
42
src/storage/Serializer.h
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
// See the file "COPYING" in the main distribution directory for copyright.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "zeek/Val.h"
|
||||
|
||||
namespace zeek::storage {
|
||||
|
||||
/**
|
||||
* Base class for a serializer used by storage backends.
|
||||
*/
|
||||
class Serializer {
|
||||
public:
|
||||
virtual ~Serializer() = default;
|
||||
|
||||
/**
|
||||
* Serializes Zeek Val data into another format.
|
||||
*
|
||||
* @param val The data to serialize.
|
||||
*
|
||||
* @return On success, a byte buffer containing the serialized data. std::nullopt will
|
||||
* be returned on failure.
|
||||
*/
|
||||
virtual std::optional<byte_buffer> Serialize(ValPtr val) = 0;
|
||||
|
||||
/**
|
||||
* Unserializes a byte buffer into Zeek Val objects of a specific type.
|
||||
*
|
||||
* @param buf The byte data to unserialize.
|
||||
* @param type The Zeek script-level type to unserialize the data into.
|
||||
*
|
||||
* @return A zeek::expected containing either the unserialized Val data on success, or
|
||||
* a string containing an error message on failure.
|
||||
*/
|
||||
virtual zeek::expected<ValPtr, std::string> Unserialize(byte_buffer_span buf, TypePtr type) = 0;
|
||||
|
||||
protected:
|
||||
Serializer(std::string name) : name(std::move(name)) {}
|
||||
std::string name;
|
||||
};
|
||||
|
||||
} // namespace zeek::storage
|
||||
|
|
@ -7,10 +7,10 @@
|
|||
|
||||
namespace zeek::storage::backend::redis {
|
||||
|
||||
class Plugin : public plugin::Plugin {
|
||||
class Plugin final : public plugin::Plugin {
|
||||
public:
|
||||
plugin::Configuration Configure() override {
|
||||
AddComponent(new storage::Component("REDIS", backend::redis::Redis::Instantiate));
|
||||
AddComponent(new storage::BackendComponent("REDIS", backend::redis::Redis::Instantiate));
|
||||
|
||||
plugin::Configuration config;
|
||||
config.name = "Zeek::Storage_Backend_Redis";
|
||||
|
|
|
|||
|
|
@ -254,24 +254,30 @@ OperationResult Redis::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool
|
|||
|
||||
auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex);
|
||||
|
||||
std::string format = "SET %s:%s %s";
|
||||
std::string format = "SET %s:%b %b";
|
||||
if ( ! overwrite )
|
||||
format.append(" NX");
|
||||
|
||||
auto json_key = key->ToJSON()->ToStdString();
|
||||
auto json_value = value->ToJSON()->ToStdString();
|
||||
auto key_data = serializer->Serialize(key);
|
||||
if ( ! key_data )
|
||||
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
|
||||
|
||||
auto val_data = serializer->Serialize(value);
|
||||
if ( ! val_data )
|
||||
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize value"};
|
||||
|
||||
int status;
|
||||
// Use built-in expiration if reading live data, since time will move
|
||||
// forward consistently. If reading pcaps, we'll do something else.
|
||||
if ( expiration_time > 0.0 && ! zeek::run_state::reading_traces ) {
|
||||
format.append(" PXAT %" PRIu64);
|
||||
status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(),
|
||||
json_value.data(), static_cast<uint64_t>(expiration_time * 1e3));
|
||||
status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), key_data->data(),
|
||||
key_data->size(), val_data->data(), val_data->size(),
|
||||
static_cast<uint64_t>(expiration_time * 1e3));
|
||||
}
|
||||
else
|
||||
status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), json_key.data(),
|
||||
json_value.data());
|
||||
status = redisAsyncCommand(async_ctx, redisPut, cb, format.c_str(), key_prefix.data(), key_data->data(),
|
||||
key_data->size(), val_data->data(), val_data->size());
|
||||
|
||||
if ( connected && status == REDIS_ERR )
|
||||
return {ReturnCode::OPERATION_FAILED, util::fmt("Failed to queue put operation: %s", async_ctx->errstr)};
|
||||
|
|
@ -284,10 +290,10 @@ OperationResult Redis::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool
|
|||
format = "ZADD %s_expire";
|
||||
if ( ! overwrite )
|
||||
format.append(" NX");
|
||||
format += " %f %s";
|
||||
format += " %f %b";
|
||||
|
||||
status = redisAsyncCommand(async_ctx, redisZADD, NULL, format.c_str(), key_prefix.data(), expiration_time,
|
||||
json_key.data());
|
||||
key_data->data(), key_data->size());
|
||||
if ( connected && status == REDIS_ERR )
|
||||
return {ReturnCode::OPERATION_FAILED, util::fmt("ZADD operation failed: %s", async_ctx->errstr)};
|
||||
|
||||
|
|
@ -307,8 +313,12 @@ OperationResult Redis::DoGet(ResultCallback* cb, ValPtr key) {
|
|||
|
||||
auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex);
|
||||
|
||||
int status = redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%s", key_prefix.data(),
|
||||
key->ToJSON()->ToStdStringView().data());
|
||||
auto key_data = serializer->Serialize(key);
|
||||
if ( ! key_data )
|
||||
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
|
||||
|
||||
int status =
|
||||
redisAsyncCommand(async_ctx, redisGet, cb, "GET %s:%b", key_prefix.data(), key_data->data(), key_data->size());
|
||||
|
||||
if ( connected && status == REDIS_ERR )
|
||||
return {ReturnCode::OPERATION_FAILED, util::fmt("Failed to queue get operation: %s", async_ctx->errstr)};
|
||||
|
|
@ -330,8 +340,12 @@ OperationResult Redis::DoErase(ResultCallback* cb, ValPtr key) {
|
|||
|
||||
auto locked_scope = conditionally_lock(zeek::run_state::reading_traces, expire_mutex);
|
||||
|
||||
int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%s", key_prefix.data(),
|
||||
key->ToJSON()->ToStdStringView().data());
|
||||
auto key_data = serializer->Serialize(key);
|
||||
if ( ! key_data )
|
||||
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
|
||||
|
||||
int status = redisAsyncCommand(async_ctx, redisErase, cb, "DEL %s:%b", key_prefix.data(), key_data->data(),
|
||||
key_data->size());
|
||||
|
||||
if ( connected && status == REDIS_ERR )
|
||||
return {ReturnCode::OPERATION_FAILED, async_ctx->errstr};
|
||||
|
|
@ -439,11 +453,11 @@ void Redis::HandleGetResult(redisReply* reply, ResultCallback* callback) {
|
|||
else if ( reply->type == REDIS_REPLY_ERROR )
|
||||
res = ParseReplyError("get", reply->str);
|
||||
else {
|
||||
auto val = zeek::detail::ValFromJSON(reply->str, val_type, Func::nil);
|
||||
if ( std::holds_alternative<ValPtr>(val) )
|
||||
res = {ReturnCode::SUCCESS, "", std::get<ValPtr>(val)};
|
||||
auto val = serializer->Unserialize({(std::byte*)reply->str, reply->len}, val_type);
|
||||
if ( val )
|
||||
res = {ReturnCode::SUCCESS, "", val.value()};
|
||||
else
|
||||
res = {ReturnCode::OPERATION_FAILED, std::get<std::string>(val)};
|
||||
res = {ReturnCode::OPERATION_FAILED, val.error()};
|
||||
}
|
||||
|
||||
freeReplyObject(reply);
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ struct redisReply;
|
|||
struct redisPollEvents;
|
||||
|
||||
namespace zeek::storage::backend::redis {
|
||||
class Redis : public Backend, public iosource::IOSource {
|
||||
class Redis final : public Backend, public iosource::IOSource {
|
||||
public:
|
||||
Redis() : Backend(SupportedModes::ASYNC, "REDIS"), IOSource(true) {}
|
||||
~Redis() override = default;
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@
|
|||
|
||||
namespace zeek::storage::backend::sqlite {
|
||||
|
||||
class Plugin : public plugin::Plugin {
|
||||
class Plugin final : public plugin::Plugin {
|
||||
public:
|
||||
plugin::Configuration Configure() override {
|
||||
AddComponent(new storage::Component("SQLITE", backend::sqlite::SQLite::Instantiate));
|
||||
AddComponent(new storage::BackendComponent("SQLITE", backend::sqlite::SQLite::Instantiate));
|
||||
|
||||
plugin::Configuration config;
|
||||
config.name = "Zeek::Storage_Backend_SQLite";
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ OperationResult SQLite::DoOpen(OpenResultCallback* cb, RecordValPtr options) {
|
|||
}
|
||||
|
||||
std::string create = "create table if not exists " + table_name + " (";
|
||||
create.append("key_str text primary key, value_str text not null, expire_time real);");
|
||||
create.append("key_str blob primary key, value_str blob not null, expire_time real);");
|
||||
|
||||
char* errorMsg = nullptr;
|
||||
if ( int res = sqlite3_exec(db, create.c_str(), NULL, NULL, &errorMsg); res != SQLITE_OK ) {
|
||||
|
|
@ -151,8 +151,9 @@ OperationResult SQLite::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool
|
|||
if ( ! db )
|
||||
return {ReturnCode::NOT_CONNECTED};
|
||||
|
||||
auto json_key = key->ToJSON();
|
||||
auto json_value = value->ToJSON();
|
||||
auto key_data = serializer->Serialize(key);
|
||||
if ( ! key_data )
|
||||
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
|
||||
|
||||
sqlite3_stmt* stmt;
|
||||
if ( ! overwrite )
|
||||
|
|
@ -160,15 +161,17 @@ OperationResult SQLite::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool
|
|||
else
|
||||
stmt = put_update_stmt.get();
|
||||
|
||||
auto key_str = json_key->ToStdStringView();
|
||||
if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC));
|
||||
if ( auto res = CheckError(sqlite3_bind_blob(stmt, 1, key_data->data(), key_data->size(), SQLITE_STATIC));
|
||||
res.code != ReturnCode::SUCCESS ) {
|
||||
sqlite3_reset(stmt);
|
||||
return res;
|
||||
}
|
||||
|
||||
auto value_str = json_value->ToStdStringView();
|
||||
if ( auto res = CheckError(sqlite3_bind_text(stmt, 2, value_str.data(), value_str.size(), SQLITE_STATIC));
|
||||
auto val_data = serializer->Serialize(value);
|
||||
if ( ! val_data )
|
||||
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize value"};
|
||||
|
||||
if ( auto res = CheckError(sqlite3_bind_blob(stmt, 2, val_data->data(), val_data->size(), SQLITE_STATIC));
|
||||
res.code != ReturnCode::SUCCESS ) {
|
||||
sqlite3_reset(stmt);
|
||||
return res;
|
||||
|
|
@ -180,7 +183,7 @@ OperationResult SQLite::DoPut(ResultCallback* cb, ValPtr key, ValPtr value, bool
|
|||
}
|
||||
|
||||
if ( overwrite ) {
|
||||
if ( auto res = CheckError(sqlite3_bind_text(stmt, 4, value_str.data(), value_str.size(), SQLITE_STATIC));
|
||||
if ( auto res = CheckError(sqlite3_bind_blob(stmt, 4, val_data->data(), val_data->size(), SQLITE_STATIC));
|
||||
res.code != ReturnCode::SUCCESS ) {
|
||||
sqlite3_reset(stmt);
|
||||
return res;
|
||||
|
|
@ -197,11 +200,13 @@ OperationResult SQLite::DoGet(ResultCallback* cb, ValPtr key) {
|
|||
if ( ! db )
|
||||
return {ReturnCode::NOT_CONNECTED};
|
||||
|
||||
auto json_key = key->ToJSON();
|
||||
auto key_data = serializer->Serialize(key);
|
||||
if ( ! key_data )
|
||||
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
|
||||
|
||||
auto stmt = get_stmt.get();
|
||||
|
||||
auto key_str = json_key->ToStdStringView();
|
||||
if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC));
|
||||
if ( auto res = CheckError(sqlite3_bind_blob(stmt, 1, key_data->data(), key_data->size(), SQLITE_STATIC));
|
||||
res.code != ReturnCode::SUCCESS ) {
|
||||
sqlite3_reset(stmt);
|
||||
return res;
|
||||
|
|
@ -217,11 +222,13 @@ OperationResult SQLite::DoErase(ResultCallback* cb, ValPtr key) {
|
|||
if ( ! db )
|
||||
return {ReturnCode::NOT_CONNECTED};
|
||||
|
||||
auto json_key = key->ToJSON();
|
||||
auto key_data = serializer->Serialize(key);
|
||||
if ( ! key_data )
|
||||
return {ReturnCode::SERIALIZATION_FAILED, "Failed to serialize key"};
|
||||
|
||||
auto stmt = erase_stmt.get();
|
||||
|
||||
auto key_str = json_key->ToStdStringView();
|
||||
if ( auto res = CheckError(sqlite3_bind_text(stmt, 1, key_str.data(), key_str.size(), SQLITE_STATIC));
|
||||
if ( auto res = CheckError(sqlite3_bind_blob(stmt, 1, key_data->data(), key_data->size(), SQLITE_STATIC));
|
||||
res.code != ReturnCode::SUCCESS ) {
|
||||
sqlite3_reset(stmt);
|
||||
return res;
|
||||
|
|
@ -266,17 +273,16 @@ OperationResult SQLite::Step(sqlite3_stmt* stmt, bool parse_value) {
|
|||
int step_status = sqlite3_step(stmt);
|
||||
if ( step_status == SQLITE_ROW ) {
|
||||
if ( parse_value ) {
|
||||
// Column 1 is the value
|
||||
const char* text = (const char*)sqlite3_column_text(stmt, 0);
|
||||
auto val = zeek::detail::ValFromJSON(text, val_type, Func::nil);
|
||||
auto blob = static_cast<const std::byte*>(sqlite3_column_blob(stmt, 0));
|
||||
size_t blob_size = sqlite3_column_bytes(stmt, 0);
|
||||
|
||||
auto val = serializer->Unserialize({blob, blob_size}, val_type);
|
||||
sqlite3_reset(stmt);
|
||||
if ( std::holds_alternative<ValPtr>(val) ) {
|
||||
ValPtr val_v = std::get<ValPtr>(val);
|
||||
ret = {ReturnCode::SUCCESS, "", val_v};
|
||||
}
|
||||
else {
|
||||
ret = {ReturnCode::OPERATION_FAILED, std::get<std::string>(val)};
|
||||
}
|
||||
|
||||
if ( val )
|
||||
ret = {ReturnCode::SUCCESS, "", val.value()};
|
||||
else
|
||||
ret = {ReturnCode::OPERATION_FAILED, val.error()};
|
||||
}
|
||||
else {
|
||||
ret = {ReturnCode::OPERATION_FAILED, "sqlite3_step should not have returned a value"};
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ struct sqlite3_stmt;
|
|||
|
||||
namespace zeek::storage::backend::sqlite {
|
||||
|
||||
class SQLite : public Backend {
|
||||
class SQLite final : public Backend {
|
||||
public:
|
||||
SQLite() : Backend(SupportedModes::SYNC, "SQLITE") {}
|
||||
~SQLite() override = default;
|
||||
|
|
|
|||
1
src/storage/serializer/CMakeLists.txt
Normal file
1
src/storage/serializer/CMakeLists.txt
Normal file
|
|
@ -0,0 +1 @@
|
|||
add_subdirectory(json)
|
||||
3
src/storage/serializer/json/CMakeLists.txt
Normal file
3
src/storage/serializer/json/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
zeek_add_plugin(
|
||||
Zeek Storage_Serializer_JSON
|
||||
SOURCES JSON.cc Plugin.cc)
|
||||
44
src/storage/serializer/json/JSON.cc
Normal file
44
src/storage/serializer/json/JSON.cc
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
// See the file "COPYING" in the main distribution directory for copyright.
|
||||
|
||||
#include "zeek/storage/serializer/json/JSON.h"
|
||||
|
||||
#include "zeek/Func.h"
|
||||
|
||||
namespace zeek::storage::serializer::json {
|
||||
|
||||
std::string JSON::versioned_name = "JSONv1";
|
||||
|
||||
std::unique_ptr<Serializer> JSON::Instantiate() { return std::make_unique<JSON>(); }
|
||||
|
||||
JSON::JSON() : Serializer("JSON") {}
|
||||
|
||||
std::optional<byte_buffer> JSON::Serialize(ValPtr val) {
|
||||
static auto byte_converter = [](u_char c) { return std::byte(c); };
|
||||
|
||||
byte_buffer buf;
|
||||
auto json = val->ToJSON();
|
||||
buf.reserve(json->Len() + versioned_name.size() + 1);
|
||||
|
||||
std::transform(versioned_name.begin(), versioned_name.end(), std::back_inserter(buf), byte_converter);
|
||||
buf.push_back(static_cast<std::byte>(';'));
|
||||
std::transform(json->Bytes(), json->Bytes() + json->Len(), std::back_inserter(buf), byte_converter);
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
zeek::expected<ValPtr, std::string> JSON::Unserialize(byte_buffer_span buf, TypePtr type) {
|
||||
std::string_view text{reinterpret_cast<std::string::const_pointer>(buf.data()), buf.size()};
|
||||
|
||||
auto semicolon = text.find(';');
|
||||
if ( semicolon == std::string::npos )
|
||||
return zeek::unexpected<std::string>("Version string missing");
|
||||
|
||||
std::string_view version = std::string_view(text).substr(0, semicolon);
|
||||
if ( version != versioned_name )
|
||||
return zeek::unexpected<std::string>(
|
||||
util::fmt("Version doesn't match: %s vs %s", version.data(), versioned_name.c_str()));
|
||||
|
||||
return zeek::detail::ValFromJSON(text.substr(semicolon + 1), type, Func::nil);
|
||||
}
|
||||
|
||||
} // namespace zeek::storage::serializer::json
|
||||
23
src/storage/serializer/json/JSON.h
Normal file
23
src/storage/serializer/json/JSON.h
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
// See the file "COPYING" in the main distribution directory for copyright.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "zeek/storage/Serializer.h"
|
||||
|
||||
namespace zeek::storage::serializer::json {
|
||||
|
||||
class JSON final : public Serializer {
|
||||
public:
|
||||
static std::unique_ptr<Serializer> Instantiate();
|
||||
|
||||
JSON();
|
||||
~JSON() override = default;
|
||||
|
||||
std::optional<byte_buffer> Serialize(ValPtr val) override;
|
||||
zeek::expected<ValPtr, std::string> Unserialize(byte_buffer_span buf, TypePtr type) override;
|
||||
|
||||
private:
|
||||
static std::string versioned_name;
|
||||
};
|
||||
|
||||
} // namespace zeek::storage::serializer::json
|
||||
22
src/storage/serializer/json/Plugin.cc
Normal file
22
src/storage/serializer/json/Plugin.cc
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
// See the file "COPYING" in the main distribution directory for copyright.
|
||||
|
||||
#include "zeek/plugin/Plugin.h"
|
||||
|
||||
#include "zeek/storage/Component.h"
|
||||
#include "zeek/storage/serializer/json/JSON.h"
|
||||
|
||||
namespace zeek::storage::serializer::json {
|
||||
|
||||
class Plugin final : public plugin::Plugin {
|
||||
public:
|
||||
plugin::Configuration Configure() override {
|
||||
AddComponent(new storage::SerializerComponent("JSON", serializer::json::JSON::Instantiate));
|
||||
|
||||
plugin::Configuration config;
|
||||
config.name = "Zeek::Storage_Serializer_JSON";
|
||||
config.description = "JSON serializer for storage framework";
|
||||
return config;
|
||||
}
|
||||
} plugin;
|
||||
|
||||
} // namespace zeek::storage::serializer::json
|
||||
|
|
@ -82,7 +82,7 @@ function Storage::Async::__open_backend%(btype: Storage::Backend, options: any,
|
|||
auto btype_val = IntrusivePtr<EnumVal>{NewRef{}, btype->AsEnumVal()};
|
||||
Tag tag{btype_val};
|
||||
|
||||
auto b = storage_mgr->Instantiate(tag);
|
||||
auto b = storage_mgr->InstantiateBackend(tag);
|
||||
|
||||
if ( ! b.has_value() ) {
|
||||
trigger->Cache(
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ function Storage::Sync::__open_backend%(btype: Storage::Backend, options: any, k
|
|||
auto btype_val = IntrusivePtr<EnumVal>{NewRef{}, btype->AsEnumVal()};
|
||||
Tag tag{btype_val};
|
||||
|
||||
auto b = storage_mgr->Instantiate(tag);
|
||||
auto b = storage_mgr->InstantiateBackend(tag);
|
||||
|
||||
if ( ! b.has_value() ) {
|
||||
emit_builtin_error(b.error().c_str());
|
||||
|
|
|
|||
|
|
@ -1363,8 +1363,8 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const {
|
|||
|
||||
auto tt = rt->GetFieldType<TableType>("cluster");
|
||||
auto json_res = detail::ValFromJSON(cluster, tt, Func::nil);
|
||||
if ( auto val = std::get_if<ValPtr>(&json_res) ) {
|
||||
rval->AssignField("cluster", *val);
|
||||
if ( json_res ) {
|
||||
rval->AssignField("cluster", json_res.value());
|
||||
}
|
||||
else {
|
||||
// This should never happen: the JSON data comes from a table[string] of
|
||||
|
|
@ -1372,7 +1372,7 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const {
|
|||
// here can be hard to debug. Other JSON code (see FromJSON()) fails
|
||||
// silently when the JSON is misformatted. We just warn:
|
||||
fprintf(stderr, "Could not parse %s's cluster table from '%s': %s\n", name.c_str(), cluster.c_str(),
|
||||
std::get<std::string>(json_res).c_str());
|
||||
json_res.error().c_str());
|
||||
rval->AssignField("cluster", make_intrusive<TableVal>(std::move(tt)));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,14 +16,12 @@
|
|||
|
||||
#include <libgen.h>
|
||||
#include <unistd.h>
|
||||
#include <array>
|
||||
#include <cinttypes>
|
||||
#include <cstdarg>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <memory> // std::unique_ptr
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
|
@ -106,6 +104,8 @@ template<typename E>
|
|||
using unexpected = nonstd::unexpected<E>;
|
||||
} // namespace zeek
|
||||
|
||||
#include "zeek/Span.h"
|
||||
|
||||
using zeek_int_t = int64_t;
|
||||
using zeek_uint_t = uint64_t;
|
||||
|
||||
|
|
@ -121,6 +121,10 @@ namespace zeek {
|
|||
class ODesc;
|
||||
class RecordVal;
|
||||
|
||||
// Byte buffer types used by serialization code in storage and cluster.
|
||||
using byte_buffer = std::vector<std::byte>;
|
||||
using byte_buffer_span = Span<const std::byte>;
|
||||
|
||||
namespace util {
|
||||
namespace detail {
|
||||
|
||||
|
|
|
|||
|
|
@ -5268,15 +5268,15 @@ function from_json%(s: string, t: any, key_func: string_mapper &default=from_jso
|
|||
auto res = zeek::detail::ValFromJSON(s->ToStdStringView(), t->AsType()->AsTypeType()->GetType(),
|
||||
key_func_ptr);
|
||||
|
||||
if ( auto val = std::get_if<zeek::ValPtr>(&res) )
|
||||
if ( res )
|
||||
{
|
||||
rval->Assign(v_idx, *val);
|
||||
rval->Assign(v_idx, res.value());
|
||||
rval->Assign(valid_idx, true);
|
||||
}
|
||||
else
|
||||
{
|
||||
rval->Assign(valid_idx, false);
|
||||
zeek::emit_builtin_error(std::get<std::string>(res).c_str());
|
||||
zeek::emit_builtin_error(res.error().c_str());
|
||||
}
|
||||
|
||||
return std::move(rval);
|
||||
|
|
|
|||
|
|
@ -324,6 +324,8 @@ void ScriptInfo::DoInitPostScript() {
|
|||
else if ( name == "base/frameworks/storage/main.zeek" ) {
|
||||
const auto& backend_id = zeek::detail::global_scope()->Find("Storage::Backend");
|
||||
types.push_back(new IdentifierInfo(backend_id, this));
|
||||
const auto& serializer_id = zeek::detail::global_scope()->Find("Storage::Serializer");
|
||||
types.push_back(new IdentifierInfo(serializer_id, this));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ get result after erase, [code=Storage::KEY_NOT_FOUND, error_str=<uninitialized>,
|
|||
close result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<uninitialized>]
|
||||
results of trying to use closed handle: get: Storage::NOT_CONNECTED, put: Storage::NOT_CONNECTED, erase: Storage::NOT_CONNECTED
|
||||
|
||||
open result 2, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::STORAGEDUMMY: open_fail was set to true, returning error, value=<opaque of BackendHandleVal>]
|
||||
open result 2, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::STORAGE_BACKEND_STORAGEDUMMY: open_fail was set to true, returning error, value=<opaque of BackendHandleVal>]
|
||||
close result on closed handle, [code=Storage::NOT_CONNECTED, error_str=Backend is closed, value=<uninitialized>]
|
||||
|
||||
open result 3, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<opaque of BackendHandleVal>]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
open_result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<opaque of BackendHandleVal>]
|
||||
Storage::backend_opened, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]]
|
||||
Storage::backend_lost, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]], Server closed the connection
|
||||
Storage::backend_opened, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]]
|
||||
Storage::backend_lost, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]], Server closed the connection
|
||||
|
|
|
|||
|
|
@ -9,5 +9,5 @@ get result same as originally inserted, T
|
|||
put result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<uninitialized>]
|
||||
get result, [code=Storage::SUCCESS, error_str=<uninitialized>, value=value2345]
|
||||
get result same as overwritten, T
|
||||
Storage::backend_opened, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]]
|
||||
Storage::backend_lost, Storage::REDIS, [redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]], Client disconnected
|
||||
Storage::backend_opened, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]]
|
||||
Storage::backend_lost, Storage::STORAGE_BACKEND_REDIS, [serializer=Storage::STORAGE_SERIALIZER_JSON, redis=[server_host=127.0.0.1, server_port=xxxx/tcp, server_unix_socket=<uninitialized>, key_prefix=testing]], Client disconnected
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Storage::backend_opened, Storage::SQLITE, [sqlite=[database_path=test.sqlite, table_name=testing, tuning_params={
|
||||
Storage::backend_opened, Storage::STORAGE_BACKEND_SQLITE, [serializer=Storage::STORAGE_SERIALIZER_JSON, sqlite=[database_path=test.sqlite, table_name=testing, tuning_params={
|
||||
[synchronous] = normal,
|
||||
[temp_store] = memory,
|
||||
[journal_mode] = WAL
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Open result, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::SQLITE: SQLite call failed: unable to open database file, value=<opaque of BackendHandleVal>]
|
||||
Open result, [code=Storage::OPERATION_FAILED, error_str=Failed to open backend Storage::STORAGE_BACKEND_SQLITE: SQLite call failed: unable to open database file, value=<opaque of BackendHandleVal>]
|
||||
Open result 2, [code=Storage::SUCCESS, error_str=<uninitialized>, value=<opaque of BackendHandleVal>]
|
||||
Put result with bad key type, [code=Storage::KEY_TYPE_MISMATCH, error_str=<uninitialized>, value=<uninitialized>]
|
||||
Put result on closed handle, [code=Storage::NOT_CONNECTED, error_str=Backend is closed, value=<uninitialized>]
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@ Plugin plugin;
|
|||
using namespace btest::plugin::Testing_StorageDummy;
|
||||
|
||||
zeek::plugin::Configuration Plugin::Configure() {
|
||||
AddComponent(new zeek::storage::Component("StorageDummy", btest::storage::backend::StorageDummy::Instantiate));
|
||||
AddComponent(
|
||||
new zeek::storage::BackendComponent("StorageDummy", btest::storage::backend::StorageDummy::Instantiate));
|
||||
|
||||
zeek::plugin::Configuration config;
|
||||
config.name = "Testing::StorageDummy";
|
||||
|
|
|
|||
|
|
@ -49,9 +49,10 @@ OperationResult StorageDummy::DoPut(ResultCallback* cb, ValPtr key, ValPtr value
|
|||
if ( timeout_put )
|
||||
return {ReturnCode::TIMEOUT};
|
||||
|
||||
auto json_key = key->ToJSON()->ToStdString();
|
||||
auto json_value = value->ToJSON()->ToStdString();
|
||||
data[json_key] = json_value;
|
||||
auto key_data = serializer->Serialize(key);
|
||||
auto val_data = serializer->Serialize(value);
|
||||
|
||||
data[*key_data] = *val_data;
|
||||
return {ReturnCode::SUCCESS};
|
||||
}
|
||||
|
||||
|
|
@ -59,31 +60,31 @@ OperationResult StorageDummy::DoPut(ResultCallback* cb, ValPtr key, ValPtr value
|
|||
* The workhorse method for Get(). This must be implemented for plugins.
|
||||
*/
|
||||
OperationResult StorageDummy::DoGet(ResultCallback* cb, ValPtr key) {
|
||||
auto json_key = key->ToJSON();
|
||||
auto it = data.find(json_key->ToStdString());
|
||||
auto key_data = serializer->Serialize(key);
|
||||
|
||||
auto it = data.find(*key_data);
|
||||
if ( it == data.end() )
|
||||
return {ReturnCode::KEY_NOT_FOUND};
|
||||
|
||||
auto val = zeek::detail::ValFromJSON(it->second.c_str(), val_type, Func::nil);
|
||||
if ( std::holds_alternative<ValPtr>(val) ) {
|
||||
ValPtr val_v = std::get<ValPtr>(val);
|
||||
return {ReturnCode::SUCCESS, "", val_v};
|
||||
}
|
||||
auto val = serializer->Unserialize(it->second, val_type);
|
||||
if ( val )
|
||||
return {ReturnCode::SUCCESS, "", val.value()};
|
||||
|
||||
return {ReturnCode::OPERATION_FAILED, std::get<std::string>(val)};
|
||||
return {ReturnCode::UNSERIALIZATION_FAILED, val.error()};
|
||||
}
|
||||
|
||||
/**
|
||||
* The workhorse method for Erase(). This must be implemented for plugins.
|
||||
*/
|
||||
OperationResult StorageDummy::DoErase(ResultCallback* cb, ValPtr key) {
|
||||
auto json_key = key->ToJSON();
|
||||
auto it = data.find(json_key->ToStdString());
|
||||
if ( it == data.end() )
|
||||
return {ReturnCode::KEY_NOT_FOUND};
|
||||
auto key_data = serializer->Serialize(key);
|
||||
|
||||
if ( auto it = data.find(*key_data); it != data.end() ) {
|
||||
data.erase(it);
|
||||
return {ReturnCode::SUCCESS};
|
||||
}
|
||||
|
||||
return {ReturnCode::KEY_NOT_FOUND};
|
||||
}
|
||||
|
||||
} // namespace btest::storage::backend
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ public:
|
|||
zeek::storage::OperationResult DoErase(zeek::storage::ResultCallback* cb, zeek::ValPtr key) override;
|
||||
|
||||
private:
|
||||
std::map<std::string, std::string> data;
|
||||
std::map<zeek::byte_buffer, zeek::byte_buffer> data;
|
||||
bool open = false;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ event zeek_init() {
|
|||
local value = "value5678";
|
||||
|
||||
# Basic operation. Open, put, and get the value back.
|
||||
local res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, string, string);
|
||||
local res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_STORAGEDUMMY, opts, string, string);
|
||||
print "open result", res;
|
||||
local b = res$value;
|
||||
|
||||
|
|
@ -63,7 +63,7 @@ event zeek_init() {
|
|||
|
||||
# Test failing to open the handle and test closing an invalid handle.
|
||||
opts$dummy = [$open_fail = T, $timeout_put = F];
|
||||
res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, string, string);
|
||||
res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_STORAGEDUMMY, opts, string, string);
|
||||
print "open result 2", res;
|
||||
res = Storage::Sync::close_backend(res$value);
|
||||
print "close result on closed handle", res;
|
||||
|
|
@ -71,7 +71,7 @@ event zeek_init() {
|
|||
|
||||
# Test timing out an async put request.
|
||||
opts$dummy = [$open_fail = F, $timeout_put = T];
|
||||
res = Storage::Sync::open_backend(Storage::STORAGEDUMMY, opts, string, string);
|
||||
res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_STORAGEDUMMY, opts, string, string);
|
||||
print "open result 3", res;
|
||||
b = res$value;
|
||||
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ event zeek_init() {
|
|||
value[2] = "b";
|
||||
value[3] = "c";
|
||||
|
||||
local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, Rec, tbl);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, Rec, tbl);
|
||||
print "open result", open_res;
|
||||
local b = open_res$value;
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ event zeek_init()
|
|||
|
||||
# Test inserting/retrieving a key/value pair that we know won't be in
|
||||
# the backend yet.
|
||||
local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string);
|
||||
print "open result", open_res;
|
||||
local b = open_res$value;
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ event setup_test()
|
|||
local opts : Storage::BackendOptions;
|
||||
opts$sqlite = [$database_path = "storage-test.sqlite", $table_name = "testing"];
|
||||
|
||||
local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string);
|
||||
print "open result", open_res;
|
||||
|
||||
b = open_res$value;
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ event zeek_init() {
|
|||
local value = "value7890";
|
||||
local value2 = "value2345";
|
||||
|
||||
local res = Storage::Sync::open_backend(Storage::SQLITE, opts, str, str);
|
||||
local res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, str, str);
|
||||
print "open result", res;
|
||||
local b = res$value;
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ event zeek_init()
|
|||
local key = "key1234";
|
||||
local value = "value5678";
|
||||
|
||||
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
|
||||
print "open result", open_res;
|
||||
local b = open_res$value;
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ event zeek_init()
|
|||
local value = "value5678";
|
||||
|
||||
when [opts, key, value] ( local open_res = Storage::Async::open_backend(
|
||||
Storage::REDIS, opts, string, string) )
|
||||
Storage::STORAGE_BACKEND_REDIS, opts, string, string) )
|
||||
{
|
||||
print "open result", open_res;
|
||||
local b = open_res$value;
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ event zeek_init()
|
|||
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
|
||||
"REDIS_PORT")), $key_prefix="testing" ];
|
||||
|
||||
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
|
||||
backend = open_res$value;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -26,13 +26,14 @@ event Storage::backend_lost(tag: Storage::Backend, config: any, reason: string)
|
|||
event zeek_init()
|
||||
{
|
||||
local opts: Storage::BackendOptions;
|
||||
opts$serializer = Storage::STORAGE_SERIALIZER_JSON;
|
||||
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
|
||||
"REDIS_PORT")), $key_prefix="testing" ];
|
||||
|
||||
local key = "key1234";
|
||||
local value = "value1234";
|
||||
|
||||
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
|
||||
print "open_result", open_res;
|
||||
|
||||
# Kill the redis server so the backend will disconnect and fire the backend_lost event.
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ event zeek_init()
|
|||
local key = "key1234";
|
||||
local value = "value1234";
|
||||
|
||||
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
|
||||
print "open_result", open_res;
|
||||
|
||||
local b = open_res$value;
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ event setup_test()
|
|||
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
|
||||
"REDIS_PORT")), $key_prefix="testing" ];
|
||||
|
||||
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
|
||||
print "open result", open_res;
|
||||
|
||||
b = open_res$value;
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ event setup_test()
|
|||
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
|
||||
"REDIS_PORT")), $key_prefix="testing" ];
|
||||
|
||||
local open_res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
|
||||
print "open result", open_res;
|
||||
|
||||
b = open_res$value;
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ event Storage::backend_lost(tag: Storage::Backend, config: any, reason: string)
|
|||
event zeek_init()
|
||||
{
|
||||
local opts: Storage::BackendOptions;
|
||||
opts$serializer = Storage::STORAGE_SERIALIZER_JSON;
|
||||
opts$redis = [ $server_host="127.0.0.1", $server_port=to_port(getenv(
|
||||
"REDIS_PORT")), $key_prefix="testing" ];
|
||||
|
||||
|
|
@ -31,7 +32,7 @@ event zeek_init()
|
|||
local value = "value1234";
|
||||
local value2 = "value2345";
|
||||
|
||||
local res = Storage::Sync::open_backend(Storage::REDIS, opts, string, string);
|
||||
local res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_REDIS, opts, string, string);
|
||||
print "open_result", res;
|
||||
|
||||
local b = res$value;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ event zeek_init()
|
|||
|
||||
# Test inserting/retrieving a key/value pair that we know won't be in
|
||||
# the backend yet.
|
||||
local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string);
|
||||
print "open result", open_res;
|
||||
|
||||
local b = open_res$value;
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ event zeek_init()
|
|||
# Test inserting/retrieving a key/value pair that we know won't be in
|
||||
# the backend yet.
|
||||
when [opts, key, value] ( local open_res = Storage::Sync::open_backend(
|
||||
Storage::SQLITE, opts, string, string) )
|
||||
Storage::STORAGE_BACKEND_SQLITE, opts, string, string) )
|
||||
{
|
||||
print "open result", open_res;
|
||||
local b = open_res$value;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ event zeek_init()
|
|||
{
|
||||
# Create a database file in the .tmp directory with a 'testing' table
|
||||
local opts: Storage::BackendOptions;
|
||||
opts$serializer = Storage::STORAGE_SERIALIZER_JSON;
|
||||
opts$sqlite = [ $database_path="test.sqlite", $table_name="testing" ];
|
||||
|
||||
local key = "key1234";
|
||||
|
|
@ -24,7 +25,7 @@ event zeek_init()
|
|||
# Test inserting/retrieving a key/value pair that we know won't be in
|
||||
# the backend yet.
|
||||
when [opts, key, value] ( local open_res = Storage::Async::open_backend(
|
||||
Storage::SQLITE, opts, string, string) )
|
||||
Storage::STORAGE_BACKEND_SQLITE, opts, string, string) )
|
||||
{
|
||||
print "open result", open_res;
|
||||
local b = open_res$value;
|
||||
|
|
|
|||
|
|
@ -14,12 +14,12 @@ event zeek_init() {
|
|||
$table_name = "testing"];
|
||||
|
||||
# This should report an error in .stderr and reporter.log
|
||||
local open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string);
|
||||
local open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string);
|
||||
print "Open result", open_res;
|
||||
|
||||
# Open a valid database file
|
||||
opts$sqlite$database_path = "test.sqlite";
|
||||
open_res = Storage::Sync::open_backend(Storage::SQLITE, opts, string, string);
|
||||
open_res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string);
|
||||
print "Open result 2", open_res;
|
||||
|
||||
local b = open_res$value;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue