Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Move byte_buffer types from cluster and storage into util

Browse source
This commit is contained in:
Tim Wojtulewicz 2025-04-04 13:22:53 -07:00
parent 40b75cb809
commit 1169fcf2a2
15 changed files with 59 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/broker/Manager.h
View file

@ -400,8 +400,7 @@ private:
// This should never be reached, broker itself doesn't call this and overrides // This should never be reached, broker itself doesn't call this and overrides
// the generic DoPublishEvent() method that would call this. // the generic DoPublishEvent() method that would call this.
bool DoPublishEvent(const std::string& topic, const std::string& format, bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) override {
const cluster::detail::byte_buffer& buf) override {
throw std::logic_error("not implemented"); throw std::logic_error("not implemented");
} }
@ -416,7 +415,7 @@ private:
} }
bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format,
cluster::detail::byte_buffer& buf) override { byte_buffer& buf) override {
// Not implemented by broker. // Not implemented by broker.
throw std::logic_error("not implemented"); throw std::logic_error("not implemented");
} }

11
src/cluster/Backend.cc
View file

@ -100,7 +100,7 @@ std::optional<detail::Event> Backend::MakeClusterEvent(FuncValPtr handler, ArgsS
// Default implementation doing the serialization. // Default implementation doing the serialization.
bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& event) { bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& event) {
cluster::detail::byte_buffer buf; byte_buffer buf;
if ( ! event_serializer->SerializeEvent(buf, event) ) if ( ! event_serializer->SerializeEvent(buf, event) )
return false; return false;
@ -111,7 +111,7 @@ bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& e
// Default implementation doing log record serialization. // Default implementation doing log record serialization.
bool Backend::DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, bool Backend::DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header,
zeek::Span<zeek::logging::detail::LogRecord> records) { zeek::Span<zeek::logging::detail::LogRecord> records) {
cluster::detail::byte_buffer buf; byte_buffer buf;
if ( ! log_serializer->SerializeLogWrite(buf, header, records) ) if ( ! log_serializer->SerializeLogWrite(buf, header, records) )
return false; return false;
@ -123,8 +123,7 @@ void Backend::EnqueueEvent(EventHandlerPtr h, zeek::Args args) {
event_handling_strategy->EnqueueLocalEvent(h, std::move(args)); event_handling_strategy->EnqueueLocalEvent(h, std::move(args));
} }
bool Backend::ProcessEventMessage(std::string_view topic, std::string_view format, bool Backend::ProcessEventMessage(std::string_view topic, std::string_view format, const byte_buffer_span payload) {
const detail::byte_buffer_span payload) {
if ( format != event_serializer->Name() ) { if ( format != event_serializer->Name() ) {
zeek::reporter->Error("ProcessEventMessage: Wrong format: %s vs %s", std::string{format}.c_str(), zeek::reporter->Error("ProcessEventMessage: Wrong format: %s vs %s", std::string{format}.c_str(),
event_serializer->Name().c_str()); event_serializer->Name().c_str());
@ -143,7 +142,7 @@ bool Backend::ProcessEventMessage(std::string_view topic, std::string_view forma
return event_handling_strategy->HandleRemoteEvent(topic, std::move(*r)); return event_handling_strategy->HandleRemoteEvent(topic, std::move(*r));
} }
bool Backend::ProcessLogMessage(std::string_view format, detail::byte_buffer_span payload) { bool Backend::ProcessLogMessage(std::string_view format, byte_buffer_span payload) {
// We could also dynamically lookup the right de-serializer, but // We could also dynamically lookup the right de-serializer, but
// for now assume we just receive what is configured. // for now assume we just receive what is configured.
if ( format != log_serializer->Name() ) { if ( format != log_serializer->Name() ) {
@ -162,7 +161,7 @@ bool Backend::ProcessLogMessage(std::string_view format, detail::byte_buffer_spa
return zeek::log_mgr->WriteBatchFromRemote(result->header, std::move(result->records)); return zeek::log_mgr->WriteBatchFromRemote(result->header, std::move(result->records));
} }
bool ThreadedBackend::ProcessBackendMessage(int tag, detail::byte_buffer_span payload) { bool ThreadedBackend::ProcessBackendMessage(int tag, byte_buffer_span payload) {
return DoProcessBackendMessage(tag, payload); return DoProcessBackendMessage(tag, payload);
} }

23
src/cluster/Backend.h
View file

@ -311,12 +311,12 @@ protected:
/** /**
* Process an incoming event message. * Process an incoming event message.
*/ */
bool ProcessEventMessage(std::string_view topic, std::string_view format, detail::byte_buffer_span payload); bool ProcessEventMessage(std::string_view topic, std::string_view format, byte_buffer_span payload);
/** /**
* Process an incoming log message. * Process an incoming log message.
*/ */
bool ProcessLogMessage(std::string_view format, detail::byte_buffer_span payload); bool ProcessLogMessage(std::string_view format, byte_buffer_span payload);
private: private:
/** /**
@ -350,7 +350,7 @@ private:
/** /**
* Publish a cluster::detail::Event to the given topic. * Publish a cluster::detail::Event to the given topic.
* *
* The default implementation serializes to a detail::byte_buffer and * The default implementation serializes to a byte_buffer and
* calls DoPublishEvent() with the resulting buffer. * calls DoPublishEvent() with the resulting buffer.
* *
* This hook method only exists for the existing Broker implementation that * This hook method only exists for the existing Broker implementation that
@ -373,8 +373,7 @@ private:
* @param buf the serialized Event. * @param buf the serialized Event.
* @return true if the message has been published successfully. * @return true if the message has been published successfully.
*/ */
virtual bool DoPublishEvent(const std::string& topic, const std::string& format, virtual bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) = 0;
const detail::byte_buffer& buf) = 0;
/** /**
* Register interest in messages that use a certain topic prefix. * Register interest in messages that use a certain topic prefix.
@ -405,7 +404,7 @@ private:
/** /**
* Serialize a log batch, then forward it to DoPublishLogWrites() below. * Serialize a log batch, then forward it to DoPublishLogWrites() below.
* The default implementation serializes to a detail::byte_buffer and * The default implementation serializes to a byte_buffer and
* calls DoPublishLogWrites() with the resulting buffer. * calls DoPublishLogWrites() with the resulting buffer.
* *
* This hook method only exists for the existing Broker implementation that * This hook method only exists for the existing Broker implementation that
@ -440,7 +439,7 @@ private:
* @return true if the message has been published successfully. * @return true if the message has been published successfully.
*/ */
virtual bool DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, const std::string& format, virtual bool DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, const std::string& format,
detail::byte_buffer& buf) = 0; byte_buffer& buf) = 0;
std::string name; std::string name;
zeek::Tag tag; zeek::Tag tag;
@ -471,7 +470,7 @@ private:
struct EventMessage { struct EventMessage {
std::string topic; std::string topic;
std::string format; std::string format;
detail::byte_buffer payload; byte_buffer payload;
auto payload_span() const { return Span(payload.data(), payload.size()); }; auto payload_span() const { return Span(payload.data(), payload.size()); };
}; };
@ -481,7 +480,7 @@ struct EventMessage {
*/ */
struct LogMessage { struct LogMessage {
std::string format; std::string format;
detail::byte_buffer payload; byte_buffer payload;
auto payload_span() const { return Span(payload.data(), payload.size()); }; auto payload_span() const { return Span(payload.data(), payload.size()); };
}; };
@ -494,7 +493,7 @@ struct LogMessage {
*/ */
struct BackendMessage { struct BackendMessage {
int tag; int tag;
detail::byte_buffer payload; byte_buffer payload;
auto payload_span() const { return Span(payload.data(), payload.size()); }; auto payload_span() const { return Span(payload.data(), payload.size()); };
}; };
@ -546,13 +545,13 @@ private:
/** /**
* Process a backend specific message queued as BackendMessage. * Process a backend specific message queued as BackendMessage.
*/ */
bool ProcessBackendMessage(int tag, detail::byte_buffer_span payload); bool ProcessBackendMessage(int tag, byte_buffer_span payload);
/** /**
* If a cluster backend produces messages of type BackendMessage, * If a cluster backend produces messages of type BackendMessage,
* this method will be invoked by the main thread to process it. * this method will be invoked by the main thread to process it.
*/ */
virtual bool DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) { return false; }; virtual bool DoProcessBackendMessage(int tag, byte_buffer_span payload) { return false; };
/** /**
* Hook method for OnLooProcess. * Hook method for OnLooProcess.

14
src/cluster/Serializer.h
View file

@ -6,7 +6,6 @@
#include <optional> #include <optional>
#include <string> #include <string>
#include <vector>
#include "zeek/Span.h" #include "zeek/Span.h"
#include "zeek/logging/Types.h" #include "zeek/logging/Types.h"
@ -15,13 +14,8 @@ namespace zeek::cluster {
namespace detail { namespace detail {
class Event; class Event;
using byte_buffer = std::vector<std::byte>;
using byte_buffer_span = Span<const std::byte>;
} // namespace detail } // namespace detail
/** /**
* This class handles encoding of events into byte buffers and back. * This class handles encoding of events into byte buffers and back.
* *
@ -40,7 +34,7 @@ public:
* *
* @returns True on success, false in exceptional cases (e.g. unsupported serialization). * @returns True on success, false in exceptional cases (e.g. unsupported serialization).
*/ */
virtual bool SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) = 0; virtual bool SerializeEvent(byte_buffer& buf, const detail::Event& event) = 0;
/** /**
* Unserialize an event from a given byte buffer. * Unserialize an event from a given byte buffer.
@ -49,7 +43,7 @@ public:
* *
* @returns The event, or std::nullopt on error. * @returns The event, or std::nullopt on error.
*/ */
virtual std::optional<cluster::detail::Event> UnserializeEvent(detail::byte_buffer_span buf) = 0; virtual std::optional<cluster::detail::Event> UnserializeEvent(byte_buffer_span buf) = 0;
/** /**
* @returns The name of this event serializer instance. * @returns The name of this event serializer instance.
@ -85,7 +79,7 @@ public:
* @param header The log batch header. * @param header The log batch header.
* @param records The actual log writes. * @param records The actual log writes.
*/ */
virtual bool SerializeLogWrite(detail::byte_buffer& buf, const logging::detail::LogWriteHeader& header, virtual bool SerializeLogWrite(byte_buffer& buf, const logging::detail::LogWriteHeader& header,
zeek::Span<logging::detail::LogRecord> records) = 0; zeek::Span<logging::detail::LogRecord> records) = 0;
/** /**
@ -93,7 +87,7 @@ public:
* *
* @param buf The span representing received log writes. * @param buf The span representing received log writes.
*/ */
virtual std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(detail::byte_buffer_span buf) = 0; virtual std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(byte_buffer_span buf) = 0;
/** /**
* @returns The name of this log serializer instance. * @returns The name of this log serializer instance.

13
src/cluster/backend/zeromq/ZeroMQ.cc
View file

@ -253,8 +253,7 @@ bool ZeroMQBackend::SpawnZmqProxyThread() {
return proxy_thread->Start(); return proxy_thread->Start();
} }
bool ZeroMQBackend::DoPublishEvent(const std::string& topic, const std::string& format, bool ZeroMQBackend::DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) {
const cluster::detail::byte_buffer& buf) {
// Publishing an event happens as a multipart message with 4 parts: // Publishing an event happens as a multipart message with 4 parts:
// //
// * The topic to publish to - this is required by XPUB/XSUB // * The topic to publish to - this is required by XPUB/XSUB
@ -336,7 +335,7 @@ bool ZeroMQBackend::DoUnsubscribe(const std::string& topic_prefix) {
} }
bool ZeroMQBackend::DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, bool ZeroMQBackend::DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format,
cluster::detail::byte_buffer& buf) { byte_buffer& buf) {
ZEROMQ_DEBUG("Publishing %zu bytes of log writes (path %s)", buf.size(), header.path.c_str()); ZEROMQ_DEBUG("Publishing %zu bytes of log writes (path %s)", buf.size(), header.path.c_str());
static std::string message_type = "log-write"; static std::string message_type = "log-write";
@ -405,7 +404,7 @@ void ZeroMQBackend::Run() {
continue; continue;
} }
detail::byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()}; byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()};
LogMessage lm{.format = std::string(msg[2].data<const char>(), msg[2].size()), LogMessage lm{.format = std::string(msg[2].data<const char>(), msg[2].size()),
.payload = std::move(payload)}; .payload = std::move(payload)};
@ -487,7 +486,7 @@ void ZeroMQBackend::Run() {
QueueMessage qm; QueueMessage qm;
auto* start = msg[0].data<std::byte>() + 1; auto* start = msg[0].data<std::byte>() + 1;
auto* end = msg[0].data<std::byte>() + msg[0].size(); auto* end = msg[0].data<std::byte>() + msg[0].size();
detail::byte_buffer topic(start, end); byte_buffer topic(start, end);
if ( first == 1 ) { if ( first == 1 ) {
qm = BackendMessage{1, std::move(topic)}; qm = BackendMessage{1, std::move(topic)};
} }
@ -516,7 +515,7 @@ void ZeroMQBackend::Run() {
if ( sender == NodeId() ) if ( sender == NodeId() )
continue; continue;
detail::byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()}; byte_buffer payload{msg[3].data<std::byte>(), msg[3].data<std::byte>() + msg[3].size()};
EventMessage em{.topic = std::string(msg[0].data<const char>(), msg[0].size()), EventMessage em{.topic = std::string(msg[0].data<const char>(), msg[0].size()),
.format = std::string(msg[2].data<const char>(), msg[2].size()), .format = std::string(msg[2].data<const char>(), msg[2].size()),
.payload = std::move(payload)}; .payload = std::move(payload)};
@ -644,7 +643,7 @@ void ZeroMQBackend::Run() {
} }
} }
bool ZeroMQBackend::DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) { bool ZeroMQBackend::DoProcessBackendMessage(int tag, byte_buffer_span payload) {
if ( tag == 0 || tag == 1 ) { if ( tag == 0 || tag == 1 ) {
std::string topic{reinterpret_cast<const char*>(payload.data()), payload.size()}; std::string topic{reinterpret_cast<const char*>(payload.data()), payload.size()};
zeek::EventHandlerPtr eh; zeek::EventHandlerPtr eh;

7
src/cluster/backend/zeromq/ZeroMQ.h
View file

@ -60,17 +60,16 @@ private:
void DoTerminate() override; void DoTerminate() override;
bool DoPublishEvent(const std::string& topic, const std::string& format, bool DoPublishEvent(const std::string& topic, const std::string& format, const byte_buffer& buf) override;
const cluster::detail::byte_buffer& buf) override;
bool DoSubscribe(const std::string& topic_prefix, SubscribeCallback cb) override; bool DoSubscribe(const std::string& topic_prefix, SubscribeCallback cb) override;
bool DoUnsubscribe(const std::string& topic_prefix) override; bool DoUnsubscribe(const std::string& topic_prefix) override;
bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format,
cluster::detail::byte_buffer& buf) override; byte_buffer& buf) override;
bool DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) override; bool DoProcessBackendMessage(int tag, byte_buffer_span payload) override;
// Script level variables. // Script level variables.
std::string connect_xsub_endpoint; std::string connect_xsub_endpoint;

6
src/cluster/serializer/binary-serialization-format/Serializer.cc
View file

@ -77,7 +77,7 @@ bool detail::BinarySerializationFormatLogSerializer::SerializeLogWrite(byte_buff
} }
std::optional<zeek::logging::detail::LogWriteBatch> detail::BinarySerializationFormatLogSerializer::UnserializeLogWrite( std::optional<zeek::logging::detail::LogWriteBatch> detail::BinarySerializationFormatLogSerializer::UnserializeLogWrite(
detail::byte_buffer_span buf) { byte_buffer_span buf) {
zeek::detail::BinarySerializationFormat fmt; zeek::detail::BinarySerializationFormat fmt;
fmt.StartRead(reinterpret_cast<const char*>(buf.data()), buf.size()); fmt.StartRead(reinterpret_cast<const char*>(buf.data()), buf.size());
@ -145,7 +145,7 @@ std::optional<zeek::logging::detail::LogWriteBatch> detail::BinarySerializationF
TEST_SUITE_BEGIN("cluster serializer binary-serialization-format"); TEST_SUITE_BEGIN("cluster serializer binary-serialization-format");
TEST_CASE("roundtrip") { TEST_CASE("roundtrip") {
detail::byte_buffer buf; zeek::byte_buffer buf;
detail::BinarySerializationFormatLogSerializer serializer; detail::BinarySerializationFormatLogSerializer serializer;
static const auto& stream_id_type = zeek::id::find_type<zeek::EnumType>("Log::ID"); static const auto& stream_id_type = zeek::id::find_type<zeek::EnumType>("Log::ID");
@ -161,7 +161,7 @@ TEST_CASE("roundtrip") {
0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00,
0x00, 0x00, 0x16, 0x01, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; 0x00, 0x00, 0x16, 0x01, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
std::byte* p = reinterpret_cast<std::byte*>(&expected_bytes[0]); std::byte* p = reinterpret_cast<std::byte*>(&expected_bytes[0]);
detail::byte_buffer expected{p, p + sizeof(expected_bytes)}; zeek::byte_buffer expected{p, p + sizeof(expected_bytes)};
auto s = stream_id_type->Lookup("Log::UNKNOWN"); auto s = stream_id_type->Lookup("Log::UNKNOWN");
REQUIRE_GE(s, 0); REQUIRE_GE(s, 0);

4
src/cluster/serializer/binary-serialization-format/Serializer.h
View file

@ -13,10 +13,10 @@ class BinarySerializationFormatLogSerializer : public cluster::LogSerializer {
public: public:
BinarySerializationFormatLogSerializer() : LogSerializer("zeek-bin-serializer") {} BinarySerializationFormatLogSerializer() : LogSerializer("zeek-bin-serializer") {}
bool SerializeLogWrite(cluster::detail::byte_buffer& buf, const logging::detail::LogWriteHeader& header, bool SerializeLogWrite(byte_buffer& buf, const logging::detail::LogWriteHeader& header,
zeek::Span<logging::detail::LogRecord> records) override; zeek::Span<logging::detail::LogRecord> records) override;
std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(detail::byte_buffer_span buf) override; std::optional<logging::detail::LogWriteBatch> UnserializeLogWrite(byte_buffer_span buf) override;
}; };
} // namespace zeek::cluster::detail } // namespace zeek::cluster::detail

10
src/cluster/serializer/broker/Serializer.cc
View file

@ -101,7 +101,7 @@ std::optional<detail::Event> detail::to_zeek_event(const broker::zeek::Event& ev
return detail::Event{handler, std::move(vl), ts}; return detail::Event{handler, std::move(vl), ts};
} }
bool detail::BrokerBinV1_Serializer::SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) { bool detail::BrokerBinV1_Serializer::SerializeEvent(byte_buffer& buf, const detail::Event& event) {
auto ev = to_broker_event(event); auto ev = to_broker_event(event);
if ( ! ev ) if ( ! ev )
return false; return false;
@ -117,7 +117,7 @@ bool detail::BrokerBinV1_Serializer::SerializeEvent(detail::byte_buffer& buf, co
return true; return true;
} }
std::optional<detail::Event> detail::BrokerBinV1_Serializer::UnserializeEvent(detail::byte_buffer_span buf) { std::optional<detail::Event> detail::BrokerBinV1_Serializer::UnserializeEvent(byte_buffer_span buf) {
auto r = broker::data_envelope::deserialize(broker::endpoint_id::nil(), broker::endpoint_id::nil(), 0, "", auto r = broker::data_envelope::deserialize(broker::endpoint_id::nil(), broker::endpoint_id::nil(), 0, "",
buf.data(), buf.size()); buf.data(), buf.size());
if ( ! r ) if ( ! r )
@ -152,7 +152,7 @@ bool detail::BrokerJsonV1_Serializer::SerializeEvent(byte_buffer& buf, const det
return true; return true;
} }
std::optional<detail::Event> detail::BrokerJsonV1_Serializer::UnserializeEvent(detail::byte_buffer_span buf) { std::optional<detail::Event> detail::BrokerJsonV1_Serializer::UnserializeEvent(byte_buffer_span buf) {
broker::variant res; broker::variant res;
auto err = auto err =
broker::format::json::v1::decode(std::string_view{reinterpret_cast<const char*>(buf.data()), buf.size()}, res); broker::format::json::v1::decode(std::string_view{reinterpret_cast<const char*>(buf.data()), buf.size()}, res);
@ -173,7 +173,7 @@ TEST_SUITE_BEGIN("cluster serializer broker");
TEST_CASE("roundtrip") { TEST_CASE("roundtrip") {
auto* handler = zeek::event_registry->Lookup("Supervisor::node_status"); auto* handler = zeek::event_registry->Lookup("Supervisor::node_status");
detail::Event e{handler, zeek::Args{zeek::make_intrusive<zeek::StringVal>("TEST"), zeek::val_mgr->Count(42)}}; detail::Event e{handler, zeek::Args{zeek::make_intrusive<zeek::StringVal>("TEST"), zeek::val_mgr->Count(42)}};
detail::byte_buffer buf; zeek::byte_buffer buf;
SUBCASE("json") { SUBCASE("json") {
detail::BrokerJsonV1_Serializer serializer; detail::BrokerJsonV1_Serializer serializer;
@ -201,7 +201,7 @@ TEST_CASE("roundtrip") {
0x01, 0x0e, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x09, 0x01, 0x0e, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x09,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
std::byte* p = reinterpret_cast<std::byte*>(&expected_bytes[0]); std::byte* p = reinterpret_cast<std::byte*>(&expected_bytes[0]);
detail::byte_buffer expected{p, p + sizeof(expected_bytes)}; zeek::byte_buffer expected{p, p + sizeof(expected_bytes)};
serializer.SerializeEvent(buf, e); serializer.SerializeEvent(buf, e);

8
src/cluster/serializer/broker/Serializer.h
View file

@ -34,9 +34,9 @@ class BrokerBinV1_Serializer : public EventSerializer {
public: public:
BrokerBinV1_Serializer() : EventSerializer("broker-bin-v1") {} BrokerBinV1_Serializer() : EventSerializer("broker-bin-v1") {}
bool SerializeEvent(detail::byte_buffer& buf, const detail::Event& event) override; bool SerializeEvent(byte_buffer& buf, const detail::Event& event) override;
std::optional<detail::Event> UnserializeEvent(detail::byte_buffer_span buf) override; std::optional<detail::Event> UnserializeEvent(byte_buffer_span buf) override;
}; };
// Implementation of the EventSerializer that uses the existing broker::detail::val_to_data() // Implementation of the EventSerializer that uses the existing broker::detail::val_to_data()
@ -45,9 +45,9 @@ class BrokerJsonV1_Serializer : public EventSerializer {
public: public:
BrokerJsonV1_Serializer() : EventSerializer("broker-json-v1") {} BrokerJsonV1_Serializer() : EventSerializer("broker-json-v1") {}
bool SerializeEvent(zeek::cluster::detail::byte_buffer& buf, const detail::Event& event) override; bool SerializeEvent(byte_buffer& buf, const detail::Event& event) override;
std::optional<detail::Event> UnserializeEvent(detail::byte_buffer_span buf) override; std::optional<detail::Event> UnserializeEvent(byte_buffer_span buf) override;
}; };
} // namespace zeek::cluster::detail } // namespace zeek::cluster::detail

10
src/storage/Serializer.h
View file

@ -2,16 +2,10 @@
#pragma once #pragma once
#include "zeek/Span.h"
#include "zeek/Val.h" #include "zeek/Val.h"
namespace zeek::storage { namespace zeek::storage {
namespace detail {
using byte_buffer = std::vector<std::byte>;
using byte_buffer_span = Span<const std::byte>;
} // namespace detail
/** /**
* Base class for a serializer used by storage backends. * Base class for a serializer used by storage backends.
*/ */
@ -27,7 +21,7 @@ public:
* @return On success, a byte buffer containing the serialized data. std::nullopt will * @return On success, a byte buffer containing the serialized data. std::nullopt will
* be returned on failure. * be returned on failure.
*/ */
virtual std::optional<detail::byte_buffer> Serialize(ValPtr val) = 0; virtual std::optional<byte_buffer> Serialize(ValPtr val) = 0;
/** /**
* Unserializes a byte buffer into Zeek Val objects of a specific type. * Unserializes a byte buffer into Zeek Val objects of a specific type.
@ -38,7 +32,7 @@ public:
* @return A zeek::expected containing either the unserialized Val data on success, or * @return A zeek::expected containing either the unserialized Val data on success, or
* a string containing an error message on failure. * a string containing an error message on failure.
*/ */
virtual zeek::expected<ValPtr, std::string> Unserialize(detail::byte_buffer_span buf, TypePtr type) = 0; virtual zeek::expected<ValPtr, std::string> Unserialize(byte_buffer_span buf, TypePtr type) = 0;
protected: protected:
Serializer(std::string name) : name(std::move(name)) {} Serializer(std::string name) : name(std::move(name)) {}

6
src/storage/serializer/json/JSON.cc
View file

@ -10,8 +10,8 @@ std::unique_ptr<Serializer> JSON::Instantiate() { return std::make_unique<JSON>(
JSON::JSON() : Serializer("JSON") {} JSON::JSON() : Serializer("JSON") {}
std::optional<detail::byte_buffer> JSON::Serialize(ValPtr val) { std::optional<byte_buffer> JSON::Serialize(ValPtr val) {
detail::byte_buffer buf; byte_buffer buf;
auto json = val->ToJSON(); auto json = val->ToJSON();
buf.reserve(json->Len()); buf.reserve(json->Len());
@ -21,7 +21,7 @@ std::optional<detail::byte_buffer> JSON::Serialize(ValPtr val) {
return buf; return buf;
} }
zeek::expected<ValPtr, std::string> JSON::Unserialize(detail::byte_buffer_span buf, TypePtr type) { zeek::expected<ValPtr, std::string> JSON::Unserialize(byte_buffer_span buf, TypePtr type) {
std::string_view text{reinterpret_cast<std::string::const_pointer>(buf.data()), buf.size()}; std::string_view text{reinterpret_cast<std::string::const_pointer>(buf.data()), buf.size()};
return zeek::detail::ValFromJSON(text, type, Func::nil); return zeek::detail::ValFromJSON(text, type, Func::nil);
} }

4
src/storage/serializer/json/JSON.h
View file

@ -13,8 +13,8 @@ public:
JSON(); JSON();
~JSON() override = default; ~JSON() override = default;
std::optional<detail::byte_buffer> Serialize(ValPtr val) override; std::optional<byte_buffer> Serialize(ValPtr val) override;
zeek::expected<ValPtr, std::string> Unserialize(detail::byte_buffer_span buf, TypePtr type) override; zeek::expected<ValPtr, std::string> Unserialize(byte_buffer_span buf, TypePtr type) override;
}; };
} // namespace zeek::storage::serializer::json } // namespace zeek::storage::serializer::json

6
src/util.h
View file

@ -104,6 +104,8 @@ template<typename E>
using unexpected = nonstd::unexpected<E>; using unexpected = nonstd::unexpected<E>;
} // namespace zeek } // namespace zeek
#include "zeek/Span.h"
using zeek_int_t = int64_t; using zeek_int_t = int64_t;
using zeek_uint_t = uint64_t; using zeek_uint_t = uint64_t;
@ -119,6 +121,10 @@ namespace zeek {
class ODesc; class ODesc;
class RecordVal; class RecordVal;
// Byte buffer types used by serialization code in storage and cluster.
using byte_buffer = std::vector<std::byte>;
using byte_buffer_span = Span<const std::byte>;
namespace util { namespace util {
namespace detail { namespace detail {

2
testing/btest/plugins/storage-plugin/src/StorageDummy.h
View file

@ -50,7 +50,7 @@ public:
zeek::storage::OperationResult DoErase(zeek::storage::ResultCallback* cb, zeek::ValPtr key) override; zeek::storage::OperationResult DoErase(zeek::storage::ResultCallback* cb, zeek::ValPtr key) override;
private: private:
std::map<zeek::storage::detail::byte_buffer, zeek::storage::detail::byte_buffer> data; std::map<zeek::byte_buffer, zeek::byte_buffer> data;
bool open = false; bool open = false;
}; };