Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Backward compatibility for OpaqueVal serialization

Browse source 
External plugins depend on the API for `OpaqueVal`. This set of changes
brings back the previous signature for the `Serialize` and `Unserialize`
member functions. The new set of functions that operate on the recently
added `BrokerData` API were renamed accordingly and use a `Data` suffix to
distinguish between the old and new interface.

For the transition period, `OpaqueVal` now has two "sets" of
serialization functions: old and new (using the suffix). By default, the
new functions call the old API and then convert to the new types. Hence,
plugins that override the old set of member functions will continue to
work. New code should only override the new set of functions.

Since the macro `DECLARE_OPAQUE_VALUE` (a convenience macro for adding a
default set of member functions to a subtype of `OpaqueVal`) might be
used by 3rd parties, the macro has been "restored" to its previous
behavior, i.e., it will override the old set of member functions. The
new macro `DECLARE_OPAQUE_VALUE_V2` is similar but overrides the new set
of functions instead.

The class `BloomFilter` uses the same member function signatures as
`OpaqueVal` for serialization. Hence, the same old/new split was
implemented to keep the APIs consistent.
This commit is contained in:
Dominik Charousset 2024-01-06 10:41:56 +01:00
parent 5ff99f7d0b
commit 1bc5fda591
14 changed files with 231 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

77
src/OpaqueVal.cc
View file

@ -70,10 +70,16 @@ OpaqueValPtr OpaqueMgr::Instantiate(const std::string& id) const {
return x != _types.end() ? (*x->second)() : nullptr; return x != _types.end() ? (*x->second)() : nullptr;
} }
std::optional<BrokerData> OpaqueVal::Serialize() const { broker::expected<broker::data> OpaqueVal::Serialize() const {
if ( auto res = SerializeData() )
return zeek::detail::BrokerDataAccess::Unbox(*res);
return {broker::make_error(broker::ec::serialization_failed)};
}
std::optional<BrokerData> OpaqueVal::SerializeData() const {
auto type = OpaqueMgr::mgr()->TypeID(this); auto type = OpaqueMgr::mgr()->TypeID(this);
auto d = DoSerialize(); auto d = DoSerializeData();
if ( ! d ) if ( ! d )
return std::nullopt; return std::nullopt;
@ -83,14 +89,16 @@ std::optional<BrokerData> OpaqueVal::Serialize() const {
return std::move(builder).Build(); return std::move(builder).Build();
} }
OpaqueValPtr OpaqueVal::Unserialize(BrokerDataView data) { OpaqueValPtr OpaqueVal::Unserialize(const broker::data& data) { return UnserializeData(BrokerDataView(&data)); }
OpaqueValPtr OpaqueVal::UnserializeData(BrokerDataView data) {
if ( ! data.IsList() ) if ( ! data.IsList() )
return nullptr; return nullptr;
return Unserialize(data.ToList()); return UnserializeData(data.ToList());
} }
OpaqueValPtr OpaqueVal::Unserialize(BrokerListView v) { OpaqueValPtr OpaqueVal::UnserializeData(BrokerListView v) {
if ( v.Size() != 2 || ! v[0].IsString() ) if ( v.Size() != 2 || ! v[0].IsString() )
return nullptr; return nullptr;
@ -100,12 +108,29 @@ OpaqueValPtr OpaqueVal::Unserialize(BrokerListView v) {
if ( ! val ) if ( ! val )
return nullptr; return nullptr;
if ( ! val->DoUnserialize(v[1]) ) if ( ! val->DoUnserializeData(v[1]) )
return nullptr; return nullptr;
return val; return val;
} }
broker::expected<broker::data> OpaqueVal::DoSerialize() const {
return {broker::make_error(broker::ec::serialization_failed)};
}
std::optional<BrokerData> OpaqueVal::DoSerializeData() const {
if ( auto res = DoSerialize() ) {
return BrokerData{std::move(*res)};
}
return std::nullopt;
}
bool OpaqueVal::DoUnserialize(const broker::data&) { return false; }
bool OpaqueVal::DoUnserializeData(BrokerDataView data) {
return DoUnserialize(zeek::detail::BrokerDataAccess::Unbox(data));
}
std::optional<BrokerData> OpaqueVal::SerializeType(const TypePtr& t) { std::optional<BrokerData> OpaqueVal::SerializeType(const TypePtr& t) {
if ( t->InternalType() == TYPE_INTERNAL_ERROR ) if ( t->InternalType() == TYPE_INTERNAL_ERROR )
return std::nullopt; return std::nullopt;
@ -158,11 +183,11 @@ TypePtr OpaqueVal::UnserializeType(BrokerDataView data) {
} }
ValPtr OpaqueVal::DoClone(CloneState* state) { ValPtr OpaqueVal::DoClone(CloneState* state) {
auto d = OpaqueVal::Serialize(); auto d = OpaqueVal::SerializeData();
if ( ! d ) if ( ! d )
return nullptr; return nullptr;
auto rval = OpaqueVal::Unserialize(d->AsView()); auto rval = OpaqueVal::UnserializeData(d->AsView());
return state->NewClone(this, std::move(rval)); return state->NewClone(this, std::move(rval));
} }
@ -440,7 +465,7 @@ StringValPtr MD5Val::DoGet() {
IMPLEMENT_OPAQUE_VALUE(MD5Val) IMPLEMENT_OPAQUE_VALUE(MD5Val)
std::optional<BrokerData> MD5Val::DoSerialize() const { std::optional<BrokerData> MD5Val::DoSerializeData() const {
BrokerListBuilder builder; BrokerListBuilder builder;
if ( ! IsValid() ) { if ( ! IsValid() ) {
@ -454,7 +479,7 @@ std::optional<BrokerData> MD5Val::DoSerialize() const {
return std::move(builder).Build(); return std::move(builder).Build();
} }
bool MD5Val::DoUnserialize(BrokerDataView data) { bool MD5Val::DoUnserializeData(BrokerDataView data) {
if ( ! data.IsList() ) if ( ! data.IsList() )
return false; return false;
auto d = data.ToList(); auto d = data.ToList();
@ -522,7 +547,7 @@ StringValPtr SHA1Val::DoGet() {
IMPLEMENT_OPAQUE_VALUE(SHA1Val) IMPLEMENT_OPAQUE_VALUE(SHA1Val)
std::optional<BrokerData> SHA1Val::DoSerialize() const { std::optional<BrokerData> SHA1Val::DoSerializeData() const {
BrokerListBuilder builder; BrokerListBuilder builder;
if ( ! IsValid() ) { if ( ! IsValid() ) {
@ -536,7 +561,7 @@ std::optional<BrokerData> SHA1Val::DoSerialize() const {
return std::move(builder).Build(); return std::move(builder).Build();
} }
bool SHA1Val::DoUnserialize(BrokerDataView data) { bool SHA1Val::DoUnserializeData(BrokerDataView data) {
if ( ! data.IsList() ) if ( ! data.IsList() )
return false; return false;
@ -608,7 +633,7 @@ StringValPtr SHA256Val::DoGet() {
IMPLEMENT_OPAQUE_VALUE(SHA256Val) IMPLEMENT_OPAQUE_VALUE(SHA256Val)
std::optional<BrokerData> SHA256Val::DoSerialize() const { std::optional<BrokerData> SHA256Val::DoSerializeData() const {
BrokerListBuilder builder; BrokerListBuilder builder;
if ( ! IsValid() ) { if ( ! IsValid() ) {
@ -621,7 +646,7 @@ std::optional<BrokerData> SHA256Val::DoSerialize() const {
return std::move(builder).Build(); return std::move(builder).Build();
} }
bool SHA256Val::DoUnserialize(BrokerDataView data) { bool SHA256Val::DoUnserializeData(BrokerDataView data) {
if ( ! data.IsList() ) if ( ! data.IsList() )
return false; return false;
@ -662,7 +687,7 @@ bool EntropyVal::Get(double* r_ent, double* r_chisq, double* r_mean, double* r_m
IMPLEMENT_OPAQUE_VALUE(EntropyVal) IMPLEMENT_OPAQUE_VALUE(EntropyVal)
std::optional<BrokerData> EntropyVal::DoSerialize() const { std::optional<BrokerData> EntropyVal::DoSerializeData() const {
constexpr size_t numMembers = 14; // RandTest has 14 non-array members. constexpr size_t numMembers = 14; // RandTest has 14 non-array members.
BrokerListBuilder builder; BrokerListBuilder builder;
@ -692,7 +717,7 @@ std::optional<BrokerData> EntropyVal::DoSerialize() const {
return std::move(builder).Build(); return std::move(builder).Build();
} }
bool EntropyVal::DoUnserialize(BrokerDataView data) { bool EntropyVal::DoUnserializeData(BrokerDataView data) {
if ( ! data.IsList() ) if ( ! data.IsList() )
return false; return false;
@ -868,7 +893,7 @@ BloomFilterVal::~BloomFilterVal() {
IMPLEMENT_OPAQUE_VALUE(BloomFilterVal) IMPLEMENT_OPAQUE_VALUE(BloomFilterVal)
std::optional<BrokerData> BloomFilterVal::DoSerialize() const { std::optional<BrokerData> BloomFilterVal::DoSerializeData() const {
BrokerListBuilder builder; BrokerListBuilder builder;
if ( type ) { if ( type ) {
@ -881,7 +906,7 @@ std::optional<BrokerData> BloomFilterVal::DoSerialize() const {
else else
builder.AddNil(); builder.AddNil();
auto bf = bloom_filter->Serialize(); auto bf = bloom_filter->SerializeData();
if ( ! bf ) if ( ! bf )
return std::nullopt; return std::nullopt;
@ -889,7 +914,7 @@ std::optional<BrokerData> BloomFilterVal::DoSerialize() const {
return std::move(builder).Build(); return std::move(builder).Build();
} }
bool BloomFilterVal::DoUnserialize(BrokerDataView data) { bool BloomFilterVal::DoUnserializeData(BrokerDataView data) {
if ( ! data.IsList() ) if ( ! data.IsList() )
return false; return false;
@ -905,7 +930,7 @@ bool BloomFilterVal::DoUnserialize(BrokerDataView data) {
return false; return false;
} }
auto bf = probabilistic::BloomFilter::Unserialize(v[1]); auto bf = probabilistic::BloomFilter::UnserializeData(v[1]);
if ( ! bf ) if ( ! bf )
return false; return false;
@ -952,7 +977,7 @@ void CardinalityVal::Add(const Val* val) {
IMPLEMENT_OPAQUE_VALUE(CardinalityVal) IMPLEMENT_OPAQUE_VALUE(CardinalityVal)
std::optional<BrokerData> CardinalityVal::DoSerialize() const { std::optional<BrokerData> CardinalityVal::DoSerializeData() const {
BrokerListBuilder builder; BrokerListBuilder builder;
builder.Reserve(2); builder.Reserve(2);
@ -974,7 +999,7 @@ std::optional<BrokerData> CardinalityVal::DoSerialize() const {
return std::move(builder).Build(); return std::move(builder).Build();
} }
bool CardinalityVal::DoUnserialize(BrokerDataView data) { bool CardinalityVal::DoUnserializeData(BrokerDataView data) {
if ( ! data.IsList() ) if ( ! data.IsList() )
return false; return false;
@ -1019,7 +1044,7 @@ bool ParaglobVal::operator==(const ParaglobVal& other) const {
IMPLEMENT_OPAQUE_VALUE(ParaglobVal) IMPLEMENT_OPAQUE_VALUE(ParaglobVal)
std::optional<BrokerData> ParaglobVal::DoSerialize() const { std::optional<BrokerData> ParaglobVal::DoSerializeData() const {
std::unique_ptr<std::vector<uint8_t>> iv = this->internal_paraglob->serialize(); std::unique_ptr<std::vector<uint8_t>> iv = this->internal_paraglob->serialize();
BrokerListBuilder builder; BrokerListBuilder builder;
builder.Reserve(iv->size()); builder.Reserve(iv->size());
@ -1028,7 +1053,7 @@ std::optional<BrokerData> ParaglobVal::DoSerialize() const {
return std::move(builder).Build(); return std::move(builder).Build();
} }
bool ParaglobVal::DoUnserialize(BrokerDataView data) { bool ParaglobVal::DoUnserializeData(BrokerDataView data) {
if ( ! data.IsList() ) if ( ! data.IsList() )
return false; return false;
@ -1067,9 +1092,9 @@ ValPtr ParaglobVal::DoClone(CloneState* state) {
} }
} }
std::optional<BrokerData> TelemetryVal::DoSerialize() const { return std::nullopt; } std::optional<BrokerData> TelemetryVal::DoSerializeData() const { return std::nullopt; }
bool TelemetryVal::DoUnserialize(BrokerDataView) { return false; } bool TelemetryVal::DoUnserializeData(BrokerDataView) { return false; }
TelemetryVal::TelemetryVal(telemetry::IntCounter) : OpaqueVal(int_counter_metric_type) {} TelemetryVal::TelemetryVal(telemetry::IntCounter) : OpaqueVal(int_counter_metric_type) {}

75
src/OpaqueVal.h
View file

@ -6,6 +6,7 @@
#include <unistd.h> #include <unistd.h>
#endif #endif
#include <broker/expected.hh>
#include <paraglob/paraglob.h> #include <paraglob/paraglob.h>
#include <sys/types.h> // for u_char #include <sys/types.h> // for u_char
#include <optional> #include <optional>
@ -18,6 +19,10 @@
#include "zeek/telemetry/Gauge.h" #include "zeek/telemetry/Gauge.h"
#include "zeek/telemetry/Histogram.h" #include "zeek/telemetry/Histogram.h"
namespace broker {
class data;
}
namespace zeek { namespace zeek {
class BrokerData; class BrokerData;
@ -86,12 +91,28 @@ private:
std::unordered_map<std::string, Factory*> _types; std::unordered_map<std::string, Factory*> _types;
}; };
/** Macro to insert into an OpaqueVal-derived class's declaration. */ /**
* Legacy macro to insert into an OpaqueVal-derived class's declaration. Overrides the "old" serialization methods
* DoSerialize and DoUnserialize.
* @deprecated Use DECLARE_OPAQUE_VALUE_V2 instead.
*/
#define DECLARE_OPAQUE_VALUE(T) \ #define DECLARE_OPAQUE_VALUE(T) \
friend class zeek::OpaqueMgr::Register<T>; \ friend class zeek::OpaqueMgr::Register<T>; \
friend zeek::IntrusivePtr<T> zeek::make_intrusive<T>(); \ friend zeek::IntrusivePtr<T> zeek::make_intrusive<T>(); \
std::optional<BrokerData> DoSerialize() const override; \ broker::expected<broker::data> DoSerialize() const override; \
bool DoUnserialize(BrokerDataView data) override; \ bool DoUnserialize(const broker::data& data) override; \
const char* OpaqueName() const override { return #T; } \
static zeek::OpaqueValPtr OpaqueInstantiate() { return zeek::make_intrusive<T>(); }
/**
* Macro to insert into an OpaqueVal-derived class's declaration. Overrides the "new" serialization methods
* DoSerializeData and DoUnserializeData.
*/
#define DECLARE_OPAQUE_VALUE_V2(T) \
friend class zeek::OpaqueMgr::Register<T>; \
friend zeek::IntrusivePtr<T> zeek::make_intrusive<T>(); \
std::optional<zeek::BrokerData> DoSerializeData() const override; \
bool DoUnserializeData(zeek::BrokerDataView data) override; \
const char* OpaqueName() const override { return #T; } \ const char* OpaqueName() const override { return #T; } \
static zeek::OpaqueValPtr OpaqueInstantiate() { return zeek::make_intrusive<T>(); } static zeek::OpaqueValPtr OpaqueInstantiate() { return zeek::make_intrusive<T>(); }
@ -117,7 +138,12 @@ public:
* @return the broker representation, or an error if serialization * @return the broker representation, or an error if serialization
* isn't supported or failed. * isn't supported or failed.
*/ */
std::optional<BrokerData> Serialize() const; [[deprecated("use SerializeData instead")]] broker::expected<broker::data> Serialize() const;
/**
* @copydoc Serialize
*/
std::optional<BrokerData> SerializeData() const;
/** /**
* Reinstantiates a value from its serialized Broker representation. * Reinstantiates a value from its serialized Broker representation.
@ -125,17 +151,27 @@ public:
* @param data Broker representation as returned by *Serialize()*. * @param data Broker representation as returned by *Serialize()*.
* @return unserialized instances with reference count at +1 * @return unserialized instances with reference count at +1
*/ */
static OpaqueValPtr Unserialize(BrokerDataView data); [[deprecated("use UnserializeData instead")]] static OpaqueValPtr Unserialize(const broker::data& data);
/** /**
* @copydoc Unserialize * @copydoc Unserialize
*/ */
static OpaqueValPtr Unserialize(BrokerListView data); static OpaqueValPtr UnserializeData(BrokerDataView data);
/**
* @copydoc Unserialize
*/
static OpaqueValPtr UnserializeData(BrokerListView data);
protected: protected:
friend class Val; friend class Val;
friend class OpaqueMgr; friend class OpaqueMgr;
/**
* @deprecated Override DoSerializeData instead.
*/
virtual broker::expected<broker::data> DoSerialize() const;
/** /**
* Must be overridden to provide a serialized version of the derived * Must be overridden to provide a serialized version of the derived
* class' state. * class' state.
@ -143,7 +179,12 @@ protected:
* @return the serialized data or an error if serialization * @return the serialized data or an error if serialization
* isn't supported or failed. * isn't supported or failed.
*/ */
virtual std::optional<BrokerData> DoSerialize() const = 0; virtual std::optional<BrokerData> DoSerializeData() const;
/**
* @deprecated Override DoUnserializeData instead.
*/
virtual bool DoUnserialize(const broker::data& data);
/** /**
* Must be overridden to recreate the derived class' state from a * Must be overridden to recreate the derived class' state from a
@ -151,7 +192,7 @@ protected:
* *
* @return true if successful. * @return true if successful.
*/ */
virtual bool DoUnserialize(BrokerDataView data) = 0; virtual bool DoUnserializeData(BrokerDataView data);
/** /**
* Internal helper for the serialization machinery. Automatically * Internal helper for the serialization machinery. Automatically
@ -248,7 +289,7 @@ protected:
bool DoFeed(const void* data, size_t size) override; bool DoFeed(const void* data, size_t size) override;
StringValPtr DoGet() override; StringValPtr DoGet() override;
DECLARE_OPAQUE_VALUE(MD5Val) DECLARE_OPAQUE_VALUE_V2(MD5Val)
private: private:
StatePtr ctx = nullptr; StatePtr ctx = nullptr;
}; };
@ -276,7 +317,7 @@ protected:
bool DoFeed(const void* data, size_t size) override; bool DoFeed(const void* data, size_t size) override;
StringValPtr DoGet() override; StringValPtr DoGet() override;
DECLARE_OPAQUE_VALUE(SHA1Val) DECLARE_OPAQUE_VALUE_V2(SHA1Val)
private: private:
StatePtr ctx = nullptr; StatePtr ctx = nullptr;
}; };
@ -304,7 +345,7 @@ protected:
bool DoFeed(const void* data, size_t size) override; bool DoFeed(const void* data, size_t size) override;
StringValPtr DoGet() override; StringValPtr DoGet() override;
DECLARE_OPAQUE_VALUE(SHA256Val) DECLARE_OPAQUE_VALUE_V2(SHA256Val)
private: private:
StatePtr ctx = nullptr; StatePtr ctx = nullptr;
}; };
@ -319,7 +360,7 @@ public:
protected: protected:
friend class Val; friend class Val;
DECLARE_OPAQUE_VALUE(EntropyVal) DECLARE_OPAQUE_VALUE_V2(EntropyVal)
private: private:
detail::RandTest state; detail::RandTest state;
}; };
@ -349,7 +390,7 @@ protected:
friend class Val; friend class Val;
BloomFilterVal(); BloomFilterVal();
DECLARE_OPAQUE_VALUE(BloomFilterVal) DECLARE_OPAQUE_VALUE_V2(BloomFilterVal)
private: private:
// Disable. // Disable.
BloomFilterVal(const BloomFilterVal&); BloomFilterVal(const BloomFilterVal&);
@ -378,7 +419,7 @@ public:
protected: protected:
CardinalityVal(); CardinalityVal();
DECLARE_OPAQUE_VALUE(CardinalityVal) DECLARE_OPAQUE_VALUE_V2(CardinalityVal)
private: private:
TypePtr type; TypePtr type;
detail::CompositeHash* hash; detail::CompositeHash* hash;
@ -395,7 +436,7 @@ public:
protected: protected:
ParaglobVal() : OpaqueVal(paraglob_type) {} ParaglobVal() : OpaqueVal(paraglob_type) {}
DECLARE_OPAQUE_VALUE(ParaglobVal) DECLARE_OPAQUE_VALUE_V2(ParaglobVal)
private: private:
std::unique_ptr<paraglob::Paraglob> internal_paraglob; std::unique_ptr<paraglob::Paraglob> internal_paraglob;
@ -419,8 +460,8 @@ protected:
explicit TelemetryVal(telemetry::DblHistogram); explicit TelemetryVal(telemetry::DblHistogram);
explicit TelemetryVal(telemetry::DblHistogramFamily); explicit TelemetryVal(telemetry::DblHistogramFamily);
std::optional<BrokerData> DoSerialize() const override; std::optional<BrokerData> DoSerializeData() const override;
bool DoUnserialize(BrokerDataView data) override; bool DoUnserializeData(BrokerDataView data) override;
}; };
template<class Handle> template<class Handle>

47
src/broker/Data.cc
View file

@ -38,17 +38,6 @@ TEST_CASE("converting Zeek to Broker protocol constants") {
CHECK_EQ(to_broker_port_proto(TRANSPORT_UNKNOWN), broker::port::protocol::unknown); CHECK_EQ(to_broker_port_proto(TRANSPORT_UNKNOWN), broker::port::protocol::unknown);
} }
namespace zeek {
class BrokerDataAccess {
public:
static broker::data& Unbox(BrokerData& data) { return data.value_; }
static broker::data&& Unbox(BrokerData&& data) { return std::move(data.value_); }
};
} // namespace zeek
namespace zeek::Broker::detail { namespace zeek::Broker::detail {
// Returns true if the given Zeek type is serialized as a broker::vector // Returns true if the given Zeek type is serialized as a broker::vector
@ -426,7 +415,7 @@ struct val_converter {
return rval; return rval;
} }
else if ( type->Tag() == TYPE_OPAQUE ) else if ( type->Tag() == TYPE_OPAQUE )
return OpaqueVal::Unserialize(BrokerListView{&a}); return OpaqueVal::UnserializeData(BrokerListView{&a});
return nullptr; return nullptr;
} }
@ -702,7 +691,7 @@ struct type_checker {
// TODO: Could avoid doing the full unserialization here // TODO: Could avoid doing the full unserialization here
// and just check if the type is a correct match. // and just check if the type is a correct match.
auto cpy = a; auto cpy = a;
auto ov = OpaqueVal::Unserialize(BrokerListView{&cpy}); auto ov = OpaqueVal::UnserializeData(BrokerListView{&cpy});
return ov != nullptr; return ov != nullptr;
} }
@ -784,7 +773,8 @@ std::optional<broker::data> val_to_data(const Val* v) {
if ( ! bc ) if ( ! bc )
return std::nullopt; return std::nullopt;
rval.emplace_back(std::move(BrokerDataAccess::Unbox(*bc))); auto& raw = zeek::detail::BrokerDataAccess::Unbox(*bc);
rval.emplace_back(std::move(raw));
} }
else { else {
reporter->InternalWarning("Closure with non-ScriptFunc"); reporter->InternalWarning("Closure with non-ScriptFunc");
@ -916,13 +906,13 @@ std::optional<broker::data> val_to_data(const Val* v) {
return {std::move(rval)}; return {std::move(rval)};
} }
case TYPE_OPAQUE: { case TYPE_OPAQUE: {
auto c = v->AsOpaqueVal()->Serialize(); auto c = v->AsOpaqueVal()->SerializeData();
if ( ! c ) { if ( ! c ) {
reporter->Error("unsupported opaque type for serialization"); reporter->Error("unsupported opaque type for serialization");
break; break;
} }
return {BrokerDataAccess::Unbox(std::move(*c))}; return {zeek::detail::BrokerDataAccess::Unbox(std::move(*c))};
} }
default: reporter->Error("unsupported Broker::Data type: %s", type_name(v->GetType()->Tag())); break; default: reporter->Error("unsupported Broker::Data type: %s", type_name(v->GetType()->Tag())); break;
} }
@ -1037,20 +1027,20 @@ const TypePtr& DataVal::ScriptDataType() {
IMPLEMENT_OPAQUE_VALUE(zeek::Broker::detail::DataVal) IMPLEMENT_OPAQUE_VALUE(zeek::Broker::detail::DataVal)
std::optional<BrokerData> DataVal::DoSerialize() const { return BrokerData{data}; } std::optional<BrokerData> DataVal::DoSerializeData() const { return BrokerData{data}; }
bool DataVal::DoUnserialize(BrokerDataView dv) { bool DataVal::DoUnserializeData(BrokerDataView dv) {
data = std::move(*dv.value_); data = std::move(*dv.value_);
return true; return true;
} }
IMPLEMENT_OPAQUE_VALUE(zeek::Broker::detail::SetIterator) IMPLEMENT_OPAQUE_VALUE(zeek::Broker::detail::SetIterator)
std::optional<BrokerData> SetIterator::DoSerialize() const { std::optional<BrokerData> SetIterator::DoSerializeData() const {
return BrokerData{broker::data{broker::vector{dat, *it}}}; return BrokerData{broker::data{broker::vector{dat, *it}}};
} }
bool SetIterator::DoUnserialize(BrokerDataView data) { bool SetIterator::DoUnserializeData(BrokerDataView data) {
auto v = get_if<broker::vector>(data.value_); auto v = get_if<broker::vector>(data.value_);
if ( ! (v && v->size() == 2) ) if ( ! (v && v->size() == 2) )
return false; return false;
@ -1071,11 +1061,11 @@ bool SetIterator::DoUnserialize(BrokerDataView data) {
IMPLEMENT_OPAQUE_VALUE(zeek::Broker::detail::TableIterator) IMPLEMENT_OPAQUE_VALUE(zeek::Broker::detail::TableIterator)
std::optional<BrokerData> TableIterator::DoSerialize() const { std::optional<BrokerData> TableIterator::DoSerializeData() const {
return BrokerData{broker::data{broker::vector{dat, it->first}}}; return BrokerData{broker::data{broker::vector{dat, it->first}}};
} }
bool TableIterator::DoUnserialize(BrokerDataView data) { bool TableIterator::DoUnserializeData(BrokerDataView data) {
auto v = get_if<broker::vector>(data.value_); auto v = get_if<broker::vector>(data.value_);
if ( ! (v && v->size() == 2) ) if ( ! (v && v->size() == 2) )
return false; return false;
@ -1096,12 +1086,12 @@ bool TableIterator::DoUnserialize(BrokerDataView data) {
IMPLEMENT_OPAQUE_VALUE(zeek::Broker::detail::VectorIterator) IMPLEMENT_OPAQUE_VALUE(zeek::Broker::detail::VectorIterator)
std::optional<BrokerData> VectorIterator::DoSerialize() const { std::optional<BrokerData> VectorIterator::DoSerializeData() const {
broker::integer difference = it - dat.begin(); broker::integer difference = it - dat.begin();
return BrokerData{broker::data{broker::vector{dat, difference}}}; return BrokerData{broker::data{broker::vector{dat, difference}}};
} }
bool VectorIterator::DoUnserialize(BrokerDataView data) { bool VectorIterator::DoUnserializeData(BrokerDataView data) {
auto v = get_if<broker::vector>(data.value_); auto v = get_if<broker::vector>(data.value_);
if ( ! (v && v->size() == 2) ) if ( ! (v && v->size() == 2) )
return false; return false;
@ -1119,12 +1109,12 @@ bool VectorIterator::DoUnserialize(BrokerDataView data) {
IMPLEMENT_OPAQUE_VALUE(zeek::Broker::detail::RecordIterator) IMPLEMENT_OPAQUE_VALUE(zeek::Broker::detail::RecordIterator)
std::optional<BrokerData> RecordIterator::DoSerialize() const { std::optional<BrokerData> RecordIterator::DoSerializeData() const {
auto difference = static_cast<broker::integer>(it - dat.begin()); auto difference = static_cast<broker::integer>(it - dat.begin());
return BrokerData{broker::data{broker::vector{dat, difference}}}; return BrokerData{broker::data{broker::vector{dat, difference}}};
} }
bool RecordIterator::DoUnserialize(BrokerDataView data) { bool RecordIterator::DoUnserializeData(BrokerDataView data) {
auto v = get_if<broker::vector>(data.value_); auto v = get_if<broker::vector>(data.value_);
if ( ! (v && v->size() == 2) ) if ( ! (v && v->size() == 2) )
return false; return false;
@ -1187,7 +1177,10 @@ BrokerListView BrokerDataView::ToList() noexcept {
return BrokerListView{std::addressof(broker::get<broker::vector>(*value_))}; return BrokerListView{std::addressof(broker::get<broker::vector>(*value_))};
} }
ValPtr BrokerDataView::ToVal(Type* type) { return zeek::Broker::detail::data_to_val(*value_, type); } ValPtr BrokerDataView::ToVal(Type* type) {
auto cpy = *value_;
return zeek::Broker::detail::data_to_val(cpy, type);
}
bool BrokerData::Convert(const Val* value) { bool BrokerData::Convert(const Val* value) {
if ( auto res = zeek::Broker::detail::val_to_data(value) ) { if ( auto res = zeek::Broker::detail::val_to_data(value) ) {

52
src/broker/Data.h
View file

@ -125,7 +125,7 @@ public:
protected: protected:
DataVal() : OpaqueVal(opaque_of_data_type) {} DataVal() : OpaqueVal(opaque_of_data_type) {}
DECLARE_OPAQUE_VALUE(zeek::Broker::detail::DataVal) DECLARE_OPAQUE_VALUE_V2(zeek::Broker::detail::DataVal)
}; };
/** /**
@ -222,7 +222,7 @@ public:
protected: protected:
SetIterator() : zeek::OpaqueVal(opaque_of_set_iterator) {} SetIterator() : zeek::OpaqueVal(opaque_of_set_iterator) {}
DECLARE_OPAQUE_VALUE(zeek::Broker::detail::SetIterator) DECLARE_OPAQUE_VALUE_V2(zeek::Broker::detail::SetIterator)
}; };
class TableIterator : public zeek::OpaqueVal { class TableIterator : public zeek::OpaqueVal {
@ -238,7 +238,7 @@ public:
protected: protected:
TableIterator() : zeek::OpaqueVal(opaque_of_table_iterator) {} TableIterator() : zeek::OpaqueVal(opaque_of_table_iterator) {}
DECLARE_OPAQUE_VALUE(zeek::Broker::detail::TableIterator) DECLARE_OPAQUE_VALUE_V2(zeek::Broker::detail::TableIterator)
}; };
class VectorIterator : public zeek::OpaqueVal { class VectorIterator : public zeek::OpaqueVal {
@ -254,7 +254,7 @@ public:
protected: protected:
VectorIterator() : zeek::OpaqueVal(opaque_of_vector_iterator) {} VectorIterator() : zeek::OpaqueVal(opaque_of_vector_iterator) {}
DECLARE_OPAQUE_VALUE(zeek::Broker::detail::VectorIterator) DECLARE_OPAQUE_VALUE_V2(zeek::Broker::detail::VectorIterator)
}; };
class RecordIterator : public zeek::OpaqueVal { class RecordIterator : public zeek::OpaqueVal {
@ -270,20 +270,33 @@ public:
protected: protected:
RecordIterator() : zeek::OpaqueVal(opaque_of_record_iterator) {} RecordIterator() : zeek::OpaqueVal(opaque_of_record_iterator) {}
DECLARE_OPAQUE_VALUE(zeek::Broker::detail::RecordIterator) DECLARE_OPAQUE_VALUE_V2(zeek::Broker::detail::RecordIterator)
}; };
} // namespace zeek::Broker::detail } // namespace zeek::Broker::detail
namespace zeek { namespace zeek {
class BrokerData;
class BrokerDataView;
class BrokerListView; class BrokerListView;
} // namespace zeek
namespace zeek::detail {
class BrokerDataAccess;
} // namespace zeek::detail
namespace zeek {
/** /**
* Non-owning reference (view) to a Broker data value. * Non-owning reference (view) to a Broker data value.
*/ */
class BrokerDataView { class BrokerDataView {
public: public:
friend class zeek::detail::BrokerDataAccess;
friend class zeek::Broker::detail::DataVal; friend class zeek::Broker::detail::DataVal;
friend class zeek::Broker::detail::SetIterator; friend class zeek::Broker::detail::SetIterator;
friend class zeek::Broker::detail::TableIterator; friend class zeek::Broker::detail::TableIterator;
@ -294,7 +307,7 @@ public:
BrokerDataView(const BrokerDataView&) noexcept = default; BrokerDataView(const BrokerDataView&) noexcept = default;
explicit BrokerDataView(broker::data* value) noexcept : value_(value) { assert(value != nullptr); } explicit BrokerDataView(const broker::data* value) noexcept : value_(value) { assert(value != nullptr); }
/** /**
* Checks whether the value represents the `nil` value. * Checks whether the value represents the `nil` value.
@ -399,7 +412,7 @@ public:
friend std::string to_string(const BrokerDataView& data) { return broker::to_string(*data.value_); } friend std::string to_string(const BrokerDataView& data) { return broker::to_string(*data.value_); }
private: private:
broker::data* value_; const broker::data* value_;
}; };
/** /**
@ -423,11 +436,13 @@ template<typename... Args>
*/ */
class BrokerListView { class BrokerListView {
public: public:
friend class zeek::detail::BrokerDataAccess;
BrokerListView() = delete; BrokerListView() = delete;
BrokerListView(const BrokerListView&) noexcept = default; BrokerListView(const BrokerListView&) noexcept = default;
explicit BrokerListView(broker::vector* values) noexcept : values_(values) { assert(values != nullptr); } explicit BrokerListView(const broker::vector* values) noexcept : values_(values) { assert(values != nullptr); }
/** /**
* Returns a view to the first element. * Returns a view to the first element.
@ -460,11 +475,9 @@ public:
[[nodiscard]] size_t IsEmpty() const noexcept { return values_->empty(); } [[nodiscard]] size_t IsEmpty() const noexcept { return values_->empty(); }
private: private:
broker::vector* values_; const broker::vector* values_;
}; };
class BrokerDataAccess;
class BrokerListBuilder; class BrokerListBuilder;
/** /**
@ -472,10 +485,10 @@ class BrokerListBuilder;
*/ */
class BrokerData { class BrokerData {
public: public:
friend class BrokerDataAccess;
friend class BrokerListBuilder; friend class BrokerListBuilder;
friend class zeek::Broker::Manager; friend class zeek::Broker::Manager;
friend class zeek::Broker::detail::StoreHandleVal; friend class zeek::Broker::detail::StoreHandleVal;
friend class zeek::detail::BrokerDataAccess;
BrokerData() = default; BrokerData() = default;
@ -651,3 +664,18 @@ private:
}; };
} // namespace zeek } // namespace zeek
namespace zeek::detail {
class BrokerDataAccess {
public:
static broker::data& Unbox(BrokerData& data) { return data.value_; }
static const broker::data& Unbox(const BrokerData& data) { return data.value_; }
static broker::data&& Unbox(BrokerData&& data) { return std::move(data.value_); }
static const broker::data& Unbox(const BrokerDataView& data) { return *data.value_; }
};
} // namespace zeek::detail

4
src/broker/Store.cc
View file

@ -40,12 +40,12 @@ void StoreHandleVal::ValDescribe(ODesc* d) const {
IMPLEMENT_OPAQUE_VALUE(StoreHandleVal) IMPLEMENT_OPAQUE_VALUE(StoreHandleVal)
std::optional<BrokerData> StoreHandleVal::DoSerialize() const { std::optional<BrokerData> StoreHandleVal::DoSerializeData() const {
// Cannot serialize. // Cannot serialize.
return std::nullopt; return std::nullopt;
} }
bool StoreHandleVal::DoUnserialize(BrokerDataView) { bool StoreHandleVal::DoUnserializeData(BrokerDataView) {
// Cannot unserialize. // Cannot unserialize.
return false; return false;
} }

2
src/broker/Store.h
View file

@ -127,7 +127,7 @@ public:
protected: protected:
IntrusivePtr<Val> DoClone(CloneState* state) override { return {NewRef{}, this}; } IntrusivePtr<Val> DoClone(CloneState* state) override { return {NewRef{}, this}; }
DECLARE_OPAQUE_VALUE(StoreHandleVal) DECLARE_OPAQUE_VALUE_V2(StoreHandleVal)
}; };
// Helper function to construct a broker backend type from script land. // Helper function to construct a broker backend type from script land.

4
src/broker/data.bif
View file

@ -44,7 +44,7 @@ function Broker::__data_type%(d: Broker::Data%): Broker::DataType
# For testing only. # For testing only.
function Broker::__opaque_clone_through_serialization%(d: any%): any function Broker::__opaque_clone_through_serialization%(d: any%): any
%{ %{
auto x = d->AsOpaqueVal()->Serialize(); auto x = d->AsOpaqueVal()->SerializeData();
if ( ! x ) if ( ! x )
{ {
@ -52,7 +52,7 @@ function Broker::__opaque_clone_through_serialization%(d: any%): any
return zeek::val_mgr->False(); return zeek::val_mgr->False();
} }
return OpaqueVal::Unserialize(x->AsView()); return OpaqueVal::UnserializeData(x->AsView());
%} %}
function Broker::__set_create%(%): Broker::Data function Broker::__set_create%(%): Broker::Data

4
src/file_analysis/analyzer/x509/X509.cc
View file

@ -552,7 +552,7 @@ ValPtr X509Val::DoClone(CloneState* state) {
IMPLEMENT_OPAQUE_VALUE(X509Val) IMPLEMENT_OPAQUE_VALUE(X509Val)
std::optional<BrokerData> X509Val::DoSerialize() const { std::optional<BrokerData> X509Val::DoSerializeData() const {
unsigned char* buf = nullptr; unsigned char* buf = nullptr;
int length = i2d_X509(certificate, &buf); int length = i2d_X509(certificate, &buf);
@ -565,7 +565,7 @@ std::optional<BrokerData> X509Val::DoSerialize() const {
return std::move(result); return std::move(result);
} }
bool X509Val::DoUnserialize(BrokerDataView data) { bool X509Val::DoUnserializeData(BrokerDataView data) {
if ( ! data.IsString() ) if ( ! data.IsString() )
return false; return false;

2
src/file_analysis/analyzer/x509/X509.h
View file

@ -184,7 +184,7 @@ protected:
*/ */
X509Val(); X509Val();
DECLARE_OPAQUE_VALUE(X509Val) DECLARE_OPAQUE_VALUE_V2(X509Val)
private: private:
::X509* certificate; // the wrapped certificate ::X509* certificate; // the wrapped certificate
}; };

6
src/logging/Manager.cc
View file

@ -56,7 +56,7 @@ public:
protected: protected:
explicit LogDelayTokenVal() : LogDelayTokenVal(0) {} explicit LogDelayTokenVal() : LogDelayTokenVal(0) {}
DECLARE_OPAQUE_VALUE(LogDelayTokenVal) DECLARE_OPAQUE_VALUE_V2(LogDelayTokenVal)
private: private:
DelayTokenType token; DelayTokenType token;
@ -67,9 +67,9 @@ ValPtr LogDelayTokenVal::DoClone(CloneState* state) {
} }
// Delay tokens are only valid on the same worker. // Delay tokens are only valid on the same worker.
std::optional<BrokerData> LogDelayTokenVal::DoSerialize() const { return std::nullopt; } std::optional<BrokerData> LogDelayTokenVal::DoSerializeData() const { return std::nullopt; }
bool LogDelayTokenVal::DoUnserialize(BrokerDataView) { return false; } bool LogDelayTokenVal::DoUnserializeData(BrokerDataView) { return false; }
IMPLEMENT_OPAQUE_VALUE(LogDelayTokenVal) IMPLEMENT_OPAQUE_VALUE(LogDelayTokenVal)

43
src/probabilistic/BloomFilter.cc
View file

@ -18,13 +18,19 @@ BloomFilter::BloomFilter(const detail::Hasher* arg_hasher) { hasher = arg_hasher
BloomFilter::~BloomFilter() { delete hasher; } BloomFilter::~BloomFilter() { delete hasher; }
std::optional<BrokerData> BloomFilter::Serialize() const { broker::expected<broker::data> BloomFilter::Serialize() const {
if ( auto res = SerializeData() )
return zeek::detail::BrokerDataAccess::Unbox(*res);
return {broker::make_error(broker::ec::serialization_failed)};
}
std::optional<BrokerData> BloomFilter::SerializeData() const {
auto h = hasher->Serialize(); auto h = hasher->Serialize();
if ( ! h ) if ( ! h )
return std::nullopt; // Cannot serialize return std::nullopt; // Cannot serialize
auto d = DoSerialize(); auto d = DoSerializeData();
if ( ! d ) if ( ! d )
return std::nullopt; // Cannot serialize return std::nullopt; // Cannot serialize
@ -37,7 +43,11 @@ std::optional<BrokerData> BloomFilter::Serialize() const {
return std::move(builder).Build(); return std::move(builder).Build();
} }
std::unique_ptr<BloomFilter> BloomFilter::Unserialize(BrokerDataView data) { std::unique_ptr<BloomFilter> BloomFilter::Unserialize(const broker::data& data) {
return UnserializeData(BrokerDataView{&data});
}
std::unique_ptr<BloomFilter> BloomFilter::UnserializeData(BrokerDataView data) {
if ( ! data.IsList() ) if ( ! data.IsList() )
return nullptr; return nullptr;
@ -56,7 +66,7 @@ std::unique_ptr<BloomFilter> BloomFilter::Unserialize(BrokerDataView data) {
default: reporter->Error("found invalid bloom filter type"); return nullptr; default: reporter->Error("found invalid bloom filter type"); return nullptr;
} }
if ( ! bf->DoUnserialize(v[2]) ) if ( ! bf->DoUnserializeData(v[2]) )
return nullptr; return nullptr;
bf->hasher = detail::Hasher::Unserialize(v[1]).release(); bf->hasher = detail::Hasher::Unserialize(v[1]).release();
@ -67,6 +77,23 @@ std::unique_ptr<BloomFilter> BloomFilter::Unserialize(BrokerDataView data) {
return bf; return bf;
} }
broker::expected<broker::data> BloomFilter::DoSerialize() const {
return {broker::make_error(broker::ec::serialization_failed)};
}
bool BloomFilter::DoUnserialize(const broker::data&) { return false; }
std::optional<BrokerData> BloomFilter::DoSerializeData() const {
if ( auto res = DoSerialize() ) {
return BrokerData{std::move(*res)};
}
return std::nullopt;
}
bool BloomFilter::DoUnserializeData(BrokerDataView data) {
return DoUnserialize(zeek::detail::BrokerDataAccess::Unbox(data));
}
size_t BasicBloomFilter::M(double fp, size_t capacity) { size_t BasicBloomFilter::M(double fp, size_t capacity) {
double ln2 = std::log(2); double ln2 = std::log(2);
return std::ceil(-(capacity * std::log(fp) / ln2 / ln2)); return std::ceil(-(capacity * std::log(fp) / ln2 / ln2));
@ -166,9 +193,9 @@ size_t BasicBloomFilter::Count(const zeek::detail::HashKey* key) const {
return 1; return 1;
} }
std::optional<BrokerData> BasicBloomFilter::DoSerialize() const { return bits->Serialize(); } std::optional<BrokerData> BasicBloomFilter::DoSerializeData() const { return bits->Serialize(); }
bool BasicBloomFilter::DoUnserialize(BrokerDataView data) { bool BasicBloomFilter::DoUnserializeData(BrokerDataView data) {
auto b = detail::BitVector::Unserialize(data); auto b = detail::BitVector::Unserialize(data);
if ( ! b ) if ( ! b )
return false; return false;
@ -280,9 +307,9 @@ size_t CountingBloomFilter::Count(const zeek::detail::HashKey* key) const {
return min; return min;
} }
std::optional<BrokerData> CountingBloomFilter::DoSerialize() const { return cells->Serialize(); } std::optional<BrokerData> CountingBloomFilter::DoSerializeData() const { return cells->Serialize(); }
bool CountingBloomFilter::DoUnserialize(BrokerDataView data) { bool CountingBloomFilter::DoUnserializeData(BrokerDataView data) {
auto c = detail::CounterVector::Unserialize(data); auto c = detail::CounterVector::Unserialize(data);
if ( ! c ) if ( ! c )
return false; return false;

21
src/probabilistic/BloomFilter.h
View file

@ -105,8 +105,11 @@ public:
*/ */
virtual std::string InternalState() const = 0; virtual std::string InternalState() const = 0;
std::optional<BrokerData> Serialize() const; [[deprecated("use SerializeData instead")]] broker::expected<broker::data> Serialize() const;
static std::unique_ptr<BloomFilter> Unserialize(BrokerDataView data); [[deprecated("use UnserializeData instead")]] static std::unique_ptr<BloomFilter> Unserialize(
const broker::data& data);
std::optional<BrokerData> SerializeData() const;
static std::unique_ptr<BloomFilter> UnserializeData(BrokerDataView data);
protected: protected:
/** /**
@ -121,8 +124,10 @@ protected:
*/ */
explicit BloomFilter(const detail::Hasher* hasher); explicit BloomFilter(const detail::Hasher* hasher);
virtual std::optional<BrokerData> DoSerialize() const = 0; virtual broker::expected<broker::data> DoSerialize() const;
virtual bool DoUnserialize(BrokerDataView data) = 0; virtual bool DoUnserialize(const broker::data& data);
virtual std::optional<BrokerData> DoSerializeData() const;
virtual bool DoUnserializeData(BrokerDataView data);
virtual BloomFilterType Type() const = 0; virtual BloomFilterType Type() const = 0;
const detail::Hasher* hasher; const detail::Hasher* hasher;
@ -201,8 +206,8 @@ protected:
void Add(const zeek::detail::HashKey* key) override; void Add(const zeek::detail::HashKey* key) override;
bool Decrement(const zeek::detail::HashKey* key) override; bool Decrement(const zeek::detail::HashKey* key) override;
size_t Count(const zeek::detail::HashKey* key) const override; size_t Count(const zeek::detail::HashKey* key) const override;
std::optional<BrokerData> DoSerialize() const override; std::optional<BrokerData> DoSerializeData() const override;
bool DoUnserialize(BrokerDataView data) override; bool DoUnserializeData(BrokerDataView data) override;
BloomFilterType Type() const override { return BloomFilterType::Basic; } BloomFilterType Type() const override { return BloomFilterType::Basic; }
private: private:
@ -264,8 +269,8 @@ protected:
void Add(const zeek::detail::HashKey* key) override; void Add(const zeek::detail::HashKey* key) override;
bool Decrement(const zeek::detail::HashKey* key) override; bool Decrement(const zeek::detail::HashKey* key) override;
size_t Count(const zeek::detail::HashKey* key) const override; size_t Count(const zeek::detail::HashKey* key) const override;
std::optional<BrokerData> DoSerialize() const override; std::optional<BrokerData> DoSerializeData() const override;
bool DoUnserialize(BrokerDataView data) override; bool DoUnserializeData(BrokerDataView data) override;
BloomFilterType Type() const override { return BloomFilterType::Counting; } BloomFilterType Type() const override { return BloomFilterType::Counting; }
private: private:

4
src/probabilistic/Topk.cc
View file

@ -360,7 +360,7 @@ void TopkVal::IncrementCounter(Element* e, unsigned int count) {
IMPLEMENT_OPAQUE_VALUE(TopkVal) IMPLEMENT_OPAQUE_VALUE(TopkVal)
std::optional<BrokerData> TopkVal::DoSerialize() const { std::optional<BrokerData> TopkVal::DoSerializeData() const {
BrokerListBuilder builder; BrokerListBuilder builder;
builder.Reserve(8); builder.Reserve(8);
@ -399,7 +399,7 @@ std::optional<BrokerData> TopkVal::DoSerialize() const {
return std::move(builder).Build(); return std::move(builder).Build();
} }
bool TopkVal::DoUnserialize(BrokerDataView data) { bool TopkVal::DoUnserializeData(BrokerDataView data) {
if ( ! data.IsList() ) if ( ! data.IsList() )
return false; return false;

2
src/probabilistic/Topk.h
View file

@ -132,7 +132,7 @@ public:
*/ */
ValPtr DoClone(CloneState* state) override; ValPtr DoClone(CloneState* state) override;
DECLARE_OPAQUE_VALUE(TopkVal) DECLARE_OPAQUE_VALUE_V2(TopkVal)
protected: protected:
/** /**