support for transmitting of capture-semantics closures via Broker, while keeping deprecated functionality

src/Frame.cc

@@ -302,24 +302,25 @@ Frame* Frame::SelectiveClone(const IDPList& selection, ScriptFunc* func) const
 	return other;
 	}
 
-broker::expected<broker::data> Frame::Serialize(const Frame* target, const IDPList& selection)
+broker::expected<broker::data> Frame::SerializeClosureFrame(const IDPList& selection)
 	{
 	broker::vector rval;
 
 	if ( selection.length() == 0 )
+		// Easy - no captures, so frame is irrelvant.
 		return {std::move(rval)};
 
 	IDPList us;
 	// and
 	IDPList them;
 
-	std::unordered_map<std::string, int> new_map;
+	OffsetMap new_map;
-	if ( target->offset_map )
+	if ( offset_map )
-		new_map = *(target->offset_map);
+		new_map = *offset_map;
 
-	for (const auto& we : selection)
+	for ( const auto& we : selection )
 		{
-		if ( target->IsOuterID(we) )
+		if ( IsOuterID(we) )
 			them.append(we);
 		else
 			{
@@ -330,18 +331,18 @@ broker::expected<broker::data> Frame::Serialize(const Frame* target, const IDPLi
 
 	if ( them.length() )
 		{
-		if ( ! target->closure )
+		if ( ! closure )
 			reporter->InternalError("Attempting to serialize values from a frame that does not exist.");
 
 		rval.emplace_back(std::string("ClosureFrame"));
 
-		auto ids = SerializeIDList(target->outer_ids);
+		auto ids = SerializeIDList(outer_ids);
 		if ( ! ids )
 			return broker::ec::invalid_data;
 
 		rval.emplace_back(*ids);
 
-		auto serialized = Frame::Serialize(target->closure, them);
+		auto serialized = closure->SerializeClosureFrame(them);
 		if ( ! serialized )
 			return broker::ec::invalid_data;
 
@@ -358,7 +359,7 @@ broker::expected<broker::data> Frame::Serialize(const Frame* target, const IDPLi
 
 	broker::vector body;
 
-	for ( int i = 0; i < target->size; ++i )
+	for ( int i = 0; i < size; ++i )
 		body.emplace_back(broker::none());
 
 	for ( const auto& id : us )
@@ -366,10 +367,10 @@ broker::expected<broker::data> Frame::Serialize(const Frame* target, const IDPLi
 		int location = id->Offset();
 
 		auto where = new_map.find(std::string(id->Name()));
-		if (where != new_map.end())
+		if ( where != new_map.end() )
 			location = where->second;
 
-		const auto& val = target->frame[location].val;
+		const auto& val = frame[location].val;
 
 		TypeTag tag = val->GetType()->Tag();
 
@@ -386,15 +387,38 @@ broker::expected<broker::data> Frame::Serialize(const Frame* target, const IDPLi
 	return {std::move(rval)};
 	}
 
-std::pair<bool, FramePtr> Frame::Unserialize(const broker::vector& data)
+broker::expected<broker::data> Frame::SerializeCopyFrame()
+	{
+	broker::vector rval;
+	rval.emplace_back(std::string("CopyFrame"));
+
+	broker::vector body;
+
+	for ( int i = 0; i < size; ++i )
+		{
+		const auto& val = frame[i].val;
+		auto expected = Broker::detail::val_to_data(val.get());
+		if ( ! expected )
+			return broker::ec::invalid_data;
+
+		TypeTag tag = val->GetType()->Tag();
+		broker::vector val_tuple {std::move(*expected),
+				static_cast<broker::integer>(tag)};
+		body.emplace_back(broker::none());
+		body[i] = val_tuple;
+		}
+
+	rval.emplace_back(body);
+
+	return {std::move(rval)};
+	}
+
+std::pair<bool, FramePtr> Frame::Unserialize(const broker::vector& data,
+				const std::vector<FuncType::Capture*>* captures)
 	{
 	if ( data.size() == 0 )
 		return std::make_pair(true, nullptr);
 
-	IDPList outer_ids;
-	OffsetMap offset_map;
-	FramePtr closure;
-
 	auto where = data.begin();
 
 	auto has_name = broker::get_if<std::string>(*where);
@@ -403,6 +427,54 @@ std::pair<bool, FramePtr> Frame::Unserialize(const broker::vector& data)
 
 	std::advance(where, 1);
 
+	if ( captures || *has_name == "CopyFrame" )
+		{
+		ASSERT(captures && *has_name == "CopyFrame");
+
+		auto has_body = broker::get_if<broker::vector>(*where);
+		if ( ! has_body )
+			return std::make_pair(false, nullptr);
+
+		broker::vector body = *has_body;
+		int frame_size = body.size();
+		auto rf = make_intrusive<Frame>(frame_size, nullptr, nullptr);
+
+		rf->closure = nullptr;
+
+		for ( int i = 0; i < frame_size; ++i )
+			{
+			auto has_vec = broker::get_if<broker::vector>(body[i]);
+			if ( ! has_vec )
+				continue;
+
+			broker::vector val_tuple = *has_vec;
+			if ( val_tuple.size() != 2 )
+				return std::make_pair(false, nullptr);
+
+			auto has_type = broker::get_if<broker::integer>(val_tuple[1]);
+			if ( ! has_type )
+				return std::make_pair(false, nullptr);
+
+			broker::integer g = *has_type;
+			Type t( static_cast<TypeTag>(g) );
+
+			auto val = Broker::detail::data_to_val(std::move(val_tuple[0]), &t);
+			if ( ! val )
+				return std::make_pair(false, nullptr);
+
+			rf->frame[i].val = std::move(val);
+			}
+
+		return std::make_pair(true, std::move(rf));
+		}
+
+
+	// Code to support deprecated semantics:
+
+	IDPList outer_ids;
+	OffsetMap offset_map;
+	FramePtr closure;
+
 	if ( *has_name == "ClosureFrame" )
 		{
 		auto has_vec = broker::get_if<broker::vector>(*where);
@@ -428,7 +500,7 @@ std::pair<bool, FramePtr> Frame::Unserialize(const broker::vector& data)
 
 		std::advance(where, 1);
 
-		auto closure_pair = Frame::Unserialize(*has_vec);
+		auto closure_pair = Frame::Unserialize(*has_vec, nullptr);
 		if ( ! closure_pair.first )
 			{
 			for ( auto& i : outer_ids )

src/Frame.h

@@ -13,6 +13,7 @@
 
 #include "zeek/ZeekList.h" // for typedef val_list
 #include "zeek/Obj.h"
+#include "zeek/Type.h"
 #include "zeek/IntrusivePtr.h"
 #include "zeek/ZeekArgs.h"
 
@@ -178,29 +179,42 @@ public:
 	Frame* SelectiveClone(const IDPList& selection, ScriptFunc* func) const;
 
 	/**
-	 * Serializes the Frame into a Broker representation.
+	 * Serializes the frame in the context of supporting the (deprecated)
-	 *
+	 * reference semantics for closures.  This can be fairly non-trivial.
-	 * Serializing a frame can be fairly non-trivial. If the frame has no
+	 * If the frame itself has no closure then the serialized frame
-	 * closure the serialized frame is just a vector:
+	 * is a vector:
 	 *
 	 * [ "Frame", [offset_map] [serialized_values] ]
 	 *
-	 * Where serialized_values are two element vectors. A serialized_value
+	 * where serialized_values are two-element vectors. A serialized_value
 	 * has the result of calling broker::data_to_val on the value in the
 	 * first index, and an integer representing that value's type in the
 	 * second index. offset_map is a serialized version of the frame's
 	 * offset_map.
 	 *
-	 * A Frame with a closure needs to serialize a little more information.
+	 * A reference-semantics frame with its own closure needs to
-	 * It is serialized as:
+	 * (recursively) serialize more information:
 	 *
 	 * [ "ClosureFrame", [outer_ids], Serialize(closure), [offset_map],
 	 *   [serialized_values] ]
 	 *
-	 * @return the broker representaton, or an error if the serialization
+	 * @return the broker representation, or an error if the serialization
 	 * failed.
 	 */
-	static broker::expected<broker::data> Serialize(const Frame* target, const IDPList& selection);
+	broker::expected<broker::data> SerializeClosureFrame(const IDPList& selection);
+
+	/**
+	 * Serializes the frame in the context of supporting copy semantics
+	 * for lambdas:
+	 *
+	 * [ "CopyFrame", serialized_values ]
+	 *
+	 * where serialized_values are two-element vectors. A serialized_value
+	 * has the result of calling broker::data_to_val on the value in the
+	 * first index, and an integer representing that value's type in the
+	 * second index.
+	 */
+	broker::expected<broker::data> SerializeCopyFrame();
 
 	/**
 	 * Instantiates a Frame from a serialized one.
@@ -208,8 +222,13 @@ public:
 	 * @return a pair in which the first item is the status of the serialization;
 	 * and the second is the unserialized frame with reference count +1, or
 	 * null if the serialization wasn't successful.
+	 *
+	 * The *captures* argument, if non-nil, specifies that the frame
+	 * reflects captures with copy-semantics rather than deprecated
+	 * reference semantics.
 	 */
-	static std::pair<bool, FramePtr> Unserialize(const broker::vector& data);
+	static std::pair<bool, FramePtr> Unserialize(const broker::vector& data,
+			const std::vector<FuncType::Capture*>* captures);
 
 	/**
 	 * Sets the IDs that the frame knows offsets for. These offsets will

src/Func.cc

@@ -596,7 +596,7 @@ void ScriptFunc::SetClosureFrame(Frame* f)
 
 bool ScriptFunc::UpdateClosure(const broker::vector& data)
 	{
-	auto result = Frame::Unserialize(data);
+	auto result = Frame::Unserialize(data, nullptr);
 
 	if ( ! result.first )
 		return false;
@@ -615,6 +615,17 @@ bool ScriptFunc::UpdateClosure(const broker::vector& data)
 	return true;
 	}
 
+bool ScriptFunc::DeserializeCaptures(const broker::vector& data)
+	{
+	auto result = Frame::Unserialize(data, GetType()->GetCaptures());
+
+ASSERT(result.first);
+
+	SetCaptures(result.second.release());
+
+	return true;
+	}
+
 
 FuncPtr ScriptFunc::DoClone()
 	{
@@ -629,12 +640,22 @@ FuncPtr ScriptFunc::DoClone()
 	other->weak_closure_ref = false;
 	other->outer_ids = outer_ids;
 
+	if ( captures_frame )
+		{
+		other->captures_frame = captures_frame->Clone();
+		other->captures_offset_mapping = new OffsetMap;
+		*other->captures_offset_mapping = *captures_offset_mapping;
+		}
+
 	return other;
 	}
 
 broker::expected<broker::data> ScriptFunc::SerializeClosure() const
 	{
-	return Frame::Serialize(closure, outer_ids);
+	if ( captures_frame )
+		return captures_frame->SerializeCopyFrame();
+	else
+		return closure->SerializeClosureFrame(outer_ids);
 	}
 
 void ScriptFunc::Describe(ODesc* d) const

src/Func.h

@@ -215,12 +215,19 @@ public:
 	bool StrengthenClosureReference(Frame* f);
 
 	/**
-	 * Serializes this function's closure.
+	 * Serializes this function's closure or capture frame.
 	 *
-	 * @return a serialized version of the function's closure.
+	 * @return a serialized version of the function's closure/capture frame.
 	 */
 	broker::expected<broker::data> SerializeClosure() const;
 
+	/**
+	 * Sets the captures frame to one built from *data*.
+	 *
+	 * @param data a serialized frame
+	 */
+	bool DeserializeCaptures(const broker::vector& data);
+
 	void AddBody(StmtPtr new_body,
 	             const std::vector<IDPtr>& new_inits,
 	             size_t new_frame_size, int priority) override;

src/broker/Data.cc

@@ -395,7 +395,7 @@ struct val_converter {
 			if ( t->Tag() != TYPE_FUNC )
 				return nullptr;
 
-			if ( a.size() == 2 ) // We have a closure.
+			if ( a.size() == 2 ) // we have a closure/capture frame
 				{
 				auto frame = broker::get_if<broker::vector>(a[1]);
 				if ( ! frame )
@@ -405,8 +405,20 @@ struct val_converter {
 				if ( ! b )
 					return nullptr;
 
-				if ( ! b->UpdateClosure(*frame) )
+				auto copy_semantics =
-					return nullptr;
+					b->GetType()->GetCaptures() != nullptr;
+
+				if ( copy_semantics )
+					{
+					if ( ! b->DeserializeCaptures(*frame) )
+						return nullptr;
+					}
+				else
+					{
+					// Support for deprecated serialization.
+					if ( ! b->UpdateClosure(*frame) )
+						return nullptr;
+					}
 				}
 
 			return rval;