mirror of
https://github.com/zeek/zeek.git
synced 2025-10-04 07:38:19 +00:00
Allow serialization of closures over Broker.
anonymous-functions, their closures, can now be sent over broker. In order to send an anonymous function the receiver must have parsed a definition of the functon, but it need not to have been evaluated. See testing/btest/language/closure-sending.zeek for an example of how this can be done. This also sends their closures as well as the closures of regular functions.
This commit is contained in:
Zeke Medley
parent
f18464f1f8
commit
f0798c4b49
19 changed files with 1060 additions and 160 deletions
305
src/Frame.cc
305
src/Frame.cc
|
|
@ -2,13 +2,16 @@
|
|||
|
||||
#include "zeek-config.h"
|
||||
|
||||
#include <string>
|
||||
#include <algorithm> // std::any_of
|
||||
|
||||
#include "Frame.h"
|
||||
#include "Stmt.h"
|
||||
#include "Func.h"
|
||||
#include "Trigger.h"
|
||||
|
||||
#include <string>
|
||||
#include <algorithm> // std::any_of
|
||||
#include "broker/Data.h"
|
||||
#include <broker/error.hh>
|
||||
|
||||
vector<Frame*> g_frame_stack;
|
||||
|
||||
|
|
@ -27,7 +30,7 @@ Frame::Frame(int arg_size, const BroFunc* func, const val_list* fn_args)
|
|||
call = 0;
|
||||
delayed = false;
|
||||
|
||||
is_view = false;
|
||||
is_view = false;
|
||||
|
||||
Clear();
|
||||
}
|
||||
|
|
@ -75,16 +78,23 @@ void Frame::SetElement(int n, Val* v)
|
|||
{
|
||||
Unref(frame[n]);
|
||||
frame[n] = v;
|
||||
}
|
||||
}
|
||||
|
||||
void Frame::SetElement(const ID* id, Val* v)
|
||||
{
|
||||
SetElement(id->Offset(), v);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Val* Frame::GetElement(ID* id) const
|
||||
Val* Frame::GetElement(const ID* id) const
|
||||
{
|
||||
return this->frame[id->Offset()];
|
||||
if (HasOuterIDs())
|
||||
{
|
||||
auto where = offset_map.find(std::string(id->Name()));
|
||||
if (where != offset_map.end())
|
||||
return frame[ where->second ];
|
||||
}
|
||||
return frame[id->Offset()];
|
||||
}
|
||||
|
||||
void Frame::AddElement(ID* id, Val* v)
|
||||
|
|
@ -155,7 +165,7 @@ Frame* Frame::Clone()
|
|||
}
|
||||
|
||||
Frame* Frame::SelectiveClone(id_list* selection)
|
||||
{
|
||||
{
|
||||
Frame* other = new Frame(size, function, func_args);
|
||||
|
||||
loop_over_list(*selection, i)
|
||||
|
|
@ -164,8 +174,148 @@ Frame* Frame::SelectiveClone(id_list* selection)
|
|||
other->frame[current->Offset()] = this->frame[current->Offset()];
|
||||
}
|
||||
|
||||
return other;
|
||||
}
|
||||
return other;
|
||||
}
|
||||
|
||||
broker::expected<broker::data> Frame::Serialize() const
|
||||
{
|
||||
broker::vector rval;
|
||||
rval.emplace_back(std::string("Frame"));
|
||||
|
||||
auto om = SerializeOffsetMap();
|
||||
if ( ! om ) return broker::ec::invalid_data;
|
||||
rval.emplace_back( *om );
|
||||
|
||||
for (int i = 0; i < size; ++i)
|
||||
{
|
||||
if ( ! frame[i] )
|
||||
{
|
||||
// data
|
||||
rval.emplace_back(broker::none());
|
||||
// type
|
||||
rval.emplace_back(broker::none());
|
||||
}
|
||||
else
|
||||
{
|
||||
auto expected = bro_broker::val_to_data(frame[i]);
|
||||
if ( ! expected )
|
||||
return broker::ec::invalid_data;
|
||||
else
|
||||
{
|
||||
// data
|
||||
rval.emplace_back(std::move(*expected));
|
||||
// type
|
||||
rval.emplace_back(static_cast<broker::integer>(frame[i]->Type()->Tag()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {std::move(rval)};
|
||||
}
|
||||
|
||||
std::pair<bool, Frame*> Frame::Unserialize(const broker::vector& data)
|
||||
{
|
||||
#define FAIL std::make_pair(false, nullptr)
|
||||
#define GET_OR_RETURN(type, name, index) \
|
||||
if (auto __##name##__ = broker::get_if<type>(data[index])) \
|
||||
name = *__##name##__; \
|
||||
else \
|
||||
return FAIL; \
|
||||
|
||||
std::string pivot;
|
||||
GET_OR_RETURN(std::string, pivot, 0)
|
||||
|
||||
if (pivot == "Frame")
|
||||
{
|
||||
|
||||
int frame_size = (data.size() - 2) / 2;
|
||||
// Cool -> We serialized a function with a null frame.
|
||||
if (frame_size == 0) return std::make_pair(true, nullptr);
|
||||
|
||||
// Unserialize the offset map.
|
||||
broker::vector o_map;
|
||||
GET_OR_RETURN(broker::vector, o_map, 1)
|
||||
|
||||
std::unordered_map<std::string, int> offset_map;
|
||||
bool status = ClosureFrame::UnserializeIntoOffsetMap(o_map, offset_map);
|
||||
|
||||
// Function / arg information updated later as needed.
|
||||
Frame* f = new Frame(frame_size, nullptr, nullptr);
|
||||
f->offset_map = std::move(offset_map);
|
||||
|
||||
for (int i = 0, j = 2; i < frame_size; ++i, j += 2)
|
||||
{
|
||||
// Null values in the serialized frame are stored as broker::none.
|
||||
if ( ! broker::get_if<broker::none>(data[j]) )
|
||||
{
|
||||
broker::integer g;
|
||||
GET_OR_RETURN(broker::integer, g, (j+1))
|
||||
|
||||
BroType t( static_cast<TypeTag>(g) );
|
||||
|
||||
auto val = bro_broker::data_to_val(std::move(data[j]), &t);
|
||||
if ( ! val ) return FAIL;
|
||||
|
||||
f->frame[i] = val;
|
||||
}
|
||||
}
|
||||
|
||||
return std::make_pair(true, f);
|
||||
}
|
||||
|
||||
else if (pivot == "ClosureFrame")
|
||||
{
|
||||
|
||||
broker::vector o_map;
|
||||
broker::vector v_closure;
|
||||
broker::vector v_body;
|
||||
|
||||
GET_OR_RETURN(broker::vector, o_map, 1)
|
||||
GET_OR_RETURN(broker::vector, v_closure, 2)
|
||||
GET_OR_RETURN(broker::vector, v_body, 3)
|
||||
|
||||
std::unordered_map<std::string, int> offset_map;
|
||||
bool status = ClosureFrame::UnserializeIntoOffsetMap(o_map, offset_map);
|
||||
|
||||
if ( ! status ) return FAIL;
|
||||
|
||||
auto result = Frame::Unserialize(v_closure);
|
||||
if ( ! result.first )
|
||||
return FAIL;
|
||||
Frame* closure = result.second;
|
||||
|
||||
result = Frame::Unserialize(v_body);
|
||||
if ( ! result.first )
|
||||
return FAIL;
|
||||
Frame* body = result.second;
|
||||
|
||||
ClosureFrame* c = new ClosureFrame(closure, body, nullptr);
|
||||
c->offset_map = std::move(offset_map);
|
||||
|
||||
return std::make_pair(true, c);
|
||||
}
|
||||
|
||||
return FAIL;
|
||||
#undef GET_OR_RETURN
|
||||
#undef FAIL
|
||||
}
|
||||
|
||||
void Frame::SetOuterIDs (std::shared_ptr<id_list> outer_ids)
|
||||
{
|
||||
// When cloning we bypass this step and just directly copy over the map,
|
||||
// hence the check.
|
||||
if ( ! outer_ids ) return;
|
||||
|
||||
if (offset_map.size()) return;
|
||||
|
||||
id_list tmp = *(outer_ids.get());
|
||||
loop_over_list(tmp, i)
|
||||
{
|
||||
ID* id = tmp[i];
|
||||
if (id)
|
||||
offset_map.emplace(id->Name(), id->Offset());
|
||||
}
|
||||
}
|
||||
|
||||
void Frame::SetTrigger(Trigger* arg_trigger)
|
||||
{
|
||||
|
|
@ -183,8 +333,14 @@ void Frame::ClearTrigger()
|
|||
trigger = 0;
|
||||
}
|
||||
|
||||
bool Frame::CaptureContains(const ID* i) const
|
||||
{
|
||||
auto where = offset_map.find(std::string(i->Name()));
|
||||
return where != offset_map.end();
|
||||
}
|
||||
|
||||
ClosureFrame::ClosureFrame(Frame* closure, Frame* not_closure,
|
||||
std::shared_ptr<id_list> outer_ids) : Frame(not_closure, true)
|
||||
std::shared_ptr<id_list> outer_ids) : Frame(not_closure, true)
|
||||
{
|
||||
assert(closure);
|
||||
assert(outer_ids);
|
||||
|
|
@ -192,19 +348,7 @@ ClosureFrame::ClosureFrame(Frame* closure, Frame* not_closure,
|
|||
this->closure = closure;
|
||||
body = not_closure;
|
||||
|
||||
// To clone a ClosureFrame we null outer_ids and then copy
|
||||
// the set over directly, hence the check.
|
||||
if (outer_ids)
|
||||
{
|
||||
// Install the closure IDs
|
||||
id_list* tmp = outer_ids.get();
|
||||
loop_over_list(*tmp, i)
|
||||
{
|
||||
ID* id = (*tmp)[i];
|
||||
if (id)
|
||||
closure_elements.push_back(id->Name());
|
||||
}
|
||||
}
|
||||
SetOuterIDs(outer_ids);
|
||||
}
|
||||
|
||||
ClosureFrame::~ClosureFrame()
|
||||
|
|
@ -215,18 +359,23 @@ ClosureFrame::~ClosureFrame()
|
|||
Unref(body);
|
||||
}
|
||||
|
||||
Val* ClosureFrame::GetElement(ID* id) const
|
||||
Val* ClosureFrame::GetElement(const ID* id) const
|
||||
{
|
||||
if ( CaptureContains(id) )
|
||||
return ClosureFrame::GatherFromClosure(this, id);
|
||||
|
||||
{
|
||||
int my_offset = offset_map.at(std::string(id->Name()));
|
||||
return ClosureFrame::GatherFromClosure(this, id, my_offset);
|
||||
}
|
||||
return this->NthElement(id->Offset());
|
||||
}
|
||||
|
||||
void ClosureFrame::SetElement(const ID* id, Val* v)
|
||||
{
|
||||
if ( CaptureContains(id) )
|
||||
ClosureFrame::SetInClosure(this, id, v);
|
||||
{
|
||||
int my_offset = offset_map.at(std::string(id->Name()));
|
||||
ClosureFrame::SetInClosure(this, id, v, my_offset);
|
||||
}
|
||||
else
|
||||
this->Frame::SetElement(id->Offset(), v);
|
||||
}
|
||||
|
|
@ -237,12 +386,12 @@ Frame* ClosureFrame::Clone()
|
|||
Frame* new_regular = body->Clone();
|
||||
|
||||
ClosureFrame* cf = new ClosureFrame(new_closure, new_regular, nullptr);
|
||||
cf->closure_elements = closure_elements;
|
||||
cf->offset_map = offset_map;
|
||||
return cf;
|
||||
}
|
||||
|
||||
Frame* ClosureFrame::SelectiveClone(id_list* choose)
|
||||
{
|
||||
{
|
||||
id_list us;
|
||||
// and
|
||||
id_list them;
|
||||
|
|
@ -261,50 +410,102 @@ Frame* ClosureFrame::SelectiveClone(id_list* choose)
|
|||
Frame* you = this->body->SelectiveClone(&them);
|
||||
|
||||
ClosureFrame* who = new ClosureFrame(me, you, nullptr);
|
||||
who->closure_elements = closure_elements;
|
||||
who->offset_map = offset_map;
|
||||
|
||||
return who;
|
||||
}
|
||||
|
||||
broker::expected<broker::data> ClosureFrame::Serialize() const
|
||||
{
|
||||
broker::vector rval;
|
||||
rval.emplace_back(std::string("ClosureFrame"));
|
||||
|
||||
auto om = SerializeOffsetMap();
|
||||
if ( ! om ) return broker::ec::invalid_data;
|
||||
rval.emplace_back( *om );
|
||||
|
||||
auto cl = closure->Serialize();
|
||||
if ( ! cl ) broker::ec::invalid_data;
|
||||
rval.emplace_back( *cl );
|
||||
|
||||
auto bo = body->Serialize();
|
||||
if ( ! bo ) broker::ec::invalid_data;
|
||||
rval.emplace_back( *bo );
|
||||
|
||||
return {std::move(rval)};
|
||||
}
|
||||
|
||||
broker::expected<broker::data> Frame::SerializeOffsetMap() const
|
||||
{
|
||||
broker::vector rval;
|
||||
|
||||
std::for_each(offset_map.begin(), offset_map.end(),
|
||||
[&rval] (const std::pair<std::string, int>& e)
|
||||
{ rval.emplace_back(e.first); rval.emplace_back(e.second);});
|
||||
|
||||
return {std::move(rval)};
|
||||
}
|
||||
|
||||
bool ClosureFrame::UnserializeIntoOffsetMap(const broker::vector& data, std::unordered_map<std::string, int>& target)
|
||||
{
|
||||
#define GET_OR_RETURN(type, name, index) \
|
||||
if (auto __##name##__ = broker::get_if<type>(data[index])) \
|
||||
name = *__##name##__; \
|
||||
else \
|
||||
return false; \
|
||||
|
||||
assert(target.size() == 0);
|
||||
|
||||
std::unordered_map<std::string, int> rval;
|
||||
|
||||
for (broker::vector::size_type i = 0; i < data.size(); i += 2)
|
||||
{
|
||||
std::string key;
|
||||
int offset;
|
||||
GET_OR_RETURN(std::string, key, i)
|
||||
GET_OR_RETURN(broker::integer, offset, i+1)
|
||||
target.insert( {std::move(key), std::move(offset)} );
|
||||
}
|
||||
|
||||
return true;
|
||||
#undef GET_OR_RETURN
|
||||
}
|
||||
|
||||
// Each ClosureFrame knows all of the outer IDs that are used inside of it. This is known at
|
||||
// parse time. These leverage that. If frame_1 encloses frame_2 then the location of a lookup
|
||||
// for an outer id in frame_2 can be determined by checking if that id is also an outer id in
|
||||
// frame_2. If it is not, then frame_2 owns the id and the lookup is done there, otherwise,
|
||||
// go deeper.
|
||||
|
||||
// Note the useage of dynamic_cast.
|
||||
|
||||
|
||||
Val* ClosureFrame::GatherFromClosure(const Frame* start, const ID* id)
|
||||
Val* ClosureFrame::GatherFromClosure(const Frame* start, const ID* id, const int offset)
|
||||
{
|
||||
const ClosureFrame* conductor = dynamic_cast<const ClosureFrame*>(start);
|
||||
|
||||
// If a subframe has outer IDs then it was serialized and passed around before this frame
|
||||
// was born. We differ to its maping as it is older and wiser. Otherwise, we use our own.
|
||||
if ( ! conductor )
|
||||
return start->NthElement(id->Offset());
|
||||
{
|
||||
if (start->HasOuterIDs())
|
||||
return start->GetElement(id);
|
||||
|
||||
return start->NthElement(offset);
|
||||
}
|
||||
|
||||
if (conductor->CaptureContains(id))
|
||||
return ClosureFrame::GatherFromClosure(conductor->closure, id);
|
||||
return ClosureFrame::GatherFromClosure(conductor->closure, id, offset);
|
||||
|
||||
return conductor->NthElement(id->Offset());
|
||||
}
|
||||
return conductor->NthElement(offset);
|
||||
}
|
||||
|
||||
void ClosureFrame::SetInClosure(Frame* start, const ID* id, Val* val)
|
||||
{
|
||||
void ClosureFrame::SetInClosure(Frame* start, const ID* id, Val* val, const int offset)
|
||||
{
|
||||
ClosureFrame* conductor = dynamic_cast<ClosureFrame*>(start);
|
||||
|
||||
if ( ! conductor )
|
||||
start->SetElement(id->Offset(), val);
|
||||
start->SetElement(offset, val);
|
||||
|
||||
else if (conductor->CaptureContains(id))
|
||||
ClosureFrame::SetInClosure(conductor->closure, id, val);
|
||||
ClosureFrame::SetInClosure(conductor->closure, id, val, offset);
|
||||
|
||||
else
|
||||
conductor->Frame::SetElement(id->Offset(), val);
|
||||
conductor->Frame::SetElement(offset, val);
|
||||
}
|
||||
|
||||
bool ClosureFrame::CaptureContains(const ID* i) const
|
||||
{
|
||||
const char* target = i->Name();
|
||||
return std::any_of(closure_elements.begin(), closure_elements.end(),
|
||||
[target](const char* in) { return strcmp(target, in) == 0; });
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue