Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/topic/vern/record-optimizations.Apr23B'

Browse source 
* origin/topic/vern/record-optimizations.Apr23B:
  different fix for MSVC compiler issues
  more general approach for addressing MSVC compiler issues with IntrusivePtr
  restored RecordType::Create, now marked as deprecated tidying of namespaces and private class members simplification of flagging record field initializations that should be skipped address peculiar MSVC compilation complaint for IntrusivePtr's
  clarifications and tidying for record field initializations
  optimize record construction by deferring initializations of aggregates
  compile-scripts-to-C++ speedups by switching to raw record access
  logging speedup by switching to raw record access
  remove redundant record coercions

Removed the `#if 0` hunk during merging: Probably could have gone with a
doctest instead.
This commit is contained in:
Arne Welzel 2023-04-19 11:58:09 +02:00
commit 89c828ac14
14 changed files with 399 additions and 205 deletions
Download patch file Download diff file Expand all files Collapse all files

21
CHANGES
View file

@ -1,3 +1,24 @@
6.0.0-dev.385 | 2023-04-19 11:58:09 +0200
* different fix for MSVC compiler issues (Vern Paxson, Corelight)
* more general approach for addressing MSVC compiler issues with IntrusivePtr (Vern Paxson, Corelight)
* restored RecordType::Create, now marked as deprecated (Vern Paxson, Corelight)
tidying of namespaces and private class members
simplification of flagging record field initializations that should be skipped
address peculiar MSVC compilation complaint for IntrusivePtr's
* clarifications and tidying for record field initializations (Vern Paxson, Corelight)
* optimize record construction by deferring initializations of aggregates (Vern Paxson, Corelight)
* compile-scripts-to-C++ speedups by switching to raw record access (Vern Paxson, Corelight)
* logging speedup by switching to raw record access (Vern Paxson, Corelight)
* remove redundant record coercions (Vern Paxson, Corelight)
6.0.0-dev.376 | 2023-04-19 10:14:02 +0200
* Improve CMake variables, update cmake submodule (Dominik Charousset, Corelight)

2
VERSION
View file

@ -1 +1 @@
6.0.0-dev.376
6.0.0-dev.385

4
src/Expr.cc
View file

@ -4215,6 +4215,10 @@ TableCoerceExpr::TableCoerceExpr(ExprPtr arg_op, TableTypePtr tt, bool type_chec
SetError();
return;
}
if ( op->Tag() == EXPR_TABLE_COERCE && op->GetType() == tt )
// Avoid double-coercion.
op = op->GetOp1();
}
SetType(std::move(tt));

249
src/Type.cc
View file

@ -992,41 +992,118 @@ void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const
}
}
// The following tracks how to initialize a given field, for fast execution
// of Create().
namespace detail
{
class FieldInit
// A record field initialization that directly assigns a fixed value ...
class DirectFieldInit final : public FieldInit
{
public:
// The type of initialization for the field.
enum
{
R_INIT_NONE, // skip this entry
DirectFieldInit(ZVal _init_val) : init_val(_init_val) { }
R_INIT_DIRECT, // look in direct_init for raw value
R_INIT_DIRECT_MANAGED, // same, but managed type
ZVal Generate() const override { return init_val; }
R_INIT_DEF, // look in def_expr for expression
R_INIT_RECORD, // field requires a new record
R_INIT_TABLE, // field requires a new table/set
R_INIT_VECTOR, // field requires a new vector
} init_type = R_INIT_NONE;
bool def_coerce = false; // whether coercion's required
// For R_INIT_DIRECT/R_INIT_DIRECT_MANAGED:
ZVal direct_init;
detail::ExprPtr def_expr;
TypePtr def_type;
RecordTypePtr r_type; // for R_INIT_RECORD
TableTypePtr t_type; // for R_INIT_TABLE
detail::AttributesPtr attrs; // attributes for R_INIT_TABLE
VectorTypePtr v_type; // for R_INIT_VECTOR
private:
ZVal init_val;
};
// ... the same, but for a value that needs memory management.
class DirectManagedFieldInit final : public FieldInit
{
public:
DirectManagedFieldInit(ZVal _init_val) : init_val(_init_val) { }
~DirectManagedFieldInit() { ZVal::DeleteManagedType(init_val); }
ZVal Generate() const override
{
zeek::Ref(init_val.ManagedVal());
return init_val;
}
private:
ZVal init_val;
};
// A record field initialization that's done by evaluating an expression.
class ExprFieldInit final : public FieldInit
{
public:
// Initialization requires evaluating the given expression,
// yielding the a value of the given type (which might require
// coercion for some records).
ExprFieldInit(detail::ExprPtr _init_expr, TypePtr _init_type)
: init_expr(std::move(_init_expr)), init_type(std::move(_init_type))
{
if ( init_type->Tag() == TYPE_RECORD && ! same_type(init_expr->GetType(), init_type) )
coerce_type = cast_intrusive<RecordType>(init_type);
}
ZVal Generate() const override
{
auto v = init_expr->Eval(nullptr);
if ( ! v )
{
reporter->Error("failed &default in record creation");
return ZVal();
}
if ( coerce_type )
v = v->AsRecordVal()->CoerceTo(coerce_type);
return ZVal(v, init_type);
}
private:
detail::ExprPtr init_expr;
TypePtr init_type;
RecordTypePtr coerce_type; // non-nil iff coercion is required
};
// A record field initialization where the field is initialized to an
// empty/default record of the given type.
class RecordFieldInit final : public FieldInit
{
public:
RecordFieldInit(RecordTypePtr _init_type) : init_type(std::move(_init_type)) { }
ZVal Generate() const override { return ZVal(new RecordVal(init_type)); }
private:
RecordTypePtr init_type;
};
// A record field initialization where the field is initialized to an
// empty table of the given type.
class TableFieldInit final : public FieldInit
{
public:
TableFieldInit(TableTypePtr _init_type, detail::AttributesPtr _attrs)
: init_type(std::move(_init_type)), attrs(std::move(_attrs))
{
}
ZVal Generate() const override { return ZVal(new TableVal(init_type, attrs)); }
private:
TableTypePtr init_type;
detail::AttributesPtr attrs;
};
// A record field initialization where the field is initialized to an
// empty vector of the given type.
class VectorFieldInit final : public FieldInit
{
public:
VectorFieldInit(VectorTypePtr _init_type) : init_type(std::move(_init_type)) { }
ZVal Generate() const override { return ZVal(new VectorVal(init_type)); }
private:
VectorTypePtr init_type;
};
} // namespace detail
RecordType::RecordType(type_decl_list* arg_types) : Type(TYPE_RECORD)
{
types = arg_types;
@ -1062,65 +1139,52 @@ RecordType::~RecordType()
delete types;
}
for ( auto fi : field_inits )
delete fi;
}
void RecordType::AddField(unsigned int field, const TypeDecl* td)
{
ASSERT(field == field_inits.size());
ASSERT(field == deferred_inits.size());
ASSERT(field == managed_fields.size());
managed_fields.push_back(ZVal::IsManagedType(td->type));
auto init = new FieldInit();
init->init_type = FieldInit::R_INIT_NONE;
init->attrs = td->attrs;
// We defer error-checking until here so that we can keep field_inits
// We defer error-checking until here so that we can keep deferred_inits
// and managed_fields correctly tracking the associated fields.
if ( field_ids.count(td->id) != 0 )
{
reporter->Error("duplicate field '%s' found in record definition", td->id);
field_inits.push_back(init);
deferred_inits.push_back(nullptr);
return;
}
field_ids.insert(std::string(td->id));
auto a = init->attrs;
auto a = td->attrs;
auto type = td->type;
auto def_attr = a ? a->Find(detail::ATTR_DEFAULT) : nullptr;
auto def_expr = def_attr ? def_attr->GetExpr() : nullptr;
std::unique_ptr<detail::FieldInit> init;
if ( def_expr && ! IsErrorType(type->Tag()) )
{
if ( type->Tag() == TYPE_RECORD && def_expr->GetType()->Tag() == TYPE_RECORD &&
! same_type(def_expr->GetType(), type) )
init->def_coerce = true;
if ( def_expr->Tag() == detail::EXPR_CONST )
{
auto v = def_expr->Eval(nullptr);
auto zv = ZVal(v, type);
if ( ZVal::IsManagedType(type) )
init->init_type = FieldInit::R_INIT_DIRECT_MANAGED;
init = std::make_unique<detail::DirectManagedFieldInit>(zv);
else
init->init_type = FieldInit::R_INIT_DIRECT;
init->direct_init = ZVal(v, type);
init = std::make_unique<detail::DirectFieldInit>(zv);
}
else
{
init->init_type = FieldInit::R_INIT_DEF;
init->def_expr = def_expr;
init->def_type = def_expr->GetType();
auto efi = std::make_unique<detail::ExprFieldInit>(def_expr, type);
creation_inits.emplace_back(std::make_pair(field, std::move(efi)));
}
}
@ -1129,25 +1193,16 @@ void RecordType::AddField(unsigned int field, const TypeDecl* td)
TypeTag tag = type->Tag();
if ( tag == TYPE_RECORD )
{
init->init_type = FieldInit::R_INIT_RECORD;
init->r_type = cast_intrusive<RecordType>(type);
}
init = std::make_unique<detail::RecordFieldInit>(cast_intrusive<RecordType>(type));
else if ( tag == TYPE_TABLE )
{
init->init_type = FieldInit::R_INIT_TABLE;
init->t_type = cast_intrusive<TableType>(type);
}
init = std::make_unique<detail::TableFieldInit>(cast_intrusive<TableType>(type), a);
else if ( tag == TYPE_VECTOR )
{
init->init_type = FieldInit::R_INIT_VECTOR;
init->v_type = cast_intrusive<VectorType>(type);
}
init = std::make_unique<detail::VectorFieldInit>(cast_intrusive<VectorType>(type));
}
field_inits.push_back(init);
deferred_inits.push_back(std::move(init));
}
bool RecordType::HasField(const char* field) const
@ -1344,64 +1399,14 @@ void RecordType::AddFieldsDirectly(const type_decl_list& others, bool add_log_at
void RecordType::Create(std::vector<std::optional<ZVal>>& r) const
{
int n = NumFields();
for ( auto& di : deferred_inits )
if ( di )
r.push_back(di->Generate());
else
r.push_back(std::nullopt);
for ( int i = 0; i < n; ++i )
{
auto* init = field_inits[i];
ZVal r_i;
switch ( init->init_type )
{
case FieldInit::R_INIT_NONE:
r.push_back(std::nullopt);
continue;
case FieldInit::R_INIT_DIRECT:
r_i = init->direct_init;
break;
case FieldInit::R_INIT_DIRECT_MANAGED:
r_i = init->direct_init;
zeek::Ref(r_i.ManagedVal());
break;
case FieldInit::R_INIT_DEF:
{
auto v = init->def_expr->Eval(nullptr);
if ( v )
{
const auto& t = init->def_type;
if ( init->def_coerce )
{
auto rt = cast_intrusive<RecordType>(t);
v = v->AsRecordVal()->CoerceTo(rt);
}
r_i = ZVal(v, t);
}
else
reporter->Error("failed &default in record creation");
}
break;
case FieldInit::R_INIT_RECORD:
r_i = ZVal(new RecordVal(init->r_type));
break;
case FieldInit::R_INIT_TABLE:
r_i = ZVal(new TableVal(init->t_type, init->attrs));
break;
case FieldInit::R_INIT_VECTOR:
r_i = ZVal(new VectorVal(init->v_type));
break;
}
r.push_back(r_i);
}
for ( auto& ci : creation_inits )
r[ci.first] = ci.second->Generate();
}
void RecordType::DescribeFields(ODesc* d) const

41
src/Type.h
View file

@ -22,6 +22,7 @@ namespace zeek
class Val;
union ZVal;
class EnumVal;
class RecordVal;
class TableVal;
using ValPtr = IntrusivePtr<Val>;
using EnumValPtr = IntrusivePtr<EnumVal>;
@ -35,6 +36,16 @@ class ListExpr;
class Attributes;
using ListExprPtr = IntrusivePtr<ListExpr>;
// The following tracks how to initialize a given record field.
class FieldInit
{
public:
virtual ~FieldInit() { }
// Return the initialization value of the field.
virtual ZVal Generate() const = 0;
};
} // namespace detail
// Zeek types.
@ -599,11 +610,6 @@ public:
using type_decl_list = PList<TypeDecl>;
// The following tracks how to initialize a given field. We don't define
// it here because it requires pulling in a bunch of low-level headers that
// would be nice to avoid.
class FieldInit;
class RecordType final : public Type
{
public:
@ -687,7 +693,9 @@ public:
* Populates a new instance of the record with its initial values.
* @param r The record's underlying value vector.
*/
void Create(std::vector<std::optional<ZVal>>& r) const;
[[deprecated("Remove in v6.1. Construct a corresponding RecordVal and build vector from "
"GetFieldAs() calls.")]] void
Create(std::vector<std::optional<ZVal>>& r) const;
void DescribeReST(ODesc* d, bool roles_only = false) const override;
void DescribeFields(ODesc* d) const;
@ -709,16 +717,29 @@ public:
detail::TraversalCode Traverse(detail::TraversalCallback* cb) const override;
protected:
private:
RecordType() { types = nullptr; }
void AddField(unsigned int field, const TypeDecl* td);
void DoDescribe(ODesc* d) const override;
// Maps each field to how to initialize it. Uses pointers due to
// keeping the FieldInit definition private to Type.cc (see above).
std::vector<FieldInit*> field_inits;
// Field initializations that can be deferred to first access,
// beneficial for fields that are separately initialized prior
// to first access. Nil pointers mean "skip initializing the field".
std::vector<std::unique_ptr<detail::FieldInit>> deferred_inits;
// Field initializations that need to be done upon record creation,
// rather than deferred. These are expressions whose value might
// change if computed later.
//
// Such initializations are uncommon, so we represent them using
// <fieldoffset, init> pairs.
std::vector<std::pair<int, std::unique_ptr<detail::FieldInit>>> creation_inits;
friend zeek::RecordVal;
const auto& DeferredInits() const { return deferred_inits; }
const auto& CreationInits() const { return creation_inits; }
// If we were willing to bound the size of records, then we could
// use std::bitset here instead.

43
src/Val.cc
View file

@ -2759,25 +2759,32 @@ RecordVal::RecordVal(RecordTypePtr t, bool init_fields) : Val(t), is_managed(t->
int n = rt->NumFields();
record_val = new std::vector<std::optional<ZVal>>;
record_val->reserve(n);
if ( run_state::is_parsing )
parse_time_records[rt.get()].emplace_back(NewRef{}, this);
record_val = new std::vector<std::optional<ZVal>>;
if ( init_fields )
{
try
record_val->resize(n);
for ( auto& e : rt->CreationInits() )
{
rt->Create(*record_val);
}
catch ( InterpreterException& e )
{
if ( run_state::is_parsing )
parse_time_records[rt.get()].pop_back();
throw;
try
{
(*record_val)[e.first] = e.second->Generate();
}
catch ( InterpreterException& e )
{
if ( run_state::is_parsing )
parse_time_records[rt.get()].pop_back();
throw;
}
}
}
else
record_val->reserve(n);
}
RecordVal::~RecordVal()
@ -2785,8 +2792,11 @@ RecordVal::~RecordVal()
auto n = record_val->size();
for ( unsigned int i = 0; i < n; ++i )
if ( HasField(i) && IsManaged(i) )
ZVal::DeleteManagedType(*(*record_val)[i]);
{
auto f_i = (*record_val)[i];
if ( f_i && IsManaged(i) )
ZVal::DeleteManagedType(*f_i);
}
delete record_val;
}
@ -2812,12 +2822,13 @@ void RecordVal::Assign(int field, ValPtr new_val)
void RecordVal::Remove(int field)
{
if ( HasField(field) )
auto& f_i = (*record_val)[field];
if ( f_i )
{
if ( IsManaged(field) )
ZVal::DeleteManagedType(*(*record_val)[field]);
ZVal::DeleteManagedType(*f_i);
(*record_val)[field] = std::nullopt;
f_i = std::nullopt;
Modified();
}

60
src/Val.h
View file

@ -52,9 +52,15 @@ class HashKey;
class ValTrace;
class ZBody;
class CPPRuntime;
} // namespace detail
namespace logging
{
class Manager;
}
namespace run_state
{
@ -1173,9 +1179,10 @@ public:
void Assign(int field, StringVal* new_val)
{
if ( HasField(field) )
ZVal::DeleteManagedType(*(*record_val)[field]);
(*record_val)[field] = ZVal(new_val);
auto& fv = (*record_val)[field];
if ( fv )
ZVal::DeleteManagedType(*fv);
fv = ZVal(new_val);
AddedField(field);
}
void Assign(int field, const char* new_val) { Assign(field, new StringVal(new_val)); }
@ -1188,7 +1195,7 @@ public:
*/
template <class T> void AssignField(const char* field_name, T&& val)
{
int idx = GetType()->AsRecordType()->FieldOffset(field_name);
int idx = rt->FieldOffset(field_name);
if ( idx < 0 )
reporter->InternalError("missing record field: %s", field_name);
Assign(idx, std::forward<T>(val));
@ -1206,7 +1213,13 @@ public:
* @param field The field index to retrieve.
* @return Whether there's a value for the given field index.
*/
bool HasField(int field) const { return (*record_val)[field] ? true : false; }
bool HasField(int field) const
{
if ( (*record_val)[field] )
return true;
return rt->DeferredInits()[field] != nullptr;
}
/**
* Returns true if the given field is in the record, false if
@ -1216,7 +1229,7 @@ public:
*/
bool HasField(const char* field) const
{
int idx = GetType()->AsRecordType()->FieldOffset(field);
int idx = rt->FieldOffset(field);
return (idx != -1) && HasField(idx);
}
@ -1227,10 +1240,17 @@ public:
*/
ValPtr GetField(int field) const
{
if ( ! HasField(field) )
return nullptr;
auto& fv = (*record_val)[field];
if ( ! fv )
{
const auto& fi = rt->DeferredInits()[field];
if ( ! fi )
return nullptr;
return (*record_val)[field]->ToVal(rt->GetFieldType(field));
fv = fi->Generate();
}
return fv->ToVal(rt->GetFieldType(field));
}
/**
@ -1358,7 +1378,7 @@ public:
template <typename T> auto GetFieldAs(const char* field) const
{
int idx = GetType()->AsRecordType()->FieldOffset(field);
int idx = rt->FieldOffset(field);
if ( idx < 0 )
reporter->InternalError("missing record field: %s", field);
@ -1403,8 +1423,10 @@ public:
static void DoneParsing();
protected:
friend class zeek::logging::Manager;
friend class zeek::detail::ValTrace;
friend class zeek::detail::ZBody;
friend class zeek::detail::CPPRuntime;
RecordValPtr DoCoerceTo(RecordTypePtr other, bool allow_orphaning) const;
@ -1429,7 +1451,18 @@ protected:
// Caller assumes responsibility for memory management. The first
// version allows manipulation of whether the field is present at all.
// The second version ensures that the optional value is present.
std::optional<ZVal>& RawOptField(int field) { return (*record_val)[field]; }
std::optional<ZVal>& RawOptField(int field)
{
auto& f = (*record_val)[field];
if ( ! f )
{
const auto& fi = rt->DeferredInits()[field];
if ( fi )
f = fi->Generate();
}
return f;
}
ZVal& RawField(int field)
{
@ -1451,8 +1484,9 @@ protected:
private:
void DeleteFieldIfManaged(unsigned int field)
{
if ( HasField(field) && IsManaged(field) )
ZVal::DeleteManagedType(*(*record_val)[field]);
auto& f = (*record_val)[field];
if ( f && IsManaged(field) )
ZVal::DeleteManagedType(*f);
}
bool IsManaged(unsigned int offset) const { return is_managed[offset]; }

1
src/ZVal.h
View file

@ -10,6 +10,7 @@ namespace zeek
{
class AddrVal;
class EnumVal;
class File;
class Func;
class ListVal;

83
src/logging/Manager.cc
View file

@ -973,11 +973,8 @@ bool Manager::Write(EnumVal* id, RecordVal* columns_arg)
return true;
}
threading::Value* Manager::ValToLogVal(Val* val, Type* ty)
threading::Value* Manager::ValToLogVal(std::optional<ZVal>& val, Type* ty)
{
if ( ! ty )
ty = val->GetType().get();
if ( ! val )
return new threading::Value(ty->Tag(), false);
@ -987,12 +984,12 @@ threading::Value* Manager::ValToLogVal(Val* val, Type* ty)
{
case TYPE_BOOL:
case TYPE_INT:
lval->val.int_val = val->InternalInt();
lval->val.int_val = val->AsInt();
break;
case TYPE_ENUM:
{
const char* s = val->GetType()->AsEnumType()->Lookup(val->InternalInt());
const char* s = ty->AsEnumType()->Lookup(val->AsInt());
if ( s )
{
@ -1002,7 +999,8 @@ threading::Value* Manager::ValToLogVal(Val* val, Type* ty)
else
{
val->GetType()->Error("enum type does not contain value", val);
auto err_msg = "enum type does not contain value:" + std::to_string(val->AsInt());
ty->Error(err_msg.c_str());
lval->val.string_val.data = util::copy_string("");
lval->val.string_val.length = 0;
}
@ -1010,31 +1008,44 @@ threading::Value* Manager::ValToLogVal(Val* val, Type* ty)
}
case TYPE_COUNT:
lval->val.uint_val = val->InternalUnsigned();
lval->val.uint_val = val->AsCount();
break;
case TYPE_PORT:
lval->val.port_val.port = val->AsPortVal()->Port();
lval->val.port_val.proto = val->AsPortVal()->PortType();
{
auto p = val->AsCount();
auto pt = TRANSPORT_UNKNOWN;
auto pm = p & PORT_SPACE_MASK;
if ( pm == TCP_PORT_MASK )
pt = TRANSPORT_TCP;
else if ( pm == UDP_PORT_MASK )
pt = TRANSPORT_UDP;
else if ( pm == ICMP_PORT_MASK )
pt = TRANSPORT_ICMP;
lval->val.port_val.port = p & ~PORT_SPACE_MASK;
lval->val.port_val.proto = pt;
break;
}
case TYPE_SUBNET:
val->AsSubNet().ConvertToThreadingValue(&lval->val.subnet_val);
val->AsSubNet()->Get().ConvertToThreadingValue(&lval->val.subnet_val);
break;
case TYPE_ADDR:
val->AsAddr().ConvertToThreadingValue(&lval->val.addr_val);
val->AsAddr()->Get().ConvertToThreadingValue(&lval->val.addr_val);
break;
case TYPE_DOUBLE:
case TYPE_TIME:
case TYPE_INTERVAL:
lval->val.double_val = val->InternalDouble();
lval->val.double_val = val->AsDouble();
break;
case TYPE_STRING:
{
const String* s = val->AsString();
const String* s = val->AsString()->AsString();
char* buf = new char[s->Len()];
memcpy(buf, s->Bytes(), s->Len());
@ -1065,31 +1076,44 @@ threading::Value* Manager::ValToLogVal(Val* val, Type* ty)
case TYPE_TABLE:
{
auto set = val->AsTableVal()->ToPureListVal();
auto tbl = val->AsTable();
auto set = tbl->ToPureListVal();
if ( ! set )
// ToPureListVal has reported an internal warning
// already. Just keep going by making something up.
set = make_intrusive<ListVal>(TYPE_INT);
auto tbl_t = cast_intrusive<TableType>(tbl->GetType());
auto& set_t = tbl_t->GetIndexTypes()[0];
bool is_managed = ZVal::IsManagedType(set_t);
lval->val.set_val.size = set->Length();
lval->val.set_val.vals = new threading::Value*[lval->val.set_val.size];
for ( zeek_int_t i = 0; i < lval->val.set_val.size; i++ )
lval->val.set_val.vals[i] = ValToLogVal(set->Idx(i).get());
{
std::optional<ZVal> s_i = ZVal(set->Idx(i), set_t);
lval->val.set_val.vals[i] = ValToLogVal(s_i, set_t.get());
if ( is_managed )
ZVal::DeleteManagedType(*s_i);
}
break;
}
case TYPE_VECTOR:
{
VectorVal* vec = val->AsVectorVal();
VectorVal* vec = val->AsVector();
lval->val.vector_val.size = vec->Size();
lval->val.vector_val.vals = new threading::Value*[lval->val.vector_val.size];
auto& vv = vec->RawVec();
auto& vt = vec->GetType()->Yield();
for ( zeek_int_t i = 0; i < lval->val.vector_val.size; i++ )
{
lval->val.vector_val.vals[i] = ValToLogVal(vec->ValAt(i).get(),
vec->GetType()->Yield().get());
lval->val.vector_val.vals[i] = ValToLogVal((*vv)[i], vt.get());
}
break;
@ -1118,7 +1142,8 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, R
for ( int i = 0; i < filter->num_fields; ++i )
{
Val* val;
std::optional<ZVal> val;
Type* vt;
if ( i < filter->num_ext_fields )
{
if ( ! ext_rec )
@ -1128,21 +1153,23 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, R
continue;
}
val = ext_rec.get();
val = ZVal(ext_rec.get());
vt = ext_rec->GetType().get();
}
else
val = columns;
{
val = ZVal(columns);
vt = columns->GetType().get();
}
// For each field, first find the right value, which can
// potentially be nested inside other records.
list<int>& indices = filter->indices[i];
ValPtr val_ptr;
for ( list<int>::iterator j = indices.begin(); j != indices.end(); ++j )
{
val_ptr = val->AsRecordVal()->GetField(*j);
val = val_ptr.get();
auto vr = val->AsRecord();
val = vr->RawOptField(*j);
if ( ! val )
{
@ -1150,10 +1177,12 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, R
vals[i] = new threading::Value(filter->fields[i]->type, false);
break;
}
vt = cast_intrusive<RecordType>(vr->GetType())->GetFieldType(*j).get();
}
if ( val )
vals[i] = ValToLogVal(val);
vals[i] = ValToLogVal(val, vt);
}
return vals;

2
src/logging/Manager.h
View file

@ -291,7 +291,7 @@ private:
threading::Value** RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns);
threading::Value* ValToLogVal(Val* val, Type* ty = nullptr);
threading::Value* ValToLogVal(std::optional<ZVal>& val, Type* ty);
Stream* FindStream(EnumVal* id);
void RemoveDisabledWriters(Stream* stream);
void InstallRotationTimer(WriterInfo* winfo);

3
src/script_opt/CPP/Compile.h
View file

@ -820,7 +820,8 @@ private:
std::string GenIndexAssign(const ExprPtr& lhs, const ExprPtr& rhs,
const std::string& rhs_val_ptr, GenType gt, bool top_level);
std::string GenFieldAssign(const ExprPtr& lhs, const ExprPtr& rhs,
const std::string& rhs_val_ptr, GenType gt, bool top_level);
const std::string& rhs_native, const std::string& rhs_val_ptr,
GenType gt, bool top_level);
std::string GenListAssign(const ExprPtr& lhs, const ExprPtr& rhs);
// Support for element-by-element vector operations.

45
src/script_opt/CPP/Exprs.cc
View file

@ -396,13 +396,35 @@ string CPPCompile::GenInExpr(const Expr* e, GenType gt)
string CPPCompile::GenFieldExpr(const FieldExpr* fe, GenType gt)
{
auto& t = fe->GetType();
auto r = fe->GetOp1();
auto f = fe->Field();
auto f_s = GenField(r, f);
auto gen = string("field_access__CPP(") + GenExpr(r, GEN_VAL_PTR) + ", " + f_s + ")";
string gen;
return GenericValPtrToGT(gen, fe->GetType(), gt);
if ( IsNativeType(t) )
{
auto nt = TypeName(t);
gen = string("field_access_") + nt + "__CPP(" + GenExpr(r, GEN_VAL_PTR) + ", " + f_s + ")";
return NativeToGT(gen, t, gt);
}
switch ( t->Tag() )
{
case TYPE_FILE:
case TYPE_FUNC:
case TYPE_VOID:
gen = string("field_access__CPP(") + GenExpr(r, GEN_VAL_PTR) + ", " + f_s + ")";
return GenericValPtrToGT(gen, t, gt);
default:
{
auto nt = TypeName(t);
return string("field_access_") + nt + "__CPP(" + GenExpr(r, GEN_VAL_PTR) + ", " + f_s +
")";
}
}
}
string CPPCompile::GenHasFieldExpr(const HasFieldExpr* hfe, GenType gt)
@ -411,8 +433,7 @@ string CPPCompile::GenHasFieldExpr(const HasFieldExpr* hfe, GenType gt)
auto f = hfe->Field();
auto f_s = GenField(r, f);
// Need to use accessors for native types.
auto gen = string("(") + GenExpr(r, GEN_DONT_CARE) + "->GetField(" + f_s + ") != nullptr)";
auto gen = GenExpr(r, GEN_DONT_CARE) + "->HasField(" + f_s + ")";
return NativeToGT(gen, hfe->GetType(), gt);
}
@ -1090,7 +1111,7 @@ string CPPCompile::GenAssign(const ExprPtr& lhs, const ExprPtr& rhs, const strin
return GenIndexAssign(lhs, rhs, rhs_val_ptr, gt, top_level);
case EXPR_FIELD:
return GenFieldAssign(lhs, rhs, rhs_val_ptr, gt, top_level);
return GenFieldAssign(lhs, rhs, rhs_native, rhs_val_ptr, gt, top_level);
case EXPR_LIST:
return GenListAssign(lhs, rhs);
@ -1159,20 +1180,24 @@ string CPPCompile::GenIndexAssign(const ExprPtr& lhs, const ExprPtr& rhs, const
return gen;
}
string CPPCompile::GenFieldAssign(const ExprPtr& lhs, const ExprPtr& rhs, const string& rhs_val_ptr,
GenType gt, bool top_level)
string CPPCompile::GenFieldAssign(const ExprPtr& lhs, const ExprPtr& rhs, const string& rhs_native,
const string& rhs_val_ptr, GenType gt, bool top_level)
{
auto rec = lhs->GetOp1();
auto rec_gen = GenExpr(rec, GEN_VAL_PTR);
auto field = GenField(rec, lhs->AsFieldExpr()->Field());
if ( top_level )
return rec_gen + "->Assign(" + field + ", " + rhs_val_ptr + ")";
else
if ( ! top_level )
{
auto gen = string("assign_field__CPP(") + rec_gen + ", " + field + ", " + rhs_val_ptr + ")";
return GenericValPtrToGT(gen, rhs->GetType(), gt);
}
auto rt = rhs ? rhs->GetType() : nullptr;
if ( rt && (IsNativeType(rt) || rt->Tag() == TYPE_STRING) )
return rec_gen + "->Assign(" + field + ", " + rhs_native + ")";
else
return rec_gen + "->Assign(" + field + ", " + rhs_val_ptr + ")";
}
string CPPCompile::GenListAssign(const ExprPtr& lhs, const ExprPtr& rhs)

44
src/script_opt/CPP/RuntimeOps.h
View file

@ -5,7 +5,7 @@
#pragma once
#include "zeek/Frame.h"
#include "zeek/Val.h"
#include "zeek/OpaqueVal.h"
#include "zeek/script_opt/CPP/Func.h"
namespace zeek
@ -16,6 +16,12 @@ using SubNetValPtr = IntrusivePtr<zeek::SubNetVal>;
namespace detail
{
class CPPRuntime
{
public:
static auto RawOptField(const RecordValPtr& rv, int field) { return rv->RawOptField(field); }
};
// Returns the concatenation of the given strings.
extern StringValPtr str_concat__CPP(const String* s1, const String* s2);
@ -109,6 +115,42 @@ inline ValPtr field_access__CPP(const RecordValPtr& rec, int field)
return v;
}
#define NATIVE_FIELD_ACCESS(type, zaccessor, vaccessor) \
inline type field_access_##type##__CPP(const RecordValPtr& r, int field) \
{ \
auto rv = CPPRuntime::RawOptField(r, field); \
if ( rv ) \
return (*rv).zaccessor(); \
return field_access__CPP(r, field)->vaccessor(); \
}
NATIVE_FIELD_ACCESS(bool, AsInt, AsBool)
NATIVE_FIELD_ACCESS(int, AsInt, AsInt)
NATIVE_FIELD_ACCESS(zeek_int_t, AsInt, AsInt)
NATIVE_FIELD_ACCESS(zeek_uint_t, AsCount, AsCount)
NATIVE_FIELD_ACCESS(double, AsDouble, AsDouble)
#define VP_FIELD_ACCESS(type, zaccessor) \
inline type##Ptr field_access_##type##__CPP(const RecordValPtr& r, int field) \
{ \
auto rv = CPPRuntime::RawOptField(r, field); \
if ( rv ) \
return {NewRef{}, rv->zaccessor()}; \
return cast_intrusive<type>(field_access__CPP(r, field)); \
}
VP_FIELD_ACCESS(StringVal, AsString)
VP_FIELD_ACCESS(AddrVal, AsAddr)
VP_FIELD_ACCESS(SubNetVal, AsSubNet)
VP_FIELD_ACCESS(ListVal, AsList)
VP_FIELD_ACCESS(OpaqueVal, AsOpaque)
VP_FIELD_ACCESS(PatternVal, AsPattern)
VP_FIELD_ACCESS(TableVal, AsTable)
VP_FIELD_ACCESS(RecordVal, AsRecord)
VP_FIELD_ACCESS(VectorVal, AsVector)
VP_FIELD_ACCESS(TypeVal, AsType)
VP_FIELD_ACCESS(Val, AsAny)
// Each of the following executes the assignment "v1[v2] = v3" for
// tables/vectors/strings.
extern ValPtr assign_to_index__CPP(TableValPtr v1, ValPtr v2, ValPtr v3);

6
testing/btest/Baseline/plugins.hooks/output
View file

@ -737,7 +737,7 @@
0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::STD_DEV, lambda_<5704045257244168718>{ SumStats::calc_std_dev(SumStats::rv)})) -> <no result>
0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::SUM, lambda_<7459411543525688824>{ SumStats::rv$sum = SumStats::rv$sum + SumStats::val})) -> <no result>
0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::TOPK, lambda_<6366101205573988923>{ topk_add(SumStats::rv$topk, to_any_coerceSumStats::obs)})) -> <no result>
0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::UNIQUE, lambda_<6609886180724383051>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce (coerce set() to set[SumStats::Observation]) to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})) -> <no result>
0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::UNIQUE, lambda_<11310474105220719698>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})) -> <no result>
0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::VARIANCE, lambda_<5978956599776442208>{ if (1 < SumStats::rv$num) SumStats::rv$var_s = SumStats::rv$var_s + ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average})) -> <no result>
0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugins, <frame>, ()) -> <no result>
0.000000 MetaHookPost CallFunction(Supervisor::__is_supervisor, <frame>, ()) -> <no result>
@ -2317,7 +2317,7 @@
0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::STD_DEV, lambda_<5704045257244168718>{ SumStats::calc_std_dev(SumStats::rv)}))
0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::SUM, lambda_<7459411543525688824>{ SumStats::rv$sum = SumStats::rv$sum + SumStats::val}))
0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::TOPK, lambda_<6366101205573988923>{ topk_add(SumStats::rv$topk, to_any_coerceSumStats::obs)}))
0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::UNIQUE, lambda_<6609886180724383051>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce (coerce set() to set[SumStats::Observation]) to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals}))
0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::UNIQUE, lambda_<11310474105220719698>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals}))
0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::VARIANCE, lambda_<5978956599776442208>{ if (1 < SumStats::rv$num) SumStats::rv$var_s = SumStats::rv$var_s + ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average}))
0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugins, <frame>, ())
0.000000 MetaHookPre CallFunction(Supervisor::__is_supervisor, <frame>, ())
@ -3896,7 +3896,7 @@
0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::STD_DEV, lambda_<5704045257244168718>{ SumStats::calc_std_dev(SumStats::rv)})
0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::SUM, lambda_<7459411543525688824>{ SumStats::rv$sum = SumStats::rv$sum + SumStats::val})
0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::TOPK, lambda_<6366101205573988923>{ topk_add(SumStats::rv$topk, to_any_coerceSumStats::obs)})
0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::UNIQUE, lambda_<6609886180724383051>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce (coerce set() to set[SumStats::Observation]) to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})
0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::UNIQUE, lambda_<11310474105220719698>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})
0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::VARIANCE, lambda_<5978956599776442208>{ if (1 < SumStats::rv$num) SumStats::rv$var_s = SumStats::rv$var_s + ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average})
0.000000 | HookCallFunction SumStats::register_observe_plugins()
0.000000 | HookCallFunction Supervisor::__is_supervisor()