diff --git a/CHANGES b/CHANGES index bd55cbe9d2..1466070107 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,24 @@ +6.0.0-dev.385 | 2023-04-19 11:58:09 +0200 + + * different fix for MSVC compiler issues (Vern Paxson, Corelight) + + * more general approach for addressing MSVC compiler issues with IntrusivePtr (Vern Paxson, Corelight) + + * restored RecordType::Create, now marked as deprecated (Vern Paxson, Corelight) + tidying of namespaces and private class members + simplification of flagging record field initializations that should be skipped + address peculiar MSVC compilation complaint for IntrusivePtr's + + * clarifications and tidying for record field initializations (Vern Paxson, Corelight) + + * optimize record construction by deferring initializations of aggregates (Vern Paxson, Corelight) + + * compile-scripts-to-C++ speedups by switching to raw record access (Vern Paxson, Corelight) + + * logging speedup by switching to raw record access (Vern Paxson, Corelight) + + * remove redundant record coercions (Vern Paxson, Corelight) + 6.0.0-dev.376 | 2023-04-19 10:14:02 +0200 * Improve CMake variables, update cmake submodule (Dominik Charousset, Corelight) diff --git a/VERSION b/VERSION index b16520cb1c..f431765c08 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.0.0-dev.376 +6.0.0-dev.385 diff --git a/src/Expr.cc b/src/Expr.cc index 68933eb228..1a7c06020a 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -4215,6 +4215,10 @@ TableCoerceExpr::TableCoerceExpr(ExprPtr arg_op, TableTypePtr tt, bool type_chec SetError(); return; } + + if ( op->Tag() == EXPR_TABLE_COERCE && op->GetType() == tt ) + // Avoid double-coercion. + op = op->GetOp1(); } SetType(std::move(tt)); diff --git a/src/Type.cc b/src/Type.cc index 9db32d6ae9..e624809b62 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -992,41 +992,118 @@ void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const } } -// The following tracks how to initialize a given field, for fast execution -// of Create(). +namespace detail + { -class FieldInit +// A record field initialization that directly assigns a fixed value ... +class DirectFieldInit final : public FieldInit { public: - // The type of initialization for the field. - enum - { - R_INIT_NONE, // skip this entry + DirectFieldInit(ZVal _init_val) : init_val(_init_val) { } - R_INIT_DIRECT, // look in direct_init for raw value - R_INIT_DIRECT_MANAGED, // same, but managed type + ZVal Generate() const override { return init_val; } - R_INIT_DEF, // look in def_expr for expression - - R_INIT_RECORD, // field requires a new record - R_INIT_TABLE, // field requires a new table/set - R_INIT_VECTOR, // field requires a new vector - } init_type = R_INIT_NONE; - - bool def_coerce = false; // whether coercion's required - - // For R_INIT_DIRECT/R_INIT_DIRECT_MANAGED: - ZVal direct_init; - - detail::ExprPtr def_expr; - TypePtr def_type; - - RecordTypePtr r_type; // for R_INIT_RECORD - TableTypePtr t_type; // for R_INIT_TABLE - detail::AttributesPtr attrs; // attributes for R_INIT_TABLE - VectorTypePtr v_type; // for R_INIT_VECTOR +private: + ZVal init_val; }; +// ... the same, but for a value that needs memory management. +class DirectManagedFieldInit final : public FieldInit + { +public: + DirectManagedFieldInit(ZVal _init_val) : init_val(_init_val) { } + ~DirectManagedFieldInit() { ZVal::DeleteManagedType(init_val); } + + ZVal Generate() const override + { + zeek::Ref(init_val.ManagedVal()); + return init_val; + } + +private: + ZVal init_val; + }; + +// A record field initialization that's done by evaluating an expression. +class ExprFieldInit final : public FieldInit + { +public: + // Initialization requires evaluating the given expression, + // yielding the a value of the given type (which might require + // coercion for some records). + ExprFieldInit(detail::ExprPtr _init_expr, TypePtr _init_type) + : init_expr(std::move(_init_expr)), init_type(std::move(_init_type)) + { + if ( init_type->Tag() == TYPE_RECORD && ! same_type(init_expr->GetType(), init_type) ) + coerce_type = cast_intrusive(init_type); + } + + ZVal Generate() const override + { + auto v = init_expr->Eval(nullptr); + if ( ! v ) + { + reporter->Error("failed &default in record creation"); + return ZVal(); + } + + if ( coerce_type ) + v = v->AsRecordVal()->CoerceTo(coerce_type); + + return ZVal(v, init_type); + } + +private: + detail::ExprPtr init_expr; + TypePtr init_type; + RecordTypePtr coerce_type; // non-nil iff coercion is required + }; + +// A record field initialization where the field is initialized to an +// empty/default record of the given type. +class RecordFieldInit final : public FieldInit + { +public: + RecordFieldInit(RecordTypePtr _init_type) : init_type(std::move(_init_type)) { } + + ZVal Generate() const override { return ZVal(new RecordVal(init_type)); } + +private: + RecordTypePtr init_type; + }; + +// A record field initialization where the field is initialized to an +// empty table of the given type. +class TableFieldInit final : public FieldInit + { +public: + TableFieldInit(TableTypePtr _init_type, detail::AttributesPtr _attrs) + : init_type(std::move(_init_type)), attrs(std::move(_attrs)) + { + } + + ZVal Generate() const override { return ZVal(new TableVal(init_type, attrs)); } + +private: + TableTypePtr init_type; + detail::AttributesPtr attrs; + }; + +// A record field initialization where the field is initialized to an +// empty vector of the given type. +class VectorFieldInit final : public FieldInit + { +public: + VectorFieldInit(VectorTypePtr _init_type) : init_type(std::move(_init_type)) { } + + ZVal Generate() const override { return ZVal(new VectorVal(init_type)); } + +private: + VectorTypePtr init_type; + }; + + } // namespace detail + RecordType::RecordType(type_decl_list* arg_types) : Type(TYPE_RECORD) { types = arg_types; @@ -1062,65 +1139,52 @@ RecordType::~RecordType() delete types; } - - for ( auto fi : field_inits ) - delete fi; } void RecordType::AddField(unsigned int field, const TypeDecl* td) { - ASSERT(field == field_inits.size()); + ASSERT(field == deferred_inits.size()); ASSERT(field == managed_fields.size()); managed_fields.push_back(ZVal::IsManagedType(td->type)); - auto init = new FieldInit(); - init->init_type = FieldInit::R_INIT_NONE; - - init->attrs = td->attrs; - - // We defer error-checking until here so that we can keep field_inits + // We defer error-checking until here so that we can keep deferred_inits // and managed_fields correctly tracking the associated fields. if ( field_ids.count(td->id) != 0 ) { reporter->Error("duplicate field '%s' found in record definition", td->id); - field_inits.push_back(init); + deferred_inits.push_back(nullptr); return; } field_ids.insert(std::string(td->id)); - auto a = init->attrs; - + auto a = td->attrs; auto type = td->type; auto def_attr = a ? a->Find(detail::ATTR_DEFAULT) : nullptr; auto def_expr = def_attr ? def_attr->GetExpr() : nullptr; + std::unique_ptr init; + if ( def_expr && ! IsErrorType(type->Tag()) ) { - if ( type->Tag() == TYPE_RECORD && def_expr->GetType()->Tag() == TYPE_RECORD && - ! same_type(def_expr->GetType(), type) ) - init->def_coerce = true; - if ( def_expr->Tag() == detail::EXPR_CONST ) { auto v = def_expr->Eval(nullptr); + auto zv = ZVal(v, type); if ( ZVal::IsManagedType(type) ) - init->init_type = FieldInit::R_INIT_DIRECT_MANAGED; + init = std::make_unique(zv); else - init->init_type = FieldInit::R_INIT_DIRECT; - - init->direct_init = ZVal(v, type); + init = std::make_unique(zv); } else { - init->init_type = FieldInit::R_INIT_DEF; - init->def_expr = def_expr; - init->def_type = def_expr->GetType(); + auto efi = std::make_unique(def_expr, type); + creation_inits.emplace_back(std::make_pair(field, std::move(efi))); } } @@ -1129,25 +1193,16 @@ void RecordType::AddField(unsigned int field, const TypeDecl* td) TypeTag tag = type->Tag(); if ( tag == TYPE_RECORD ) - { - init->init_type = FieldInit::R_INIT_RECORD; - init->r_type = cast_intrusive(type); - } + init = std::make_unique(cast_intrusive(type)); else if ( tag == TYPE_TABLE ) - { - init->init_type = FieldInit::R_INIT_TABLE; - init->t_type = cast_intrusive(type); - } + init = std::make_unique(cast_intrusive(type), a); else if ( tag == TYPE_VECTOR ) - { - init->init_type = FieldInit::R_INIT_VECTOR; - init->v_type = cast_intrusive(type); - } + init = std::make_unique(cast_intrusive(type)); } - field_inits.push_back(init); + deferred_inits.push_back(std::move(init)); } bool RecordType::HasField(const char* field) const @@ -1344,64 +1399,14 @@ void RecordType::AddFieldsDirectly(const type_decl_list& others, bool add_log_at void RecordType::Create(std::vector>& r) const { - int n = NumFields(); + for ( auto& di : deferred_inits ) + if ( di ) + r.push_back(di->Generate()); + else + r.push_back(std::nullopt); - for ( int i = 0; i < n; ++i ) - { - auto* init = field_inits[i]; - - ZVal r_i; - - switch ( init->init_type ) - { - case FieldInit::R_INIT_NONE: - r.push_back(std::nullopt); - continue; - - case FieldInit::R_INIT_DIRECT: - r_i = init->direct_init; - break; - - case FieldInit::R_INIT_DIRECT_MANAGED: - r_i = init->direct_init; - zeek::Ref(r_i.ManagedVal()); - break; - - case FieldInit::R_INIT_DEF: - { - auto v = init->def_expr->Eval(nullptr); - if ( v ) - { - const auto& t = init->def_type; - - if ( init->def_coerce ) - { - auto rt = cast_intrusive(t); - v = v->AsRecordVal()->CoerceTo(rt); - } - - r_i = ZVal(v, t); - } - else - reporter->Error("failed &default in record creation"); - } - break; - - case FieldInit::R_INIT_RECORD: - r_i = ZVal(new RecordVal(init->r_type)); - break; - - case FieldInit::R_INIT_TABLE: - r_i = ZVal(new TableVal(init->t_type, init->attrs)); - break; - - case FieldInit::R_INIT_VECTOR: - r_i = ZVal(new VectorVal(init->v_type)); - break; - } - - r.push_back(r_i); - } + for ( auto& ci : creation_inits ) + r[ci.first] = ci.second->Generate(); } void RecordType::DescribeFields(ODesc* d) const diff --git a/src/Type.h b/src/Type.h index d4bc1e130b..be7667d8bf 100644 --- a/src/Type.h +++ b/src/Type.h @@ -22,6 +22,7 @@ namespace zeek class Val; union ZVal; class EnumVal; +class RecordVal; class TableVal; using ValPtr = IntrusivePtr; using EnumValPtr = IntrusivePtr; @@ -35,6 +36,16 @@ class ListExpr; class Attributes; using ListExprPtr = IntrusivePtr; +// The following tracks how to initialize a given record field. +class FieldInit + { +public: + virtual ~FieldInit() { } + + // Return the initialization value of the field. + virtual ZVal Generate() const = 0; + }; + } // namespace detail // Zeek types. @@ -599,11 +610,6 @@ public: using type_decl_list = PList; -// The following tracks how to initialize a given field. We don't define -// it here because it requires pulling in a bunch of low-level headers that -// would be nice to avoid. -class FieldInit; - class RecordType final : public Type { public: @@ -687,7 +693,9 @@ public: * Populates a new instance of the record with its initial values. * @param r The record's underlying value vector. */ - void Create(std::vector>& r) const; + [[deprecated("Remove in v6.1. Construct a corresponding RecordVal and build vector from " + "GetFieldAs() calls.")]] void + Create(std::vector>& r) const; void DescribeReST(ODesc* d, bool roles_only = false) const override; void DescribeFields(ODesc* d) const; @@ -709,16 +717,29 @@ public: detail::TraversalCode Traverse(detail::TraversalCallback* cb) const override; -protected: +private: RecordType() { types = nullptr; } void AddField(unsigned int field, const TypeDecl* td); void DoDescribe(ODesc* d) const override; - // Maps each field to how to initialize it. Uses pointers due to - // keeping the FieldInit definition private to Type.cc (see above). - std::vector field_inits; + // Field initializations that can be deferred to first access, + // beneficial for fields that are separately initialized prior + // to first access. Nil pointers mean "skip initializing the field". + std::vector> deferred_inits; + + // Field initializations that need to be done upon record creation, + // rather than deferred. These are expressions whose value might + // change if computed later. + // + // Such initializations are uncommon, so we represent them using + // pairs. + std::vector>> creation_inits; + + friend zeek::RecordVal; + const auto& DeferredInits() const { return deferred_inits; } + const auto& CreationInits() const { return creation_inits; } // If we were willing to bound the size of records, then we could // use std::bitset here instead. diff --git a/src/Val.cc b/src/Val.cc index 208c564bf6..9b94c2dd97 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2759,25 +2759,32 @@ RecordVal::RecordVal(RecordTypePtr t, bool init_fields) : Val(t), is_managed(t-> int n = rt->NumFields(); - record_val = new std::vector>; - record_val->reserve(n); - if ( run_state::is_parsing ) parse_time_records[rt.get()].emplace_back(NewRef{}, this); + record_val = new std::vector>; + if ( init_fields ) { - try + record_val->resize(n); + + for ( auto& e : rt->CreationInits() ) { - rt->Create(*record_val); - } - catch ( InterpreterException& e ) - { - if ( run_state::is_parsing ) - parse_time_records[rt.get()].pop_back(); - throw; + try + { + (*record_val)[e.first] = e.second->Generate(); + } + catch ( InterpreterException& e ) + { + if ( run_state::is_parsing ) + parse_time_records[rt.get()].pop_back(); + throw; + } } } + + else + record_val->reserve(n); } RecordVal::~RecordVal() @@ -2785,8 +2792,11 @@ RecordVal::~RecordVal() auto n = record_val->size(); for ( unsigned int i = 0; i < n; ++i ) - if ( HasField(i) && IsManaged(i) ) - ZVal::DeleteManagedType(*(*record_val)[i]); + { + auto f_i = (*record_val)[i]; + if ( f_i && IsManaged(i) ) + ZVal::DeleteManagedType(*f_i); + } delete record_val; } @@ -2812,12 +2822,13 @@ void RecordVal::Assign(int field, ValPtr new_val) void RecordVal::Remove(int field) { - if ( HasField(field) ) + auto& f_i = (*record_val)[field]; + if ( f_i ) { if ( IsManaged(field) ) - ZVal::DeleteManagedType(*(*record_val)[field]); + ZVal::DeleteManagedType(*f_i); - (*record_val)[field] = std::nullopt; + f_i = std::nullopt; Modified(); } diff --git a/src/Val.h b/src/Val.h index e73536f127..7290a6eacd 100644 --- a/src/Val.h +++ b/src/Val.h @@ -52,9 +52,15 @@ class HashKey; class ValTrace; class ZBody; +class CPPRuntime; } // namespace detail +namespace logging + { +class Manager; + } + namespace run_state { @@ -1173,9 +1179,10 @@ public: void Assign(int field, StringVal* new_val) { - if ( HasField(field) ) - ZVal::DeleteManagedType(*(*record_val)[field]); - (*record_val)[field] = ZVal(new_val); + auto& fv = (*record_val)[field]; + if ( fv ) + ZVal::DeleteManagedType(*fv); + fv = ZVal(new_val); AddedField(field); } void Assign(int field, const char* new_val) { Assign(field, new StringVal(new_val)); } @@ -1188,7 +1195,7 @@ public: */ template void AssignField(const char* field_name, T&& val) { - int idx = GetType()->AsRecordType()->FieldOffset(field_name); + int idx = rt->FieldOffset(field_name); if ( idx < 0 ) reporter->InternalError("missing record field: %s", field_name); Assign(idx, std::forward(val)); @@ -1206,7 +1213,13 @@ public: * @param field The field index to retrieve. * @return Whether there's a value for the given field index. */ - bool HasField(int field) const { return (*record_val)[field] ? true : false; } + bool HasField(int field) const + { + if ( (*record_val)[field] ) + return true; + + return rt->DeferredInits()[field] != nullptr; + } /** * Returns true if the given field is in the record, false if @@ -1216,7 +1229,7 @@ public: */ bool HasField(const char* field) const { - int idx = GetType()->AsRecordType()->FieldOffset(field); + int idx = rt->FieldOffset(field); return (idx != -1) && HasField(idx); } @@ -1227,10 +1240,17 @@ public: */ ValPtr GetField(int field) const { - if ( ! HasField(field) ) - return nullptr; + auto& fv = (*record_val)[field]; + if ( ! fv ) + { + const auto& fi = rt->DeferredInits()[field]; + if ( ! fi ) + return nullptr; - return (*record_val)[field]->ToVal(rt->GetFieldType(field)); + fv = fi->Generate(); + } + + return fv->ToVal(rt->GetFieldType(field)); } /** @@ -1358,7 +1378,7 @@ public: template auto GetFieldAs(const char* field) const { - int idx = GetType()->AsRecordType()->FieldOffset(field); + int idx = rt->FieldOffset(field); if ( idx < 0 ) reporter->InternalError("missing record field: %s", field); @@ -1403,8 +1423,10 @@ public: static void DoneParsing(); protected: + friend class zeek::logging::Manager; friend class zeek::detail::ValTrace; friend class zeek::detail::ZBody; + friend class zeek::detail::CPPRuntime; RecordValPtr DoCoerceTo(RecordTypePtr other, bool allow_orphaning) const; @@ -1429,7 +1451,18 @@ protected: // Caller assumes responsibility for memory management. The first // version allows manipulation of whether the field is present at all. // The second version ensures that the optional value is present. - std::optional& RawOptField(int field) { return (*record_val)[field]; } + std::optional& RawOptField(int field) + { + auto& f = (*record_val)[field]; + if ( ! f ) + { + const auto& fi = rt->DeferredInits()[field]; + if ( fi ) + f = fi->Generate(); + } + + return f; + } ZVal& RawField(int field) { @@ -1451,8 +1484,9 @@ protected: private: void DeleteFieldIfManaged(unsigned int field) { - if ( HasField(field) && IsManaged(field) ) - ZVal::DeleteManagedType(*(*record_val)[field]); + auto& f = (*record_val)[field]; + if ( f && IsManaged(field) ) + ZVal::DeleteManagedType(*f); } bool IsManaged(unsigned int offset) const { return is_managed[offset]; } diff --git a/src/ZVal.h b/src/ZVal.h index 202659d17b..04a0717bb8 100644 --- a/src/ZVal.h +++ b/src/ZVal.h @@ -10,6 +10,7 @@ namespace zeek { class AddrVal; +class EnumVal; class File; class Func; class ListVal; diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index b02a51c1e6..6d9955b8e5 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -973,11 +973,8 @@ bool Manager::Write(EnumVal* id, RecordVal* columns_arg) return true; } -threading::Value* Manager::ValToLogVal(Val* val, Type* ty) +threading::Value* Manager::ValToLogVal(std::optional& val, Type* ty) { - if ( ! ty ) - ty = val->GetType().get(); - if ( ! val ) return new threading::Value(ty->Tag(), false); @@ -987,12 +984,12 @@ threading::Value* Manager::ValToLogVal(Val* val, Type* ty) { case TYPE_BOOL: case TYPE_INT: - lval->val.int_val = val->InternalInt(); + lval->val.int_val = val->AsInt(); break; case TYPE_ENUM: { - const char* s = val->GetType()->AsEnumType()->Lookup(val->InternalInt()); + const char* s = ty->AsEnumType()->Lookup(val->AsInt()); if ( s ) { @@ -1002,7 +999,8 @@ threading::Value* Manager::ValToLogVal(Val* val, Type* ty) else { - val->GetType()->Error("enum type does not contain value", val); + auto err_msg = "enum type does not contain value:" + std::to_string(val->AsInt()); + ty->Error(err_msg.c_str()); lval->val.string_val.data = util::copy_string(""); lval->val.string_val.length = 0; } @@ -1010,31 +1008,44 @@ threading::Value* Manager::ValToLogVal(Val* val, Type* ty) } case TYPE_COUNT: - lval->val.uint_val = val->InternalUnsigned(); + lval->val.uint_val = val->AsCount(); break; case TYPE_PORT: - lval->val.port_val.port = val->AsPortVal()->Port(); - lval->val.port_val.proto = val->AsPortVal()->PortType(); + { + auto p = val->AsCount(); + + auto pt = TRANSPORT_UNKNOWN; + auto pm = p & PORT_SPACE_MASK; + if ( pm == TCP_PORT_MASK ) + pt = TRANSPORT_TCP; + else if ( pm == UDP_PORT_MASK ) + pt = TRANSPORT_UDP; + else if ( pm == ICMP_PORT_MASK ) + pt = TRANSPORT_ICMP; + + lval->val.port_val.port = p & ~PORT_SPACE_MASK; + lval->val.port_val.proto = pt; break; + } case TYPE_SUBNET: - val->AsSubNet().ConvertToThreadingValue(&lval->val.subnet_val); + val->AsSubNet()->Get().ConvertToThreadingValue(&lval->val.subnet_val); break; case TYPE_ADDR: - val->AsAddr().ConvertToThreadingValue(&lval->val.addr_val); + val->AsAddr()->Get().ConvertToThreadingValue(&lval->val.addr_val); break; case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - lval->val.double_val = val->InternalDouble(); + lval->val.double_val = val->AsDouble(); break; case TYPE_STRING: { - const String* s = val->AsString(); + const String* s = val->AsString()->AsString(); char* buf = new char[s->Len()]; memcpy(buf, s->Bytes(), s->Len()); @@ -1065,31 +1076,44 @@ threading::Value* Manager::ValToLogVal(Val* val, Type* ty) case TYPE_TABLE: { - auto set = val->AsTableVal()->ToPureListVal(); + auto tbl = val->AsTable(); + auto set = tbl->ToPureListVal(); + if ( ! set ) // ToPureListVal has reported an internal warning // already. Just keep going by making something up. set = make_intrusive(TYPE_INT); + auto tbl_t = cast_intrusive(tbl->GetType()); + auto& set_t = tbl_t->GetIndexTypes()[0]; + bool is_managed = ZVal::IsManagedType(set_t); + lval->val.set_val.size = set->Length(); lval->val.set_val.vals = new threading::Value*[lval->val.set_val.size]; for ( zeek_int_t i = 0; i < lval->val.set_val.size; i++ ) - lval->val.set_val.vals[i] = ValToLogVal(set->Idx(i).get()); + { + std::optional s_i = ZVal(set->Idx(i), set_t); + lval->val.set_val.vals[i] = ValToLogVal(s_i, set_t.get()); + if ( is_managed ) + ZVal::DeleteManagedType(*s_i); + } break; } case TYPE_VECTOR: { - VectorVal* vec = val->AsVectorVal(); + VectorVal* vec = val->AsVector(); lval->val.vector_val.size = vec->Size(); lval->val.vector_val.vals = new threading::Value*[lval->val.vector_val.size]; + auto& vv = vec->RawVec(); + auto& vt = vec->GetType()->Yield(); + for ( zeek_int_t i = 0; i < lval->val.vector_val.size; i++ ) { - lval->val.vector_val.vals[i] = ValToLogVal(vec->ValAt(i).get(), - vec->GetType()->Yield().get()); + lval->val.vector_val.vals[i] = ValToLogVal((*vv)[i], vt.get()); } break; @@ -1118,7 +1142,8 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, R for ( int i = 0; i < filter->num_fields; ++i ) { - Val* val; + std::optional val; + Type* vt; if ( i < filter->num_ext_fields ) { if ( ! ext_rec ) @@ -1128,21 +1153,23 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, R continue; } - val = ext_rec.get(); + val = ZVal(ext_rec.get()); + vt = ext_rec->GetType().get(); } else - val = columns; + { + val = ZVal(columns); + vt = columns->GetType().get(); + } // For each field, first find the right value, which can // potentially be nested inside other records. list& indices = filter->indices[i]; - ValPtr val_ptr; - for ( list::iterator j = indices.begin(); j != indices.end(); ++j ) { - val_ptr = val->AsRecordVal()->GetField(*j); - val = val_ptr.get(); + auto vr = val->AsRecord(); + val = vr->RawOptField(*j); if ( ! val ) { @@ -1150,10 +1177,12 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, R vals[i] = new threading::Value(filter->fields[i]->type, false); break; } + + vt = cast_intrusive(vr->GetType())->GetFieldType(*j).get(); } if ( val ) - vals[i] = ValToLogVal(val); + vals[i] = ValToLogVal(val, vt); } return vals; diff --git a/src/logging/Manager.h b/src/logging/Manager.h index 6f926242f3..cb2e554d5c 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -291,7 +291,7 @@ private: threading::Value** RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns); - threading::Value* ValToLogVal(Val* val, Type* ty = nullptr); + threading::Value* ValToLogVal(std::optional& val, Type* ty); Stream* FindStream(EnumVal* id); void RemoveDisabledWriters(Stream* stream); void InstallRotationTimer(WriterInfo* winfo); diff --git a/src/script_opt/CPP/Compile.h b/src/script_opt/CPP/Compile.h index 9a99db499c..6b110a00de 100644 --- a/src/script_opt/CPP/Compile.h +++ b/src/script_opt/CPP/Compile.h @@ -820,7 +820,8 @@ private: std::string GenIndexAssign(const ExprPtr& lhs, const ExprPtr& rhs, const std::string& rhs_val_ptr, GenType gt, bool top_level); std::string GenFieldAssign(const ExprPtr& lhs, const ExprPtr& rhs, - const std::string& rhs_val_ptr, GenType gt, bool top_level); + const std::string& rhs_native, const std::string& rhs_val_ptr, + GenType gt, bool top_level); std::string GenListAssign(const ExprPtr& lhs, const ExprPtr& rhs); // Support for element-by-element vector operations. diff --git a/src/script_opt/CPP/Exprs.cc b/src/script_opt/CPP/Exprs.cc index 6dd82356a5..b0ec6cd8e4 100644 --- a/src/script_opt/CPP/Exprs.cc +++ b/src/script_opt/CPP/Exprs.cc @@ -396,13 +396,35 @@ string CPPCompile::GenInExpr(const Expr* e, GenType gt) string CPPCompile::GenFieldExpr(const FieldExpr* fe, GenType gt) { + auto& t = fe->GetType(); auto r = fe->GetOp1(); auto f = fe->Field(); auto f_s = GenField(r, f); - auto gen = string("field_access__CPP(") + GenExpr(r, GEN_VAL_PTR) + ", " + f_s + ")"; + string gen; - return GenericValPtrToGT(gen, fe->GetType(), gt); + if ( IsNativeType(t) ) + { + auto nt = TypeName(t); + gen = string("field_access_") + nt + "__CPP(" + GenExpr(r, GEN_VAL_PTR) + ", " + f_s + ")"; + return NativeToGT(gen, t, gt); + } + + switch ( t->Tag() ) + { + case TYPE_FILE: + case TYPE_FUNC: + case TYPE_VOID: + gen = string("field_access__CPP(") + GenExpr(r, GEN_VAL_PTR) + ", " + f_s + ")"; + return GenericValPtrToGT(gen, t, gt); + + default: + { + auto nt = TypeName(t); + return string("field_access_") + nt + "__CPP(" + GenExpr(r, GEN_VAL_PTR) + ", " + f_s + + ")"; + } + } } string CPPCompile::GenHasFieldExpr(const HasFieldExpr* hfe, GenType gt) @@ -411,8 +433,7 @@ string CPPCompile::GenHasFieldExpr(const HasFieldExpr* hfe, GenType gt) auto f = hfe->Field(); auto f_s = GenField(r, f); - // Need to use accessors for native types. - auto gen = string("(") + GenExpr(r, GEN_DONT_CARE) + "->GetField(" + f_s + ") != nullptr)"; + auto gen = GenExpr(r, GEN_DONT_CARE) + "->HasField(" + f_s + ")"; return NativeToGT(gen, hfe->GetType(), gt); } @@ -1090,7 +1111,7 @@ string CPPCompile::GenAssign(const ExprPtr& lhs, const ExprPtr& rhs, const strin return GenIndexAssign(lhs, rhs, rhs_val_ptr, gt, top_level); case EXPR_FIELD: - return GenFieldAssign(lhs, rhs, rhs_val_ptr, gt, top_level); + return GenFieldAssign(lhs, rhs, rhs_native, rhs_val_ptr, gt, top_level); case EXPR_LIST: return GenListAssign(lhs, rhs); @@ -1159,20 +1180,24 @@ string CPPCompile::GenIndexAssign(const ExprPtr& lhs, const ExprPtr& rhs, const return gen; } -string CPPCompile::GenFieldAssign(const ExprPtr& lhs, const ExprPtr& rhs, const string& rhs_val_ptr, - GenType gt, bool top_level) +string CPPCompile::GenFieldAssign(const ExprPtr& lhs, const ExprPtr& rhs, const string& rhs_native, + const string& rhs_val_ptr, GenType gt, bool top_level) { auto rec = lhs->GetOp1(); auto rec_gen = GenExpr(rec, GEN_VAL_PTR); auto field = GenField(rec, lhs->AsFieldExpr()->Field()); - if ( top_level ) - return rec_gen + "->Assign(" + field + ", " + rhs_val_ptr + ")"; - else + if ( ! top_level ) { auto gen = string("assign_field__CPP(") + rec_gen + ", " + field + ", " + rhs_val_ptr + ")"; return GenericValPtrToGT(gen, rhs->GetType(), gt); } + + auto rt = rhs ? rhs->GetType() : nullptr; + if ( rt && (IsNativeType(rt) || rt->Tag() == TYPE_STRING) ) + return rec_gen + "->Assign(" + field + ", " + rhs_native + ")"; + else + return rec_gen + "->Assign(" + field + ", " + rhs_val_ptr + ")"; } string CPPCompile::GenListAssign(const ExprPtr& lhs, const ExprPtr& rhs) diff --git a/src/script_opt/CPP/RuntimeOps.h b/src/script_opt/CPP/RuntimeOps.h index 85cc61eeca..4318e7c7ff 100644 --- a/src/script_opt/CPP/RuntimeOps.h +++ b/src/script_opt/CPP/RuntimeOps.h @@ -5,7 +5,7 @@ #pragma once #include "zeek/Frame.h" -#include "zeek/Val.h" +#include "zeek/OpaqueVal.h" #include "zeek/script_opt/CPP/Func.h" namespace zeek @@ -16,6 +16,12 @@ using SubNetValPtr = IntrusivePtr; namespace detail { +class CPPRuntime + { +public: + static auto RawOptField(const RecordValPtr& rv, int field) { return rv->RawOptField(field); } + }; + // Returns the concatenation of the given strings. extern StringValPtr str_concat__CPP(const String* s1, const String* s2); @@ -109,6 +115,42 @@ inline ValPtr field_access__CPP(const RecordValPtr& rec, int field) return v; } +#define NATIVE_FIELD_ACCESS(type, zaccessor, vaccessor) \ + inline type field_access_##type##__CPP(const RecordValPtr& r, int field) \ + { \ + auto rv = CPPRuntime::RawOptField(r, field); \ + if ( rv ) \ + return (*rv).zaccessor(); \ + return field_access__CPP(r, field)->vaccessor(); \ + } + +NATIVE_FIELD_ACCESS(bool, AsInt, AsBool) +NATIVE_FIELD_ACCESS(int, AsInt, AsInt) +NATIVE_FIELD_ACCESS(zeek_int_t, AsInt, AsInt) +NATIVE_FIELD_ACCESS(zeek_uint_t, AsCount, AsCount) +NATIVE_FIELD_ACCESS(double, AsDouble, AsDouble) + +#define VP_FIELD_ACCESS(type, zaccessor) \ + inline type##Ptr field_access_##type##__CPP(const RecordValPtr& r, int field) \ + { \ + auto rv = CPPRuntime::RawOptField(r, field); \ + if ( rv ) \ + return {NewRef{}, rv->zaccessor()}; \ + return cast_intrusive(field_access__CPP(r, field)); \ + } + +VP_FIELD_ACCESS(StringVal, AsString) +VP_FIELD_ACCESS(AddrVal, AsAddr) +VP_FIELD_ACCESS(SubNetVal, AsSubNet) +VP_FIELD_ACCESS(ListVal, AsList) +VP_FIELD_ACCESS(OpaqueVal, AsOpaque) +VP_FIELD_ACCESS(PatternVal, AsPattern) +VP_FIELD_ACCESS(TableVal, AsTable) +VP_FIELD_ACCESS(RecordVal, AsRecord) +VP_FIELD_ACCESS(VectorVal, AsVector) +VP_FIELD_ACCESS(TypeVal, AsType) +VP_FIELD_ACCESS(Val, AsAny) + // Each of the following executes the assignment "v1[v2] = v3" for // tables/vectors/strings. extern ValPtr assign_to_index__CPP(TableValPtr v1, ValPtr v2, ValPtr v3); diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index ebc63eaed5..30dd240bad 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -737,7 +737,7 @@ 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, , (SumStats::STD_DEV, lambda_<5704045257244168718>{ SumStats::calc_std_dev(SumStats::rv)})) -> 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, , (SumStats::SUM, lambda_<7459411543525688824>{ SumStats::rv$sum = SumStats::rv$sum + SumStats::val})) -> 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, , (SumStats::TOPK, lambda_<6366101205573988923>{ topk_add(SumStats::rv$topk, to_any_coerceSumStats::obs)})) -> -0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, , (SumStats::UNIQUE, lambda_<6609886180724383051>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce (coerce set() to set[SumStats::Observation]) to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})) -> +0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, , (SumStats::UNIQUE, lambda_<11310474105220719698>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})) -> 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, , (SumStats::VARIANCE, lambda_<5978956599776442208>{ if (1 < SumStats::rv$num) SumStats::rv$var_s = SumStats::rv$var_s + ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average})) -> 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(Supervisor::__is_supervisor, , ()) -> @@ -2317,7 +2317,7 @@ 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, , (SumStats::STD_DEV, lambda_<5704045257244168718>{ SumStats::calc_std_dev(SumStats::rv)})) 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, , (SumStats::SUM, lambda_<7459411543525688824>{ SumStats::rv$sum = SumStats::rv$sum + SumStats::val})) 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, , (SumStats::TOPK, lambda_<6366101205573988923>{ topk_add(SumStats::rv$topk, to_any_coerceSumStats::obs)})) -0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, , (SumStats::UNIQUE, lambda_<6609886180724383051>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce (coerce set() to set[SumStats::Observation]) to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})) +0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, , (SumStats::UNIQUE, lambda_<11310474105220719698>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})) 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, , (SumStats::VARIANCE, lambda_<5978956599776442208>{ if (1 < SumStats::rv$num) SumStats::rv$var_s = SumStats::rv$var_s + ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average})) 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugins, , ()) 0.000000 MetaHookPre CallFunction(Supervisor::__is_supervisor, , ()) @@ -3896,7 +3896,7 @@ 0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::STD_DEV, lambda_<5704045257244168718>{ SumStats::calc_std_dev(SumStats::rv)}) 0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::SUM, lambda_<7459411543525688824>{ SumStats::rv$sum = SumStats::rv$sum + SumStats::val}) 0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::TOPK, lambda_<6366101205573988923>{ topk_add(SumStats::rv$topk, to_any_coerceSumStats::obs)}) -0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::UNIQUE, lambda_<6609886180724383051>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce (coerce set() to set[SumStats::Observation]) to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals}) +0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::UNIQUE, lambda_<11310474105220719698>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals}) 0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::VARIANCE, lambda_<5978956599776442208>{ if (1 < SumStats::rv$num) SumStats::rv$var_s = SumStats::rv$var_s + ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average}) 0.000000 | HookCallFunction SumStats::register_observe_plugins() 0.000000 | HookCallFunction Supervisor::__is_supervisor()