diff --git a/CHANGES b/CHANGES index 956817a65d..476396a99f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,11 @@ +7.2.0-dev.400 | 2025-03-19 12:46:41 -0700 + + * -O gen-C++ support for pattern-to-pattern comparison (Vern Paxson, Corelight) + + * -O gen-C++ support for pattern vector comparisons (Vern Paxson, Corelight) + + * -O gen-C++ support for type expressions (Vern Paxson, Corelight) + 7.2.0-dev.395 | 2025-03-19 11:58:41 -0700 * CI: Run btests on macOS under sudo (Tim Wojtulewicz, Corelight) diff --git a/VERSION b/VERSION index 71ef35b8bb..29c986fe1e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.2.0-dev.395 +7.2.0-dev.400 diff --git a/src/script_opt/CPP/Consts.cc b/src/script_opt/CPP/Consts.cc index 1c51560253..d29f1d6b5f 100644 --- a/src/script_opt/CPP/Consts.cc +++ b/src/script_opt/CPP/Consts.cc @@ -103,6 +103,8 @@ shared_ptr CPPCompile::RegisterConstant(const ValPtr& vp, int& con case TYPE_FUNC: gi = make_shared(this, vp); break; + case TYPE_TYPE: gi = make_shared(this, vp); break; + default: reporter->InternalError("bad constant type in CPPCompile::AddConstant"); break; } diff --git a/src/script_opt/CPP/Driver.cc b/src/script_opt/CPP/Driver.cc index f3cb663aa3..335d520401 100644 --- a/src/script_opt/CPP/Driver.cc +++ b/src/script_opt/CPP/Driver.cc @@ -294,6 +294,7 @@ void CPPCompile::GenProlog() { const_info[TYPE_TABLE] = CreateCompoundInitInfo("Table", "ValPtr"); const_info[TYPE_FUNC] = CreateCompoundInitInfo("Func", "ValPtr"); const_info[TYPE_FILE] = CreateCompoundInitInfo("File", "ValPtr"); + const_info[TYPE_TYPE] = CreateCompoundInitInfo("TypeVal", "Ptr"); type_info = CreateCompoundInitInfo("Type", "Ptr"); attr_info = CreateCompoundInitInfo("Attr", "Ptr"); diff --git a/src/script_opt/CPP/Exprs.cc b/src/script_opt/CPP/Exprs.cc index 5a39d4d850..b51e802cc2 100644 --- a/src/script_opt/CPP/Exprs.cc +++ b/src/script_opt/CPP/Exprs.cc @@ -998,10 +998,23 @@ string CPPCompile::GenEQ(const Expr* e, GenType gt, const char* op, const char* auto tag = op1->GetType()->Tag(); string negated(e->Tag() == EXPR_EQ ? "" : "! "); - if ( tag == TYPE_PATTERN ) - return NativeToGT(negated + GenExpr(op1, GEN_DONT_CARE) + "->MatchExactly(" + GenExpr(op2, GEN_DONT_CARE) + - "->AsString())", - e->GetType(), gt); + if ( tag == TYPE_PATTERN ) { + auto gen1 = GenExpr(op1, GEN_DONT_CARE); + auto gen2 = GenExpr(op2, GEN_DONT_CARE); + string gen; + + if ( op2->GetType()->Tag() == TYPE_PATTERN ) { + gen1 += "->AsPattern()->PatternText()"; + gen2 += "->AsPattern()->PatternText()"; + + gen = "(strcmp(" + gen1 + ", " + gen2 + ") == 0)"; + } + + else + gen = gen1 + "->MatchExactly(" + gen2 + "->AsString())"; + + return NativeToGT(negated + gen, e->GetType(), gt); + } if ( tag == TYPE_FUNC ) { auto gen_f1 = GenExpr(op1, GEN_DONT_CARE); @@ -1156,8 +1169,12 @@ string CPPCompile::GenVectorOp(const Expr* e, string op1, string op2, const char auto invoke = string(vec_op) + "__CPP(" + op1 + ", " + op2 + ")"; - if ( op2_t->Yield()->Tag() == TYPE_STRING ) + auto tag2 = op2_t->Yield()->Tag(); + + if ( tag2 == TYPE_STRING ) return string("str_vec_op_") + invoke; + if ( tag2 == TYPE_PATTERN ) + return string("pat_vec_op_") + invoke; auto gen = string("vec_op_") + invoke; diff --git a/src/script_opt/CPP/InitsInfo.cc b/src/script_opt/CPP/InitsInfo.cc index 122b37b9b2..2500b42806 100644 --- a/src/script_opt/CPP/InitsInfo.cc +++ b/src/script_opt/CPP/InitsInfo.cc @@ -299,6 +299,11 @@ void FuncConstInfo::InitializerVals(std::vector& ivs) const { } } +void TypeConstInfo::InitializerVals(std::vector& ivs) const { + auto& t = tv->GetType()->AsTypeType()->GetType(); + ivs.emplace_back(Fmt(t->Tag())); +} + AttrInfo::AttrInfo(CPPCompile* _c, const AttrPtr& attr) : CompoundItemInfo(_c) { vals.emplace_back(Fmt(static_cast(attr->Tag()))); auto a_e = attr->GetExpr(); diff --git a/src/script_opt/CPP/InitsInfo.h b/src/script_opt/CPP/InitsInfo.h index b7e58b63d1..54c4600af1 100644 --- a/src/script_opt/CPP/InitsInfo.h +++ b/src/script_opt/CPP/InitsInfo.h @@ -467,6 +467,16 @@ private: FuncVal* fv; }; +class TypeConstInfo : public CompoundItemInfo { +public: + TypeConstInfo(CPPCompile* _c, ValPtr v) : CompoundItemInfo(_c, v), tv(v->AsTypeVal()) {} + + void InitializerVals(std::vector& ivs) const override; + +private: + TypeVal* tv; +}; + // Initialization information for single attributes and sets of attributes. class AttrInfo : public CompoundItemInfo { public: diff --git a/src/script_opt/CPP/RuntimeInits.cc b/src/script_opt/CPP/RuntimeInits.cc index b7f28a7403..a24a3a6cdb 100644 --- a/src/script_opt/CPP/RuntimeInits.cc +++ b/src/script_opt/CPP/RuntimeInits.cc @@ -149,6 +149,14 @@ void CPP_IndexedInits::Generate(InitsManager* im, std::vector& iv ivec[offset] = lookup_func__CPP(fn, num_bodies, hashes, im->Types(t)); } +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const { + auto bt = base_type(static_cast(init_vals[0])); + auto t = make_intrusive(bt); + ivec[offset] = make_intrusive(t); +} + template void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, ValElemVec& init_vals) const { @@ -216,6 +224,7 @@ template class CPP_IndexedInits; template class CPP_IndexedInits; template class CPP_IndexedInits; template class CPP_IndexedInits; +template class CPP_IndexedInits; template class CPP_IndexedInits; template class CPP_IndexedInits; template class CPP_IndexedInits; diff --git a/src/script_opt/CPP/RuntimeInits.h b/src/script_opt/CPP/RuntimeInits.h index 4ced70af63..d20f4b7bb5 100644 --- a/src/script_opt/CPP/RuntimeInits.h +++ b/src/script_opt/CPP/RuntimeInits.h @@ -16,6 +16,7 @@ namespace zeek::detail { using FileValPtr = IntrusivePtr; using FuncValPtr = IntrusivePtr; +using TypeValPtr = IntrusivePtr; class InitsManager; @@ -274,6 +275,7 @@ protected: void Generate(InitsManager* im, std::vector& ivec, int offset, ValElemVec& init_vals) const; void Generate(InitsManager* im, std::vector& ivec, int offset, ValElemVec& init_vals) const; void Generate(InitsManager* im, std::vector& ivec, int offset, ValElemVec& init_vals) const; + void Generate(InitsManager* im, std::vector& ivec, int offset, ValElemVec& init_vals) const; void Generate(InitsManager* im, std::vector& ivec, int offset, ValElemVec& init_vals) const; void Generate(InitsManager* im, std::vector& ivec, int offset, ValElemVec& init_vals) const; diff --git a/src/script_opt/CPP/RuntimeVec.cc b/src/script_opt/CPP/RuntimeVec.cc index fc5ccc6f1f..7c6e6ab717 100644 --- a/src/script_opt/CPP/RuntimeVec.cc +++ b/src/script_opt/CPP/RuntimeVec.cc @@ -3,6 +3,7 @@ #include "zeek/script_opt/CPP/RuntimeVec.h" #include "zeek/Overflow.h" +#include "zeek/RE.h" #include "zeek/ZeekString.h" namespace zeek::detail { @@ -320,6 +321,29 @@ static VectorValPtr str_vec_op_kernel__CPP(const VectorValPtr& v1, const VectorV return v_result; } +// Kernel for element-by-element pattern relationals. "is_eq" governs +// whether the operation is equality (true) or inequality (false). +static VectorValPtr pat_vec_op_kernel__CPP(const VectorValPtr& v1, const VectorValPtr& v2, bool is_eq) { + auto res_type = make_intrusive(base_type(TYPE_BOOL)); + auto v_result = make_intrusive(res_type); + auto n = v1->Size(); + + for ( unsigned int i = 0; i < n; ++i ) { + auto v1_i = v1->ValAt(i); + auto v2_i = v2->ValAt(i); + if ( ! v1_i || ! v2_i ) + continue; + + auto p1 = v1_i->AsPattern(); + auto p2 = v2_i->AsPattern(); + + bool elem_eq = strcmp(p1->PatternText(), p2->PatternText()) == 0; + v_result->Assign(i, val_mgr->Bool(elem_eq == is_eq)); + } + + return v_result; +} + VectorValPtr str_vec_op_lt__CPP(const VectorValPtr& v1, const VectorValPtr& v2) { return str_vec_op_kernel__CPP(v1, v2, -1, -1); } @@ -339,6 +363,13 @@ VectorValPtr str_vec_op_ge__CPP(const VectorValPtr& v1, const VectorValPtr& v2) return str_vec_op_kernel__CPP(v1, v2, 0, 1); } +VectorValPtr pat_vec_op_eq__CPP(const VectorValPtr& v1, const VectorValPtr& v2) { + return pat_vec_op_kernel__CPP(v1, v2, true); +} +VectorValPtr pat_vec_op_ne__CPP(const VectorValPtr& v1, const VectorValPtr& v2) { + return pat_vec_op_kernel__CPP(v1, v2, false); +} + VectorValPtr vector_select__CPP(const VectorValPtr& v1, VectorValPtr v2, VectorValPtr v3) { auto vt = v2->GetType(); auto v_result = make_intrusive(vt); diff --git a/src/script_opt/CPP/RuntimeVec.h b/src/script_opt/CPP/RuntimeVec.h index 14c852b9c8..1e83159f55 100644 --- a/src/script_opt/CPP/RuntimeVec.h +++ b/src/script_opt/CPP/RuntimeVec.h @@ -72,6 +72,10 @@ extern VectorValPtr str_vec_op_ne__CPP(const VectorValPtr& v1, const VectorValPt extern VectorValPtr str_vec_op_gt__CPP(const VectorValPtr& v1, const VectorValPtr& v2); extern VectorValPtr str_vec_op_ge__CPP(const VectorValPtr& v1, const VectorValPtr& v2); +// Pattern vector relationals. +extern VectorValPtr pat_vec_op_eq__CPP(const VectorValPtr& v1, const VectorValPtr& v2); +extern VectorValPtr pat_vec_op_ne__CPP(const VectorValPtr& v1, const VectorValPtr& v2); + // Support for vector conditional ('?:') expressions. Using the boolean // vector v1 as a selector, returns a new vector populated with the // elements selected out of v2 and v3. diff --git a/testing/btest/Baseline.cpp/bifs.is_event_handled/err b/testing/btest/Baseline.cpp/bifs.is_event_handled/err new file mode 100644 index 0000000000..0679d8c5a7 --- /dev/null +++ b/testing/btest/Baseline.cpp/bifs.is_event_handled/err @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/is_event_handled.zeek (C++), line 9: is_event_handled: 'myfunc1' is not an event (<___>testing_btest__tmp_bifs_is_event_handled_is_event_handled_zeek__global_stmts__zf: function() : void()) +error in <...>/is_event_handled.zeek (C++), line 9: is_event_handled: 'conn_id' is not an event (<___>testing_btest__tmp_bifs_is_event_handled_is_event_handled_zeek__global_stmts__zf: function() : void()) diff --git a/testing/btest/Baseline.cpp/bifs.is_event_handled/out b/testing/btest/Baseline.cpp/bifs.is_event_handled/out new file mode 100644 index 0000000000..311b22d902 --- /dev/null +++ b/testing/btest/Baseline.cpp/bifs.is_event_handled/out @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +T +F +F +F diff --git a/testing/btest/Baseline.cpp/bifs.lookup_connection/.stderr b/testing/btest/Baseline.cpp/bifs.lookup_connection/.stderr new file mode 100644 index 0000000000..483e41d576 --- /dev/null +++ b/testing/btest/Baseline.cpp/bifs.lookup_connection/.stderr @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1362692526.869344 error in <...>/lookup_connection.zeek (C++), line 6: invalid connection ID record encountered: the proto field has the "unknown" 65535 value. Did you forget to set it? (<___>testing_btest__tmp_bifs_lookup_connection_lookup_connection_zeek__new_connection__0__zf: function() : void()) +1362692526.869344 error in <...>/lookup_connection.zeek (C++), line 6: connection ID not a known connection (<___>testing_btest__tmp_bifs_lookup_connection_lookup_connection_zeek__new_connection__0__zf: function() : void() and [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=65535]) +1362692526.869344 error in <...>/lookup_connection.zeek (C++), line 29: invalid connection ID record encountered: the proto field has the "unknown" 65535 value. Did you forget to set it? (<___>testing_btest__tmp_bifs_lookup_connection_lookup_connection_zeek__new_connection__1__zf: function() : void()) +1362692526.869344 error in <...>/lookup_connection.zeek (C++), line 47: invalid connection ID record encountered (<___>testing_btest__tmp_bifs_lookup_connection_lookup_connection_zeek__new_connection__2__zf: function() : void()) +1362692526.869344 error in <...>/lookup_connection.zeek (C++), line 47: connection ID not a known connection (<___>testing_btest__tmp_bifs_lookup_connection_lookup_connection_zeek__new_connection__2__zf: function() : void() and [orig_h=, orig_p=, resp_h=, resp_p=, proto=65535]) +1362692526.869344 error in <...>/lookup_connection.zeek (C++), line 47: invalid connection ID record encountered (<___>testing_btest__tmp_bifs_lookup_connection_lookup_connection_zeek__new_connection__2__zf: function() : void()) diff --git a/testing/btest/Baseline.cpp/bifs.to_count/err b/testing/btest/Baseline.cpp/bifs.to_count/err new file mode 100644 index 0000000000..c9e9f52fb8 --- /dev/null +++ b/testing/btest/Baseline.cpp/bifs.to_count/err @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/to_count.zeek (C++), line 5: bad conversion to count (<___>testing_btest__tmp_bifs_to_count_to_count_zeek__zeek_init__36__zf: function() : void() and -2) +error in <...>/to_count.zeek (C++), line 5: bad conversion to count (<___>testing_btest__tmp_bifs_to_count_to_count_zeek__zeek_init__36__zf: function() : void() and ) +error in <...>/to_count.zeek (C++), line 5: bad conversion to count (<___>testing_btest__tmp_bifs_to_count_to_count_zeek__zeek_init__36__zf: function() : void() and not a count) diff --git a/testing/btest/Baseline.cpp/bifs.to_count/out b/testing/btest/Baseline.cpp/bifs.to_count/out new file mode 100644 index 0000000000..7404ddbeb0 --- /dev/null +++ b/testing/btest/Baseline.cpp/bifs.to_count/out @@ -0,0 +1,17 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0 +2 +3 +4 +7 +0 +18446744073709551611 +205 +206 +172 +35 +195 +195 +0 +123 +9223372036854775808 and 9223372036854775808 are the same diff --git a/testing/btest/Baseline.cpp/bifs.to_int/err b/testing/btest/Baseline.cpp/bifs.to_int/err new file mode 100644 index 0000000000..dc469f3d55 --- /dev/null +++ b/testing/btest/Baseline.cpp/bifs.to_int/err @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/to_int.zeek (C++), line 5: bad conversion to integer (<___>testing_btest__tmp_bifs_to_int_to_int_zeek__zeek_init__36__zf: function() : void() and not an int) diff --git a/testing/btest/Baseline.cpp/bifs.to_int/out b/testing/btest/Baseline.cpp/bifs.to_int/out new file mode 100644 index 0000000000..9527cdaf52 --- /dev/null +++ b/testing/btest/Baseline.cpp/bifs.to_int/out @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1 +-1 +188 +39 +243 +243 +4294967296 +0 +205 +206 +3 +4 +-3 +-4 diff --git a/testing/btest/Baseline.cpp/cluster.generic.cluster-publish-errors/.stderr b/testing/btest/Baseline.cpp/cluster.generic.cluster-publish-errors/.stderr new file mode 100644 index 0000000000..d8982afb3c --- /dev/null +++ b/testing/btest/Baseline.cpp/cluster.generic.cluster-publish-errors/.stderr @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/cluster-publish-errors.zeek (C++), line 55: Publish of unknown record type 'Cluster::MyEvent' (<___>testing_btest__tmp_cluster_generic_cluster_publish_errors_cluster_publish_errors_zeek__zeek_init__40__zf: function() : void()) +error in <...>/cluster-publish-errors.zeek (C++), line 62: Publish of unknown record type 'Cluster::MyEvent' (<___>testing_btest__tmp_cluster_generic_cluster_publish_errors_cluster_publish_errors_zeek__zeek_init__41__zf: function() : void()) +error in <...>/cluster-publish-errors.zeek (C++), line 69: Publish of unknown record type 'Cluster::MyEvent' (<___>testing_btest__tmp_cluster_generic_cluster_publish_errors_cluster_publish_errors_zeek__zeek_init__42__zf: function() : void()) diff --git a/testing/btest/Baseline.cpp/cluster.generic.cluster-publish-errors/.stdout b/testing/btest/Baseline.cpp/cluster.generic.cluster-publish-errors/.stdout new file mode 100644 index 0000000000..53ade358c6 --- /dev/null +++ b/testing/btest/Baseline.cpp/cluster.generic.cluster-publish-errors/.stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Broker::make_event with Cluster::publish() +r=, T +Broker::make_event with Cluster::publish_hrw() +r=, T +Broker::make_event with Cluster::publish_rr() +r=, T +Cluster::publish() with wrong event +r=, F +Cluster::publish_hrw() with wrong event +r=, F +Cluster::publish_rr() with wrong event +r=, F diff --git a/testing/btest/Baseline.cpp/language.assert-misc/out b/testing/btest/Baseline.cpp/language.assert-misc/out index efaee8ca3d..c732788c0f 100644 --- a/testing/btest/Baseline.cpp/language.assert-misc/out +++ b/testing/btest/Baseline.cpp/language.assert-misc/out @@ -1,7 +1,7 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. f, lambda_<10820400278317158366>: function() : void compiled-C++ -g, lambda_<9730512750166342063>: function() : void +g, lambda_<16208386833253569415>: function() : void compiled-C++ test_function, test_function: function() : void compiled-C++ diff --git a/testing/btest/Baseline.cpp/scripts.base.protocols.quic.analyzer-confirmations/conn.log.cut b/testing/btest/Baseline.cpp/scripts.base.protocols.quic.analyzer-confirmations/conn.log.cut new file mode 100644 index 0000000000..46d72b1541 --- /dev/null +++ b/testing/btest/Baseline.cpp/scripts.base.protocols.quic.analyzer-confirmations/conn.log.cut @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid history service +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 Dd quic,ssl diff --git a/testing/btest/Baseline.cpp/scripts.base.protocols.quic.analyzer-confirmations/out b/testing/btest/Baseline.cpp/scripts.base.protocols.quic.analyzer-confirmations/out new file mode 100644 index 0000000000..594e75d111 --- /dev/null +++ b/testing/btest/Baseline.cpp/scripts.base.protocols.quic.analyzer-confirmations/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +analyzer_confirmation, 1692198386.837988, CHhAvVGS1DHFjwGM9, AllAnalyzers::ANALYZER_ANALYZER_QUIC +analyzer_confirmation, 1692198386.837988, CHhAvVGS1DHFjwGM9, AllAnalyzers::ANALYZER_ANALYZER_SSL