From 18be4ba91b4bd87fffd1501d724bc526be2cf51f Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 15 Jun 2023 11:08:14 -0700 Subject: [PATCH] -O gen-C++ refinements for BiF failures, negative vector indices, boolean vector operations --- src/script_opt/CPP/Compile.h | 2 +- src/script_opt/CPP/Exprs.cc | 21 +++++- src/script_opt/CPP/RuntimeOps.cc | 4 ++ src/script_opt/CPP/RuntimeOps.h | 18 +++++- src/script_opt/CPP/RuntimeVec.cc | 64 ++++++++++++------- src/script_opt/CPP/maint/README | 4 +- .../.stderr | 5 ++ .../btest/broker/remote_event_auto_ts.zeek | 3 + testing/btest/broker/remote_event_ts.zeek | 3 + 9 files changed, 94 insertions(+), 30 deletions(-) create mode 100644 testing/btest/Baseline.cpp/scripts.base.frameworks.cluster.publish-hrw-type-check/.stderr diff --git a/src/script_opt/CPP/Compile.h b/src/script_opt/CPP/Compile.h index 6b110a00de..d0809fbb6a 100644 --- a/src/script_opt/CPP/Compile.h +++ b/src/script_opt/CPP/Compile.h @@ -776,7 +776,7 @@ private: std::string GenConstExpr(const ConstExpr* c, GenType gt); std::string GenIncrExpr(const Expr* e, GenType gt, bool is_incr, bool top_level); std::string GenCondExpr(const Expr* e, GenType gt); - std::string GenCallExpr(const CallExpr* c, GenType gt); + std::string GenCallExpr(const CallExpr* c, GenType gt, bool top_level); std::string GenInExpr(const Expr* e, GenType gt); std::string GenFieldExpr(const FieldExpr* fe, GenType gt); std::string GenHasFieldExpr(const HasFieldExpr* hfe, GenType gt); diff --git a/src/script_opt/CPP/Exprs.cc b/src/script_opt/CPP/Exprs.cc index b0ec6cd8e4..c9cde5ab0c 100644 --- a/src/script_opt/CPP/Exprs.cc +++ b/src/script_opt/CPP/Exprs.cc @@ -114,7 +114,7 @@ string CPPCompile::GenExpr(const Expr* e, GenType gt, bool top_level) case EXPR_COND: return GenCondExpr(e, gt); case EXPR_CALL: - return GenCallExpr(e->AsCallExpr(), gt); + return GenCallExpr(e->AsCallExpr(), gt, top_level); case EXPR_LIST: return GenListExpr(e, gt, false); case EXPR_IN: @@ -291,7 +291,7 @@ string CPPCompile::GenCondExpr(const Expr* e, GenType gt) return string("(") + gen1 + ") ? (" + gen2 + ") : (" + gen3 + ")"; } -string CPPCompile::GenCallExpr(const CallExpr* c, GenType gt) +string CPPCompile::GenCallExpr(const CallExpr* c, GenType gt, bool top_level) { const auto& t = c->GetType(); auto f = c->Func(); @@ -347,7 +347,18 @@ string CPPCompile::GenCallExpr(const CallExpr* c, GenType gt) // Indirect call. gen = string("(") + gen + ")->AsFunc()"; - string invoke_func = is_async ? "when_invoke__CPP" : "invoke__CPP"; + string invoke_func; + + if ( is_async ) + invoke_func = "when_invoke__CPP"; + else if ( t->Tag() == TYPE_VOID ) + { + ASSERT(top_level); + invoke_func = "invoke_void__CPP"; + } + else + invoke_func = "invoke__CPP"; + auto args_list = string(", {") + GenExpr(args_l, GEN_VAL_PTR) + "}"; auto invoker = invoke_func + "(" + gen + args_list + ", f__CPP"; @@ -356,6 +367,10 @@ string CPPCompile::GenCallExpr(const CallExpr* c, GenType gt) invoker += ")"; + if ( top_level ) + // No need to use accessor. + return invoker; + if ( IsNativeType(t) && gt != GEN_VAL_PTR ) return invoker + NativeAccessor(t); diff --git a/src/script_opt/CPP/RuntimeOps.cc b/src/script_opt/CPP/RuntimeOps.cc index 61732c6f21..b74ac496e7 100644 --- a/src/script_opt/CPP/RuntimeOps.cc +++ b/src/script_opt/CPP/RuntimeOps.cc @@ -51,9 +51,13 @@ ValPtr index_table__CPP(const TableValPtr& t, vector indices) ValPtr index_vec__CPP(const VectorValPtr& vec, int index) { + if ( index < 0 ) + index += vec->Size(); + auto v = vec->ValAt(index); if ( ! v ) reporter->CPPRuntimeError("no such index"); + return v; } diff --git a/src/script_opt/CPP/RuntimeOps.h b/src/script_opt/CPP/RuntimeOps.h index 4318e7c7ff..8ed8622941 100644 --- a/src/script_opt/CPP/RuntimeOps.h +++ b/src/script_opt/CPP/RuntimeOps.h @@ -48,12 +48,28 @@ extern ValPtr when_index_vec__CPP(const VectorValPtr& vec, int index); // custom one for those occurring inside a "when" clause. extern ValPtr when_index_slice__CPP(VectorVal* vec, const ListVal* lv); +// Calls out to the given script or BiF function, which does not return +// a value. +inline ValPtr invoke_void__CPP(Func* f, std::vector args, Frame* frame) + { + return f->Invoke(&args, frame); + } + +// Used for error propagation by failed calls. +class CPPInterpreterException : public InterpreterException + { + }; + // Calls out to the given script or BiF function. A separate function because // of the need to (1) construct the "args" vector using {} initializers, // but (2) needing to have the address of that vector. inline ValPtr invoke__CPP(Func* f, std::vector args, Frame* frame) { - return f->Invoke(&args, frame); + auto v = f->Invoke(&args, frame); + if ( ! v ) + throw CPPInterpreterException(); + + return v; } // The same, but raises an interpreter exception if the function does diff --git a/src/script_opt/CPP/RuntimeVec.cc b/src/script_opt/CPP/RuntimeVec.cc index bc0be1182e..6208a6f7e0 100644 --- a/src/script_opt/CPP/RuntimeVec.cc +++ b/src/script_opt/CPP/RuntimeVec.cc @@ -26,12 +26,17 @@ static bool check_vec_sizes__CPP(const VectorValPtr& v1, const VectorValPtr& v2) // (for example, adding one vector of "interval" to another), which // we want to do using the low-level representations. We'll later // convert the vector to the high-level representation if needed. -static VectorTypePtr base_vector_type__CPP(const VectorTypePtr& vt) +// +// One exception: for booleans ("is_bool" is true), we use those directly. +static VectorTypePtr base_vector_type__CPP(const VectorTypePtr& vt, bool is_bool = false) { switch ( vt->Yield()->InternalType() ) { case TYPE_INTERNAL_INT: - return make_intrusive(base_type(TYPE_INT)); + { + auto base_tag = is_bool ? TYPE_BOOL : TYPE_INT; + return make_intrusive(base_type(base_tag)); + } case TYPE_INTERNAL_UNSIGNED: return make_intrusive(base_type(TYPE_COUNT)); @@ -119,36 +124,27 @@ VEC_OP1(comp, ~, ) } // Analogous to VEC_OP1, instantiates a function for a given binary operation, -// which might-or-might-not be supported for low-level "double" types. +// with customimzable kernels for "int" and "double" operations. // This version is for operations whose result type is the same as the // operand type. -#define VEC_OP2(name, op, double_kernel, zero_check) \ +#define VEC_OP2(name, op, int_kernel, double_kernel, zero_check, is_bool) \ VectorValPtr vec_op_##name##__CPP(const VectorValPtr& v1, const VectorValPtr& v2) \ { \ if ( ! check_vec_sizes__CPP(v1, v2) ) \ return nullptr; \ \ - auto vt = base_vector_type__CPP(v1->GetType()); \ + auto vt = base_vector_type__CPP(v1->GetType(), is_bool); \ auto v_result = make_intrusive(vt); \ \ switch ( vt->Yield()->InternalType() ) \ { \ - case TYPE_INTERNAL_INT: \ - { \ - if ( vt->Yield()->Tag() == TYPE_BOOL ) \ - VEC_OP2_KERNEL(AsBool, BoolVal, op, zero_check) \ - else \ - VEC_OP2_KERNEL(AsInt, IntVal, op, zero_check) \ - break; \ - } \ - \ case TYPE_INTERNAL_UNSIGNED: \ { \ VEC_OP2_KERNEL(AsCount, CountVal, op, zero_check) \ break; \ } \ \ - double_kernel \ + int_kernel double_kernel \ \ default : break; \ } \ @@ -156,9 +152,29 @@ VEC_OP1(comp, ~, ) return v_result; \ } +// Instantiates a regular int_kernel for a binary operation. +#define VEC_OP2_WITH_INT(name, op, double_kernel, zero_check) \ + VEC_OP2( \ + name, op, case TYPE_INTERNAL_INT \ + : { \ + VEC_OP2_KERNEL(AsInt, IntVal, op, zero_check) \ + break; \ + }, \ + double_kernel, zero_check, false) + +// Instantiates an int_kernel for boolean operations. +#define VEC_OP2_WITH_BOOL(name, op, zero_check) \ + VEC_OP2( \ + name, op, case TYPE_INTERNAL_INT \ + : { \ + VEC_OP2_KERNEL(AsBool, BoolVal, op, zero_check) \ + break; \ + }, \ + , zero_check, true) + // Instantiates a double_kernel for a binary operation. #define VEC_OP2_WITH_DOUBLE(name, op, zero_check) \ - VEC_OP2( \ + VEC_OP2_WITH_INT( \ name, op, case TYPE_INTERNAL_DOUBLE \ : { \ VEC_OP2_KERNEL(AsDouble, DoubleVal, op, zero_check) \ @@ -171,14 +187,14 @@ VEC_OP2_WITH_DOUBLE(add, +, 0) VEC_OP2_WITH_DOUBLE(sub, -, 0) VEC_OP2_WITH_DOUBLE(mul, *, 0) VEC_OP2_WITH_DOUBLE(div, /, 1) -VEC_OP2(mod, %, , 1) -VEC_OP2(and, &, , 0) -VEC_OP2(or, |, , 0) -VEC_OP2(xor, ^, , 0) -VEC_OP2(andand, &&, , 0) -VEC_OP2(oror, ||, , 0) -VEC_OP2(lshift, <<, , 0) -VEC_OP2(rshift, >>, , 0) +VEC_OP2_WITH_INT(mod, %, , 1) +VEC_OP2_WITH_INT(and, &, , 0) +VEC_OP2_WITH_INT(or, |, , 0) +VEC_OP2_WITH_INT(xor, ^, , 0) +VEC_OP2_WITH_BOOL(andand, &&, 0) +VEC_OP2_WITH_BOOL(oror, ||, 0) +VEC_OP2_WITH_INT(lshift, <<, , 0) +VEC_OP2_WITH_INT(rshift, >>, , 0) // A version of VEC_OP2 that instead supports relational operations, so // the result type is always vector-of-bool. diff --git a/src/script_opt/CPP/maint/README b/src/script_opt/CPP/maint/README index 072b9dd222..d48ba575b6 100644 --- a/src/script_opt/CPP/maint/README +++ b/src/script_opt/CPP/maint/README @@ -34,7 +34,9 @@ The maintenance workflow: 5. Run "check-CPP-gen.sh" for each Zeek file that passed "check-zeek.sh". This will generate a corresponding file in CPP-test/out* indicating whether "-O gen-C++" can successfully run on the input. Presently, it should - be able to do so for all of them. + be able to do so for all of them except a few that have conditional code, + which I've left active (no @TEST-REQUIRES to prune) given hopes of + soon being able to support (most) conditional code for C++ compilation. This step is parallelizable, say using xargs -P 10 -n 1. diff --git a/testing/btest/Baseline.cpp/scripts.base.frameworks.cluster.publish-hrw-type-check/.stderr b/testing/btest/Baseline.cpp/scripts.base.frameworks.cluster.publish-hrw-type-check/.stderr new file mode 100644 index 0000000000..928f4eecec --- /dev/null +++ b/testing/btest/Baseline.cpp/scripts.base.frameworks.cluster.publish-hrw-type-check/.stderr @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/publish-hrw-type-check.zeek (C++), line 13: expected type Cluster::Pool for pool (<___>testing_btest__tmp_scripts_base_frameworks_cluster_publish_hrw_type_check_publish_hrw_type_check_zeek__zeek_init__34__zf()) +error in <...>/publish-hrw-type-check.zeek (C++), line 13: expected type Cluster::Pool for pool (<___>testing_btest__tmp_scripts_base_frameworks_cluster_publish_hrw_type_check_publish_hrw_type_check_zeek__zeek_init__34__zf()) +error in <...>/publish-hrw-type-check.zeek (C++), line 13: expected type Cluster::Pool for pool (<___>testing_btest__tmp_scripts_base_frameworks_cluster_publish_hrw_type_check_publish_hrw_type_check_zeek__zeek_init__34__zf()) +error in <...>/publish-hrw-type-check.zeek (C++), line 13: expected type string for key, got port (<___>testing_btest__tmp_scripts_base_frameworks_cluster_publish_hrw_type_check_publish_hrw_type_check_zeek__zeek_init__34__zf()) diff --git a/testing/btest/broker/remote_event_auto_ts.zeek b/testing/btest/broker/remote_event_auto_ts.zeek index 31bc2da59f..fe2b24f870 100644 --- a/testing/btest/broker/remote_event_auto_ts.zeek +++ b/testing/btest/broker/remote_event_auto_ts.zeek @@ -1,3 +1,6 @@ +# Not compatible with -O C++ testing since includes two distinct scripts. +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" +# # @TEST-GROUP: broker # # @TEST-PORT: BROKER_PORT diff --git a/testing/btest/broker/remote_event_ts.zeek b/testing/btest/broker/remote_event_ts.zeek index e4f2f508d1..f822fde332 100644 --- a/testing/btest/broker/remote_event_ts.zeek +++ b/testing/btest/broker/remote_event_ts.zeek @@ -1,3 +1,6 @@ +# Not compatible with -O C++ testing since includes two distinct scripts. +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" +# # @TEST-GROUP: broker # # @TEST-PORT: BROKER_PORT