diff --git a/CHANGES b/CHANGES index efb5234d66..e2f2f893f4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,29 @@ + +4.1.0-dev.114 | 2021-01-14 15:00:48 -0800 + + * Add support for rewriting the ASTs for script functions in "reduced" form + + In reduced ASTs, expressions have only variables or constants for operands + (no subexpressions - those are replaced by temporary variables). In + addition, some expressions are transformed into alternatives (new specialized + expressions, or in some cases new internal types of statements) to + more directly encapsulate their operation. + + The functionality is enabled by using -O xform. You can dump the + before-and-after ASTs using -O dump-xform (very large output!). The new + command-line option --optimize-only=X restricts the processing to only the + function named X, and also automatically turns on dump-xform. + + * Add new btest alternative for script transformation (xform) (Vern Paxson, Corelight) + + * split bifs.string_utils into a non-error test and an only-errors test (Vern Paxson, Corelight) + + To help control for differing error propagation + + * Add convenience function, obj_desc(), for accessing object descriptions (Vern Paxson, Corelight) + + * removed unused EventBodyList subclass (Vern Paxson, Corelight) + 4.1.0-dev.82 | 2021-01-14 13:27:23 -0700 * Use static_cast instead of dynamic_cast for Val conversions (Tim Wojtulewicz) diff --git a/VERSION b/VERSION index b827922118..99020c3e35 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.1.0-dev.82 +4.1.0-dev.114 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ca7736485f..05822ff342 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -325,8 +325,10 @@ set(MAIN_SRCS script_opt/Expr.cc script_opt/Inline.cc script_opt/ProfileFunc.cc + script_opt/Reduce.cc script_opt/ScriptOpt.cc script_opt/Stmt.cc + script_opt/TempVar.cc nb_dns.c digest.h diff --git a/src/Desc.cc b/src/Desc.cc index f23c1d5e00..dc8c39b848 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -427,4 +427,17 @@ bool ODesc::FindType(const Type* type) return false; } + +std::string obj_desc(const Obj* o) + { + static ODesc d; + + d.Clear(); + o->Describe(&d); + d.SP(); + o->GetLocationInfo()->Describe(&d); + + return std::string(d.Description()); + } + } // namespace zeek diff --git a/src/Desc.h b/src/Desc.h index 8711e2c4f9..54cf79e60d 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -210,6 +210,13 @@ protected: std::set encountered_types; }; +// Returns a string representation of an object's description. Used for +// debugging and error messages. takes a bare pointer rather than an +// IntrusivePtr because the latter is harder to deal with when making +// calls from a debugger like lldb, which is the main use of this function. +class Obj; +extern std::string obj_desc(const Obj* o); + } // namespace zeek using BroFile [[deprecated("Remove in v4.1. Use zeek::File.")]] = zeek::File; diff --git a/src/Expr.cc b/src/Expr.cc index f1431a1f25..7bbd4ab590 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -41,6 +41,10 @@ const char* expr_name(BroExprTag t) "coerce", "record_coerce", "table_coerce", "vector_coerce", "sizeof", "cast", "is", "[:]=", "inline()", + "[]=", "$=", + "vec+=", + "to_any_coerce", "from_any_coerce", + "any[]", "nop", }; @@ -95,12 +99,24 @@ NameExpr* Expr::AsNameExpr() return (NameExpr*) this; } +NameExprPtr Expr::AsNameExprPtr() + { + CHECK_TAG(tag, EXPR_NAME, "ExprVal::AsNameExpr", expr_name) + return {NewRef{}, (NameExpr*) this}; + } + const ConstExpr* Expr::AsConstExpr() const { CHECK_TAG(tag, EXPR_CONST, "ExprVal::AsConstExpr", expr_name) return (const ConstExpr*) this; } +ConstExprPtr Expr::AsConstExprPtr() + { + CHECK_TAG(tag, EXPR_CONST, "ExprVal::AsConstExpr", expr_name) + return {NewRef{}, (ConstExpr*) this}; + } + const CallExpr* Expr::AsCallExpr() const { CHECK_TAG(tag, EXPR_CALL, "ExprVal::AsCallExpr", expr_name) @@ -143,6 +159,12 @@ EventExprPtr Expr::AsEventExprPtr() return {NewRef{}, (EventExpr*) this}; } +RefExprPtr Expr::AsRefExprPtr() + { + CHECK_TAG(tag, EXPR_REF, "ExprVal::AsRefExpr", expr_name) + return {NewRef{}, (RefExpr*) this}; + } + bool Expr::CanAdd() const { return false; @@ -182,6 +204,133 @@ void Expr::Assign(Frame* /* f */, ValPtr /* v */) Internal("Expr::Assign called"); } +void Expr::AssignToIndex(ValPtr v1, ValPtr v2, ValPtr v3) const + { + bool iterators_invalidated; + + auto error_msg = assign_to_index(std::move(v1), std::move(v2), std::move(v3), + iterators_invalidated); + + if ( iterators_invalidated ) + { + ODesc d; + Describe(&d); + reporter->PushLocation(GetLocationInfo()); + reporter->Warning("possible loop/iterator invalidation caused by expression: %s", d.Description()); + reporter->PopLocation(); + } + + if ( error_msg ) + RuntimeErrorWithCallStack(error_msg); + } + +static int get_slice_index(int idx, int len) + { + if ( abs(idx) > len ) + idx = idx > 0 ? len : 0; // Clamp maximum positive/negative indices. + else if ( idx < 0 ) + idx += len; // Map to a positive index. + + return idx; + } + +const char* assign_to_index(ValPtr v1, ValPtr v2, ValPtr v3, + bool& iterators_invalidated) + { + iterators_invalidated = false; + + if ( ! v1 || ! v2 || ! v3 ) + return nullptr; + + // Hold an extra reference in case the ownership transfer + // to the table/vector goes wrong and we still want to obtain + // diagnostic info from the original value after the assignment + // already unref'd. + auto v_extra = v3; + + switch ( v1->GetType()->Tag() ) { + case TYPE_VECTOR: + { + const ListVal* lv = v2->AsListVal(); + VectorVal* v1_vect = v1->AsVectorVal(); + + if ( lv->Length() > 1 ) + { + auto len = v1_vect->Size(); + bro_int_t first = get_slice_index(lv->Idx(0)->CoerceToInt(), len); + bro_int_t last = get_slice_index(lv->Idx(1)->CoerceToInt(), len); + + // Remove the elements from the vector within the slice. + for ( auto idx = first; idx < last; idx++ ) + v1_vect->Remove(first); + + // Insert the new elements starting at the first + // position. + + VectorVal* v_vect = v3->AsVectorVal(); + + for ( auto idx = 0u; idx < v_vect->Size(); + idx++, first++ ) + v1_vect->Insert(first, v_vect->At(idx)); + } + + else if ( ! v1_vect->Assign(lv->Idx(0)->CoerceToUnsigned(), std::move(v3)) ) + { + v3 = std::move(v_extra); + + if ( v3 ) + { + ODesc d; + v3->Describe(&d); + const auto& vt = v3->GetType(); + auto vtt = vt->Tag(); + std::string tn = vtt == TYPE_RECORD ? + vt->GetName() : type_name(vtt); + return util::fmt("vector index assignment failed for invalid type '%s', value: %s", + tn.data(), d.Description()); + } + else + return "assignment failed with null value"; + } + break; + } + + case TYPE_TABLE: + { + if ( ! v1->AsTableVal()->Assign(std::move(v2), std::move(v3), true, &iterators_invalidated) ) + { + v3 = std::move(v_extra); + + if ( v3 ) + { + ODesc d; + v3->Describe(&d); + const auto& vt = v3->GetType(); + auto vtt = vt->Tag(); + std::string tn = vtt == TYPE_RECORD ? + vt->GetName() : type_name(vtt); + return util::fmt("table index assignment failed for invalid type '%s', value: %s", + tn.data(), d.Description()); + } + else + return "assignment failed with null value"; + } + + break; + } + + case TYPE_STRING: + return "assignment via string index accessor not allowed"; + break; + + default: + return "bad index expression type in assignment"; + break; + } + + return nullptr; + } + TypePtr Expr::InitType() const { return type; @@ -312,6 +461,12 @@ NameExpr::NameExpr(IDPtr arg_id, bool const_init) h->SetUsed(); } +// This isn't in-lined to avoid needing to pull in ID.h. +IDPtr NameExpr::IdPtr() + { + return id; + } + ValPtr NameExpr::Eval(Frame* f) const { ValPtr v; @@ -437,7 +592,14 @@ ValPtr UnaryExpr::Eval(Frame* f) const if ( ! v ) return nullptr; - if ( is_vector(v) && Tag() != EXPR_IS && Tag() != EXPR_CAST ) + if ( is_vector(v) && Tag() != EXPR_IS && Tag() != EXPR_CAST && + // The following allows passing vectors-by-reference to + // functions that use vector-of-any for generic vector + // manipulation ... + Tag() != EXPR_TO_ANY_COERCE && + // ... and the following to avoid vectorizing operations + // on vector-of-any's + Tag() != EXPR_FROM_ANY_COERCE ) { VectorVal* v_op = v->AsVectorVal(); VectorTypePtr out_t; @@ -458,9 +620,7 @@ ValPtr UnaryExpr::Eval(Frame* f) const return result; } else - { return Fold(v.get()); - } } bool UnaryExpr::IsPure() const @@ -2130,7 +2290,8 @@ void RefExpr::Assign(Frame* f, ValPtr v) AssignExpr::AssignExpr(ExprPtr arg_op1, ExprPtr arg_op2, bool arg_is_init, ValPtr arg_val, - const AttributesPtr& attrs) + const AttributesPtr& attrs, + bool typecheck) : BinaryExpr(EXPR_ASSIGN, arg_is_init ? std::move(arg_op1) : arg_op1->MakeLvalue(), std::move(arg_op2)) @@ -2153,9 +2314,10 @@ AssignExpr::AssignExpr(ExprPtr arg_op1, return; } - // We discard the status from TypeCheck since it has already - // generated error messages. - (void) TypeCheck(attrs); + if ( typecheck ) + // We discard the status from TypeCheck since it has already + // generated error messages. + (void) TypeCheck(attrs); val = std::move(arg_val); @@ -2755,16 +2917,6 @@ ValPtr IndexExpr::Eval(Frame* f) const return Fold(v1.get(), v2.get()); } -static int get_slice_index(int idx, int len) - { - if ( abs(idx) > len ) - idx = idx > 0 ? len : 0; // Clamp maximum positive/negative indices. - else if ( idx < 0 ) - idx += len; // Map to a positive index. - - return idx; - } - ValPtr IndexExpr::Fold(Val* v1, Val* v2) const { if ( IsError() ) @@ -2856,105 +3008,9 @@ void IndexExpr::Assign(Frame* f, ValPtr v) return; auto v1 = op1->Eval(f); - - if ( ! v1 ) - return; - auto v2 = op2->Eval(f); - if ( ! v1 || ! v2 ) - return; - - // Hold an extra reference to 'arg_v' in case the ownership transfer to - // the table/vector goes wrong and we still want to obtain diagnostic info - // from the original value after the assignment already unref'd. - auto v_extra = v; - - switch ( v1->GetType()->Tag() ) { - case TYPE_VECTOR: - { - const ListVal* lv = v2->AsListVal(); - VectorVal* v1_vect = v1->AsVectorVal(); - - if ( lv->Length() > 1 ) - { - auto len = v1_vect->Size(); - bro_int_t first = get_slice_index(lv->Idx(0)->CoerceToInt(), len); - bro_int_t last = get_slice_index(lv->Idx(1)->CoerceToInt(), len); - - // Remove the elements from the vector within the slice - for ( auto idx = first; idx < last; idx++ ) - v1_vect->Remove(first); - - // Insert the new elements starting at the first position - VectorVal* v_vect = v->AsVectorVal(); - - for ( auto idx = 0u; idx < v_vect->Size(); idx++, first++ ) - v1_vect->Insert(first, v_vect->At(idx)); - } - else if ( ! v1_vect->Assign(lv->Idx(0)->CoerceToUnsigned(), std::move(v)) ) - { - v = std::move(v_extra); - - if ( v ) - { - ODesc d; - v->Describe(&d); - const auto& vt = v->GetType(); - auto vtt = vt->Tag(); - std::string tn = vtt == TYPE_RECORD ? vt->GetName() : type_name(vtt); - RuntimeErrorWithCallStack(util::fmt( - "vector index assignment failed for invalid type '%s', value: %s", - tn.data(), d.Description())); - } - else - RuntimeErrorWithCallStack("assignment failed with null value"); - } - break; - } - - case TYPE_TABLE: - { - bool iterators_invalidated = false; - - if ( ! v1->AsTableVal()->Assign(std::move(v2), std::move(v), true, &iterators_invalidated) ) - { - v = std::move(v_extra); - - if ( v ) - { - ODesc d; - v->Describe(&d); - const auto& vt = v->GetType(); - auto vtt = vt->Tag(); - std::string tn = vtt == TYPE_RECORD ? vt->GetName() : type_name(vtt); - RuntimeErrorWithCallStack(util::fmt( - "table index assignment failed for invalid type '%s', value: %s", - tn.data(), d.Description())); - } - else - RuntimeErrorWithCallStack("assignment failed with null value"); - } - - if ( iterators_invalidated ) - { - ODesc d; - Describe(&d); - reporter->PushLocation(GetLocationInfo()); - reporter->Warning("possible loop/iterator invalidation caused by expression: %s", d.Description()); - reporter->PopLocation(); - } - } - break; - - case TYPE_STRING: - RuntimeErrorWithCallStack("assignment via string index accessor not allowed"); - break; - - default: - RuntimeErrorWithCallStack("bad index expression type in assignment"); - break; - } + AssignToIndex(v1, v2, v); } void IndexExpr::ExprDescribe(ODesc* d) const @@ -5222,7 +5278,7 @@ bool check_and_promote_args(ListExpr* const args, RecordType* types) if ( el.length() < ntypes ) { - ExprPList def_elements; + std::vector def_elements; // Start from rightmost parameter, work backward to fill in missing // arguments using &default expressions. @@ -5237,11 +5293,21 @@ bool check_and_promote_args(ListExpr* const args, RecordType* types) return false; } - def_elements.push_front(def_attr->GetExpr().get()); + // Don't use the default expression directly, as + // doing so will wind up sharing its code across + // different invocations that use the default + // argument. That works okay for the interpreter, + // but if we transform the code we want that done + // separately for each instance, rather than + // one instance inheriting the transformed version + // from another. + const auto& e = def_attr->GetExpr(); + def_elements.emplace_back(e->Duplicate()); } - for ( const auto& elem : def_elements ) - el.push_back(elem->Ref()); + auto ne = def_elements.size(); + while ( ne ) + el.push_back(def_elements[--ne].release()); } TypeList* tl = new TypeList(); diff --git a/src/Expr.h b/src/Expr.h index f294df6098..688fb97c32 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -68,6 +68,15 @@ enum BroExprTag : int { EXPR_IS, EXPR_INDEX_SLICE_ASSIGN, EXPR_INLINE, + + // The following types of expressions are only created for + // ASTs transformed to reduced form; they aren't germane for + // ASTs produced by parsing .zeek script files. + EXPR_INDEX_ASSIGN, EXPR_FIELD_LHS_ASSIGN, + EXPR_APPEND_TO, + EXPR_TO_ANY_COERCE, EXPR_FROM_ANY_COERCE, + EXPR_ANY_INDEX, + EXPR_NOP, #define NUM_EXPRS (int(EXPR_NOP) + 1) @@ -75,18 +84,27 @@ enum BroExprTag : int { extern const char* expr_name(BroExprTag t); -class ListExpr; -class NameExpr; -class ConstExpr; -class IndexExpr; class AssignExpr; class CallExpr; +class ConstExpr; class EventExpr; -class Stmt; +class FieldAssignExpr; +class FieldExpr; +class ForExpr; +class IndexExpr; +class ListExpr; +class NameExpr; +class RefExpr; class Expr; -using ExprPtr = IntrusivePtr; +using CallExprPtr = IntrusivePtr; +using ConstExprPtr = IntrusivePtr; using EventExprPtr = IntrusivePtr; +using ExprPtr = IntrusivePtr; +using NameExprPtr = IntrusivePtr; +using RefExprPtr = IntrusivePtr; + +class Stmt; using StmtPtr = IntrusivePtr; class Expr : public Obj { @@ -182,13 +200,17 @@ public: ctype* As ## ctype (); \ IntrusivePtr As ## ctype ## Ptr (); + ZEEK_EXPR_ACCESSOR_DECLS(AssignExpr) + ZEEK_EXPR_ACCESSOR_DECLS(CallExpr) + ZEEK_EXPR_ACCESSOR_DECLS(ConstExpr) + ZEEK_EXPR_ACCESSOR_DECLS(EventExpr) + ZEEK_EXPR_ACCESSOR_DECLS(FieldAssignExpr) + ZEEK_EXPR_ACCESSOR_DECLS(FieldExpr) + ZEEK_EXPR_ACCESSOR_DECLS(ForExpr) + ZEEK_EXPR_ACCESSOR_DECLS(IndexExpr) ZEEK_EXPR_ACCESSOR_DECLS(ListExpr) ZEEK_EXPR_ACCESSOR_DECLS(NameExpr) - ZEEK_EXPR_ACCESSOR_DECLS(ConstExpr) - ZEEK_EXPR_ACCESSOR_DECLS(CallExpr) - ZEEK_EXPR_ACCESSOR_DECLS(AssignExpr) - ZEEK_EXPR_ACCESSOR_DECLS(IndexExpr) - ZEEK_EXPR_ACCESSOR_DECLS(EventExpr) + ZEEK_EXPR_ACCESSOR_DECLS(RefExpr) void Describe(ODesc* d) const override final; @@ -200,6 +222,113 @@ public: // Recursively traverses the AST to inline eligible function calls. virtual ExprPtr Inline(Inliner* inl) { return ThisPtr(); } + // True if the expression can serve as an operand to a reduced + // expression. + bool IsSingleton(Reducer* r) const + { + return (tag == EXPR_NAME && IsReduced(r)) || tag == EXPR_CONST; + } + + // True if the expression has no side effects, false otherwise. + virtual bool HasNoSideEffects() const { return IsPure(); } + + // True if the expression is in fully reduced form: a singleton + // or an assignment to an operator with singleton operands. + virtual bool IsReduced(Reducer* c) const; + + // True if the expression's operands are singletons. + virtual bool HasReducedOps(Reducer* c) const; + + // True if (a) the expression has at least one operand, and (b) all + // of its operands are constant. + bool HasConstantOps() const + { + return GetOp1() && GetOp1()->IsConst() && + (! GetOp2() || + (GetOp2()->IsConst() && + (! GetOp3() || GetOp3()->IsConst()))); + } + + // True if the expression is reduced to a form that can be + // used in a conditional. + bool IsReducedConditional(Reducer* c) const; + + // True if the expression is reduced to a form that can be + // used in a field assignment. + bool IsReducedFieldAssignment(Reducer* c) const; + + // True if this expression can be the RHS for a field assignment. + bool IsFieldAssignable(const Expr* e) const; + + // True if the expression will transform to one of another type + // upon reduction, for non-constant operands. "Transform" means + // something beyond assignment to a temporary. Necessary so that + // we know to fully reduce such expressions if they're the RHS + // of an assignment. + virtual bool WillTransform(Reducer* c) const { return false; } + + // The same, but for the expression when used in a conditional context. + virtual bool WillTransformInConditional(Reducer* c) const + { return false; } + + // Returns the current expression transformed into "new_me". + ExprPtr TransformMe(ExprPtr new_me, Reducer* c, StmtPtr& red_stmt); + + // Returns a set of predecessor statements in red_stmt (which might + // be nil if no reduction necessary), and the reduced version of + // the expression, suitable for replacing previous uses. The + // second version always yields a singleton suitable for use + // as an operand. The first version does this too except + // for assignment statements; thus, its form is not guarantee + // suitable for use as an operand. + virtual ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt); + virtual ExprPtr ReduceToSingleton(Reducer* c, StmtPtr& red_stmt) + { return Reduce(c, red_stmt); } + + // Reduces the expression to one whose operands are singletons. + // Returns a predecessor statement (which might be a StmtList), if any. + virtual StmtPtr ReduceToSingletons(Reducer* c); + + // Reduces the expression to one that can appear as a conditional. + ExprPtr ReduceToConditional(Reducer* c, StmtPtr& red_stmt); + + // Reduces the expression to one that can appear as a field + // assignment. + ExprPtr ReduceToFieldAssignment(Reducer* c, StmtPtr& red_stmt); + + // Helper function for factoring out complexities related to + // index-based assignment. + void AssignToIndex(ValPtr v1, ValPtr v2, ValPtr v3) const; + + // Returns a new expression corresponding to a temporary + // that's been assigned to the given expression via red_stmt. + ExprPtr AssignToTemporary(ExprPtr e, Reducer* c, StmtPtr& red_stmt); + // Same but for this expression. + ExprPtr AssignToTemporary(Reducer* c, StmtPtr& red_stmt) + { return AssignToTemporary(ThisPtr(), c, red_stmt); } + + // If the expression always evaluates to the same value, returns + // that value. Otherwise, returns nullptr. + virtual ValPtr FoldVal() const { return nullptr; } + + // Returns a Val or a constant Expr corresponding to zero. + ValPtr MakeZero(TypeTag t) const; + ConstExprPtr MakeZeroExpr(TypeTag t) const; + + // Returns the expression's operands, or nil if it doesn't + // have the given operand. + virtual ExprPtr GetOp1() const; + virtual ExprPtr GetOp2() const; + virtual ExprPtr GetOp3() const; + + // Sets the operands to new values. + virtual void SetOp1(ExprPtr new_op); + virtual void SetOp2(ExprPtr new_op); + virtual void SetOp3(ExprPtr new_op); + + // Helper function to reduce boring code runs. + StmtPtr MergeStmts(StmtPtr s1, StmtPtr s2, StmtPtr s3 = nullptr) const; + // Access to the original expression from which this one is derived, // or this one if we don't have an original. Returns a bare pointer // rather than an ExprPtr to emphasize that the access is read-only. @@ -272,6 +401,7 @@ public: explicit NameExpr(IDPtr id, bool const_init = false); ID* Id() const { return id.get(); } + IDPtr IdPtr(); ValPtr Eval(Frame* f) const override; void Assign(Frame* f, ValPtr v) override; @@ -282,10 +412,20 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool HasNoSideEffects() const override { return true; } + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override { return IsReduced(c); } + bool WillTransform(Reducer* c) const override { return ! IsReduced(c); } + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + ValPtr FoldVal() const override; protected: void ExprDescribe(ODesc* d) const override; + // Returns true if our identifier is a global with a constant value + // that can be propagated; used for optimization. + bool FoldableGlobal() const; + IDPtr id; bool in_const_init; }; @@ -303,6 +443,7 @@ public: // Optimization-related: ExprPtr Duplicate() override; + ValPtr FoldVal() const override { return val; } protected: void ExprDescribe(ODesc* d) const override; @@ -325,6 +466,14 @@ public: // Optimization-related: ExprPtr Inline(Inliner* inl) override; + bool HasNoSideEffects() const override; + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + + ExprPtr GetOp1() const override final { return op; } + void SetOp1(ExprPtr _op) override final { op = std::move(_op); } + protected: UnaryExpr(BroExprTag arg_tag, ExprPtr arg_op); @@ -353,6 +502,17 @@ public: // Optimization-related: ExprPtr Inline(Inliner* inl) override; + bool HasNoSideEffects() const override; + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + + ExprPtr GetOp1() const override final { return op1; } + ExprPtr GetOp2() const override final { return op2; } + + void SetOp1(ExprPtr _op) override final { op1 = std::move(_op); } + void SetOp2(ExprPtr _op) override final { op2 = std::move(_op); } + protected: BinaryExpr(BroExprTag arg_tag, ExprPtr arg_op1, ExprPtr arg_op2) @@ -425,6 +585,12 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool HasNoSideEffects() const override; + bool WillTransform(Reducer* c) const override { return true; } + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override { return false; } + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + ExprPtr ReduceToSingleton(Reducer* c, StmtPtr& red_stmt) override; }; class ComplementExpr final : public UnaryExpr { @@ -433,6 +599,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; protected: ValPtr Fold(Val* v) const override; @@ -444,6 +612,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; protected: ValPtr Fold(Val* v) const override; @@ -455,6 +625,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; protected: ValPtr Fold(Val* v) const override; @@ -466,6 +638,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; protected: ValPtr Fold(Val* v) const override; @@ -490,6 +664,11 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + +protected: + ExprPtr BuildSub(const ExprPtr& op1, const ExprPtr& op2); }; class AddToExpr final : public BinaryExpr { @@ -499,6 +678,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override { return true; } + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; }; class RemoveFromExpr final : public BinaryExpr { @@ -508,6 +689,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override { return true; } + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; }; class SubExpr final : public BinaryExpr { @@ -516,6 +699,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; }; class TimesExpr final : public BinaryExpr { @@ -525,6 +710,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; }; class DivideExpr final : public BinaryExpr { @@ -533,6 +720,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; protected: ValPtr AddrFold(Val* v1, Val* v2) const override; @@ -555,6 +744,13 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override { return true; } + bool WillTransformInConditional(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + +protected: + bool IsTrue(const ExprPtr& e) const; + bool IsFalse(const ExprPtr& e) const; }; class BitExpr final : public BinaryExpr { @@ -563,6 +759,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; }; class EqExpr final : public BinaryExpr { @@ -572,6 +770,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; protected: ValPtr Fold(Val* v1, Val* v2) const override; @@ -584,6 +784,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; }; class CondExpr final : public Expr { @@ -603,6 +805,20 @@ public: ExprPtr Duplicate() override; ExprPtr Inline(Inliner* inl) override; + bool WillTransform(Reducer* c) const override; + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + StmtPtr ReduceToSingletons(Reducer* c) override; + + ExprPtr GetOp1() const override final { return op1; } + ExprPtr GetOp2() const override final { return op2; } + ExprPtr GetOp3() const override final { return op3; } + + void SetOp1(ExprPtr _op) override final { op1 = std::move(_op); } + void SetOp2(ExprPtr _op) override final { op2 = std::move(_op); } + void SetOp3(ExprPtr _op) override final { op3 = std::move(_op); } + protected: void ExprDescribe(ODesc* d) const override; @@ -620,6 +836,14 @@ public: // Optimization-related: ExprPtr Duplicate() override; + + bool WillTransform(Reducer* c) const override; + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + + // Reduce to simplifed LHS form, i.e., a reference to only a name. + StmtPtr ReduceToLHS(Reducer* c); }; class AssignExpr : public BinaryExpr { @@ -628,7 +852,8 @@ public: // yet still perform the assignment. Used for triggers. AssignExpr(ExprPtr op1, ExprPtr op2, bool is_init, ValPtr val = nullptr, - const AttributesPtr& attrs = nullptr); + const AttributesPtr& attrs = nullptr, + bool type_check = true); ValPtr Eval(Frame* f) const override; void EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f) const override; @@ -637,20 +862,29 @@ public: ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; bool IsPure() const override; - void SetOp2(ExprPtr e) - { - op2 = std::move(e); - } - // Optimization-related: ExprPtr Duplicate() override; + bool HasNoSideEffects() const override; + bool WillTransform(Reducer* c) const override { return true; } + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + ExprPtr ReduceToSingleton(Reducer* c, StmtPtr& red_stmt) override; + + // Whether this is an assignment to a temporary. + bool IsTemp() const { return is_temp; } + void SetIsTemp() { is_temp = true; } + protected: bool TypeCheck(const AttributesPtr& attrs = nullptr); bool TypeCheckArithmetics(TypeTag bt1, TypeTag bt2); bool is_init; ValPtr val; // optional + + // Optimization-related: + bool is_temp = false; }; class IndexSliceAssignExpr final : public AssignExpr { @@ -688,6 +922,9 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool HasReducedOps(Reducer* c) const override; + StmtPtr ReduceToSingletons(Reducer* c) override; + protected: ValPtr Fold(Val* v1, Val* v2) const override; @@ -797,6 +1034,10 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + StmtPtr ReduceToSingletons(Reducer* c) override; + protected: ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; @@ -823,6 +1064,10 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + StmtPtr ReduceToSingletons(Reducer* c) override; + protected: ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; @@ -849,6 +1094,10 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + StmtPtr ReduceToSingletons(Reducer* c) override; + protected: ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; @@ -867,6 +1116,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool HasReducedOps(Reducer* c) const override; + protected: ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; @@ -885,6 +1136,9 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override { return true; } + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + protected: void ExprDescribe(ODesc* d) const override; @@ -898,6 +1152,9 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + protected: ValPtr FoldSingleVal(Val* v, InternalTypeTag t) const; ValPtr Fold(Val* v) const override; @@ -974,6 +1231,16 @@ public: ExprPtr Duplicate() override; ExprPtr Inline(Inliner* inl) override; + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + + ExprPtr GetOp1() const override final; + ExprPtr GetOp2() const override final; + + void SetOp1(ExprPtr _op) override final; + void SetOp2(ExprPtr _op) override final; + protected: void ExprDescribe(ODesc* d) const override; @@ -988,6 +1255,8 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool HasReducedOps(Reducer* c) const override; + protected: ValPtr Fold(Val* v1, Val* v2) const override; @@ -1011,6 +1280,11 @@ public: ExprPtr Duplicate() override; ExprPtr Inline(Inliner* inl) override; + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + StmtPtr ReduceToSingletons(Reducer* c) override; + protected: void ExprDescribe(ODesc* d) const override; @@ -1038,6 +1312,8 @@ public: ExprPtr Duplicate() override; ExprPtr Inline(Inliner* inl) override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + protected: void ExprDescribe(ODesc* d) const override; @@ -1052,30 +1328,8 @@ private: std::string my_name; }; -class EventExpr final : public Expr { -public: - EventExpr(const char* name, ListExprPtr args); - - const char* Name() const { return name.c_str(); } - ListExpr* Args() const { return args.get(); } - EventHandlerPtr Handler() const { return handler; } - - ValPtr Eval(Frame* f) const override; - - TraversalCode Traverse(TraversalCallback* cb) const override; - - // Optimization-related: - ExprPtr Duplicate() override; - ExprPtr Inline(Inliner* inl) override; - -protected: - void ExprDescribe(ODesc* d) const override; - - std::string name; - EventHandlerPtr handler; - ListExprPtr args; -}; - +// This comes before EventExpr so that EventExpr::GetOp1 can return its +// arguments as convertible to ExprPtr. class ListExpr : public Expr { public: ListExpr(); @@ -1103,6 +1357,11 @@ public: ExprPtr Duplicate() override; ExprPtr Inline(Inliner* inl) override; + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + StmtPtr ReduceToSingletons(Reducer* c) override; + protected: ValPtr AddSetInit(const zeek::Type* t, ValPtr aggr) const; @@ -1111,6 +1370,38 @@ protected: ExprPList exprs; }; +class EventExpr final : public Expr { +public: + EventExpr(const char* name, ListExprPtr args); + + const char* Name() const { return name.c_str(); } + ListExpr* Args() const { return args.get(); } + EventHandlerPtr Handler() const { return handler; } + + ValPtr Eval(Frame* f) const override; + + TraversalCode Traverse(TraversalCallback* cb) const override; + + // Optimization-related: + ExprPtr Duplicate() override; + ExprPtr Inline(Inliner* inl) override; + + bool IsReduced(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + StmtPtr ReduceToSingletons(Reducer* c) override; + + ExprPtr GetOp1() const override final { return args; } + void SetOp1(ExprPtr _op) override final + { args = {NewRef{}, _op->AsListExpr()}; } + +protected: + void ExprDescribe(ODesc* d) const override; + + std::string name; + EventHandlerPtr handler; + ListExprPtr args; +}; + class RecordAssignExpr final : public ListExpr { public: @@ -1159,6 +1450,9 @@ public: ExprPtr Duplicate() override; + bool IsReduced(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + TraversalCode Traverse(TraversalCallback* cb) const override; protected: @@ -1170,6 +1464,134 @@ protected: StmtPtr body; }; +// A companion to AddToExpr that's for vector-append, instantiated during +// the reduction process. +class AppendToExpr : public BinaryExpr { +public: + AppendToExpr(ExprPtr op1, ExprPtr op2); + ValPtr Eval(Frame* f) const override; + + bool IsReduced(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + + ExprPtr Duplicate() override; +}; + +// An internal class for reduced form. +class IndexAssignExpr : public BinaryExpr { +public: + // "op1[op2] = op3", all reduced. + IndexAssignExpr(ExprPtr op1, ExprPtr op2, ExprPtr op3); + + ValPtr Eval(Frame* f) const override; + + ExprPtr Duplicate() override; + + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + ExprPtr ReduceToSingleton(Reducer* c, StmtPtr& red_stmt) override; + + ExprPtr GetOp3() const override final { return op3; } + void SetOp3(ExprPtr _op) override final { op3 = std::move(_op); } + + TraversalCode Traverse(TraversalCallback* cb) const override; + +protected: + void ExprDescribe(ODesc* d) const override; + + ExprPtr op3; // assignment RHS +}; + +// An internal class for reduced form. +class FieldLHSAssignExpr : public BinaryExpr { +public: + // "op1$field = RHS", where RHS is reduced with respect to + // ReduceToFieldAssignment(). + FieldLHSAssignExpr(ExprPtr op1, ExprPtr op2, const char* field_name, + int field); + + const char* FieldName() const { return field_name; } + int Field() const { return field; } + + ValPtr Eval(Frame* f) const override; + + ExprPtr Duplicate() override; + + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + ExprPtr ReduceToSingleton(Reducer* c, StmtPtr& red_stmt) override; + +protected: + void ExprDescribe(ODesc* d) const override; + + const char* field_name; + int field; +}; + +// Expression to explicitly capture conversion to an "any" type, rather +// than it occurring implicitly during script interpretation. +class CoerceToAnyExpr : public UnaryExpr { +public: + CoerceToAnyExpr(ExprPtr op); + +protected: + ValPtr Fold(Val* v) const override; + + ExprPtr Duplicate() override; +}; + +// Same, but for conversion from an "any" type. +class CoerceFromAnyExpr : public UnaryExpr { +public: + CoerceFromAnyExpr(ExprPtr op, TypePtr to_type); + +protected: + ValPtr Fold(Val* v) const override; + + ExprPtr Duplicate() override; +}; + +// Expression used to explicitly capture [a, b, c, ...] = x assignments. +class AnyIndexExpr : public UnaryExpr { +public: + AnyIndexExpr(ExprPtr op, int index); + + int Index() const { return index; } + +protected: + ValPtr Fold(Val* v) const override; + + void ExprDescribe(ODesc* d) const override; + + ExprPtr Duplicate() override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + + int index; +}; + +// Used internally for optimization, when a placeholder is needed. +class NopExpr : public Expr { +public: + explicit NopExpr() : Expr(EXPR_NOP) { } + + ValPtr Eval(Frame* f) const override; + + ExprPtr Duplicate() override; + + TraversalCode Traverse(TraversalCallback* cb) const override; + +protected: + void ExprDescribe(ODesc* d) const override; +}; + + +// Assigns v1[v2] = v3. Returns an error message, or nullptr on success. +// Factored out so that compiled code can call it as well as the interpreter. +extern const char* assign_to_index(ValPtr v1, ValPtr v2, ValPtr v3, + bool& iterators_invalidated); + inline Val* Expr::ExprVal() const { diff --git a/src/Func.cc b/src/Func.cc index ed90e3b95a..4610e47f08 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -556,6 +556,21 @@ void ScriptFunc::AddBody(StmtPtr new_body, sort(bodies.begin(), bodies.end()); } +void ScriptFunc::ReplaceBody(const StmtPtr& old_body, StmtPtr new_body) + { + bool found_it = false; + + for ( auto& body : bodies ) + if ( body.stmts.get() == old_body.get() ) + { + body.stmts = new_body; + found_it = true; + } + + ASSERT(found_it); + current_body = new_body; + } + void ScriptFunc::AddClosure(IDPList ids, Frame* f) { if ( ! f ) diff --git a/src/Func.h b/src/Func.h index a1cf89ec95..f73cd091d5 100644 --- a/src/Func.h +++ b/src/Func.h @@ -235,6 +235,11 @@ public: const std::vector& new_inits, size_t new_frame_size, int priority) override; + // Replace the given current instance of a function body with + // a new one. + void ReplaceBody(const detail::StmtPtr& old_body, + detail::StmtPtr new_body); + StmtPtr CurrentBody() const { return current_body; } /** diff --git a/src/Options.cc b/src/Options.cc index b3ce952ac5..6ab5b5bb0c 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -106,6 +106,7 @@ void usage(const char* prog, int code) fprintf(stderr, " -I|--print-id | print out given ID\n"); fprintf(stderr, " -N|--print-plugins | print available plugins and exit (-NN for verbose)\n"); fprintf(stderr, " -O|--optimize[=