diff --git a/src/script_opt/CPP/Compile.h b/src/script_opt/CPP/Compile.h index 5d818c1756..6b5cb20cf2 100644 --- a/src/script_opt/CPP/Compile.h +++ b/src/script_opt/CPP/Compile.h @@ -135,7 +135,8 @@ class CPPCompile { public: CPPCompile(std::vector& _funcs, ProfileFuncs& pfs, const std::string& gen_name, const std::string& addl_name, - CPPHashManager& _hm, bool _update, bool _standalone); + CPPHashManager& _hm, bool _update, bool _standalone, + bool report_uncompilable); ~CPPCompile(); private: @@ -145,7 +146,7 @@ private: // // Main driver, invoked by constructor. - void Compile(); + void Compile(bool report_uncompilable); // Generate the beginning of the compiled code: run-time functions, // namespace, auxiliary globals. @@ -161,8 +162,11 @@ private: void GenEpilog(); // True if the given function (plus body and profile) is one - // that should be compiled. - bool IsCompilable(const FuncInfo& func); + // that should be compiled. If non-nil, sets reason to the + // the reason why, if there's a fundamental problem. If however + // the function should be skipped for other reasons, then sets + // it to nil. + bool IsCompilable(const FuncInfo& func, const char** reason = nullptr); // The set of functions/bodies we're compiling. std::vector& funcs; diff --git a/src/script_opt/CPP/Driver.cc b/src/script_opt/CPP/Driver.cc index 7d8d016922..d9d0081300 100644 --- a/src/script_opt/CPP/Driver.cc +++ b/src/script_opt/CPP/Driver.cc @@ -14,7 +14,8 @@ using namespace std; CPPCompile::CPPCompile(vector& _funcs, ProfileFuncs& _pfs, const string& gen_name, const string& _addl_name, - CPPHashManager& _hm, bool _update, bool _standalone) + CPPHashManager& _hm, bool _update, bool _standalone, + bool report_uncompilable) : funcs(_funcs), pfs(_pfs), hm(_hm), update(_update), standalone(_standalone) { @@ -67,7 +68,7 @@ CPPCompile::CPPCompile(vector& _funcs, ProfileFuncs& _pfs, fclose(addl_f); } - Compile(); + Compile(report_uncompilable); } CPPCompile::~CPPCompile() @@ -75,7 +76,7 @@ CPPCompile::~CPPCompile() fclose(write_file); } -void CPPCompile::Compile() +void CPPCompile::Compile(bool report_uncompilable) { // Get the working directory so we can use it in diagnostic messages // as a way to identify this compilation. Only germane when doing @@ -100,8 +101,13 @@ void CPPCompile::Compile() // Can't be called directly. continue; - if ( IsCompilable(func) ) + const char* reason; + if ( IsCompilable(func, &reason) ) compilable_funcs.insert(BodyName(func)); + else if ( reason && report_uncompilable ) + fprintf(stderr, + "%s cannot be compiled to C++ due to %s\n", + func.Func()->Name(), reason); auto h = func.Profile()->HashVal(); if ( hm.HasHash(h) ) @@ -341,17 +347,24 @@ void CPPCompile::GenEpilog() Emit("} // zeek::detail"); } -bool CPPCompile::IsCompilable(const FuncInfo& func) +bool CPPCompile::IsCompilable(const FuncInfo& func, const char** reason) { + if ( ! is_CPP_compilable(func.Profile(), reason) ) + return false; + + if ( reason ) + // Indicate that there's no fundamental reason it can't be + // compiled. + *reason = nullptr; + if ( func.ShouldSkip() ) - // Caller marked this function as one to skip. return false; if ( hm.HasHash(func.Profile()->HashVal()) ) // We've already compiled it. return false; - return is_CPP_compilable(func.Profile()); + return true; } } // zeek::detail diff --git a/src/script_opt/CPP/Util.cc b/src/script_opt/CPP/Util.cc index 4c86ccbcf9..30adbb2578 100644 --- a/src/script_opt/CPP/Util.cc +++ b/src/script_opt/CPP/Util.cc @@ -33,13 +33,21 @@ string scope_prefix(int scope) return scope_prefix(to_string(scope)); } -bool is_CPP_compilable(const ProfileFunc* pf) +bool is_CPP_compilable(const ProfileFunc* pf, const char** reason) { if ( pf->NumWhenStmts() > 0 ) + { + if ( reason ) + *reason = "use of \"when\""; return false; + } if ( pf->TypeSwitches().size() > 0 ) + { + if ( reason ) + *reason = "use of type-based \"switch\""; return false; + } return true; } diff --git a/src/script_opt/CPP/Util.h b/src/script_opt/CPP/Util.h index 25b7a0abc6..a44fe8cfd8 100644 --- a/src/script_opt/CPP/Util.h +++ b/src/script_opt/CPP/Util.h @@ -19,8 +19,11 @@ extern std::string scope_prefix(const std::string& scope); // Same, but for scopes identified with numbers. extern std::string scope_prefix(int scope); -// True if the given function is compilable to C++. -extern bool is_CPP_compilable(const ProfileFunc* pf); +// True if the given function is compilable to C++. If it isn't, and +// the second argument is non-nil, then on return it points to text +// explaining why not. +extern bool is_CPP_compilable(const ProfileFunc* pf, + const char** reason = nullptr); // Helper utilities for file locking, to ensure that hash files // don't receive conflicting writes due to concurrent compilations. diff --git a/src/script_opt/GenRDs.cc b/src/script_opt/GenRDs.cc index 0ab24eea19..8c59ce4a4d 100644 --- a/src/script_opt/GenRDs.cc +++ b/src/script_opt/GenRDs.cc @@ -10,6 +10,13 @@ namespace zeek::detail { +RD_Decorate::RD_Decorate(std::shared_ptr _pf, const Func* f, + ScopePtr scope, StmtPtr body) +: pf(std::move(_pf)) + { + TraverseFunction(f, scope, body); + } + void RD_Decorate::TraverseFunction(const Func* f, ScopePtr scope, StmtPtr body) { func_flavor = f->Flavor(); diff --git a/src/script_opt/GenRDs.h b/src/script_opt/GenRDs.h index cfdfc5e34c..73e1ca4149 100644 --- a/src/script_opt/GenRDs.h +++ b/src/script_opt/GenRDs.h @@ -50,9 +50,12 @@ private: class RD_Decorate : public TraversalCallback { public: - RD_Decorate(std::shared_ptr _pf) : pf(std::move(_pf)) - { } + RD_Decorate(std::shared_ptr _pf, const Func* f, + ScopePtr scope, StmtPtr body); + const DefSetsMgr* GetDefSetsMgr() const { return &mgr; } + +private: // Traverses the given function body, using the first two // arguments for context. void TraverseFunction(const Func* f, ScopePtr scope, StmtPtr body); @@ -62,9 +65,6 @@ public: TraversalCode PreExpr(const Expr*) override; TraversalCode PostExpr(const Expr*) override; - const DefSetsMgr* GetDefSetsMgr() const { return &mgr; } - -private: // The following implement various types of "confluence", i.e., // situations in which control flow merges from multiple possible // paths to a given point. diff --git a/src/script_opt/ProfileFunc.cc b/src/script_opt/ProfileFunc.cc index 1d18037834..a1b9399146 100644 --- a/src/script_opt/ProfileFunc.cc +++ b/src/script_opt/ProfileFunc.cc @@ -459,7 +459,7 @@ ProfileFuncs::ProfileFuncs(std::vector& funcs, auto pf = std::make_unique(f.Func(), f.Body(), full_record_hashes); - if ( ! pred || (*pred)(pf.get()) ) + if ( ! pred || (*pred)(pf.get(), nullptr) ) MergeInProfile(pf.get()); else f.SetSkip(true); diff --git a/src/script_opt/ProfileFunc.h b/src/script_opt/ProfileFunc.h index f2fc206310..c653281c7e 100644 --- a/src/script_opt/ProfileFunc.h +++ b/src/script_opt/ProfileFunc.h @@ -281,7 +281,7 @@ protected: // profile is compilable. Alternatively we could derive subclasses // from ProfileFuncs and use a virtual method for this, but that seems // heavier-weight for what's really a simple notion. -typedef bool (*is_compilable_pred)(const ProfileFunc*); +typedef bool (*is_compilable_pred)(const ProfileFunc*, const char** reason); // Collectively profile an entire collection of functions. class ProfileFuncs { diff --git a/src/script_opt/Reduce.cc b/src/script_opt/Reduce.cc index 481e3a9afd..023285e322 100644 --- a/src/script_opt/Reduce.cc +++ b/src/script_opt/Reduce.cc @@ -7,6 +7,8 @@ #include "zeek/Stmt.h" #include "zeek/Desc.h" #include "zeek/Reporter.h" +#include "zeek/script_opt/ExprOptInfo.h" +#include "zeek/script_opt/StmtOptInfo.h" #include "zeek/script_opt/ProfileFunc.h" #include "zeek/script_opt/Reduce.h" #include "zeek/script_opt/TempVar.h" @@ -33,6 +35,10 @@ ExprPtr Reducer::GenTemporaryExpr(const TypePtr& t, ExprPtr rhs) { auto e = make_intrusive(GenTemporary(t, rhs)); e->SetLocationInfo(rhs->GetLocationInfo()); + + // No need to associate with current statement, since these + // are not generated during optimization. + return e; } @@ -41,7 +47,13 @@ NameExprPtr Reducer::UpdateName(NameExprPtr n) if ( NameIsReduced(n.get()) ) return n; - return make_intrusive(FindNewLocal(n)); + auto ne = make_intrusive(FindNewLocal(n)); + + // This name can be used by follow-on optimization analysis, + // so need to associate it with its statement. + BindExprToCurrStmt(ne); + + return ne; } bool Reducer::NameIsReduced(const NameExpr* n) const @@ -106,6 +118,8 @@ bool Reducer::ID_IsReduced(const ID* id) const NameExprPtr Reducer::GenInlineBlockName(const IDPtr& id) { + // We do this during reduction, not optimization, so no need + // to associate with curr_stmt. return make_intrusive(GenLocal(id)); } @@ -118,6 +132,7 @@ NameExprPtr Reducer::PushInlineBlock(TypePtr type) IDPtr ret_id = install_ID("@retvar", "", false, false); ret_id->SetType(type); + ret_id->GetOptInfo()->SetTemp(); // Track this as a new local *if* we're in the outermost inlining // block. If we're recursively deeper into inlining, then this @@ -141,48 +156,22 @@ bool Reducer::SameVal(const Val* v1, const Val* v2) const return v1 == v2; } -ExprPtr Reducer::NewVarUsage(IDPtr var, const DefPoints* dps, const Expr* orig) +ExprPtr Reducer::NewVarUsage(IDPtr var, const Expr* orig) { - if ( ! dps ) - reporter->InternalError("null defpoints in NewVarUsage"); - auto var_usage = make_intrusive(var); - SetDefPoints(var_usage.get(), dps); - TrackExprReplacement(orig, var_usage.get()); + BindExprToCurrStmt(var_usage); return var_usage; } -const DefPoints* Reducer::GetDefPoints(const NameExpr* var) +void Reducer::BindExprToCurrStmt(const ExprPtr& e) { - auto dps = FindDefPoints(var); - - if ( ! dps ) - { - auto id = var->Id(); - auto di = mgr->GetConstID_DI(id); - auto rds = mgr->GetPreMaxRDs(GetRDLookupObj(var)); - - dps = rds->GetDefPoints(di); - - SetDefPoints(var, dps); - } - - return dps; + e->GetOptInfo()->stmt_num = curr_stmt->GetOptInfo()->stmt_num; } -const DefPoints* Reducer::FindDefPoints(const NameExpr* var) const +void Reducer::BindStmtToCurrStmt(const StmtPtr& s) { - auto dps = var_usage_to_DPs.find(var); - if ( dps == var_usage_to_DPs.end() ) - return nullptr; - else - return dps->second; - } - -void Reducer::SetDefPoints(const NameExpr* var, const DefPoints* dps) - { - var_usage_to_DPs[var] = dps; + s->GetOptInfo()->stmt_num = curr_stmt->GetOptInfo()->stmt_num; } bool Reducer::SameOp(const Expr* op1, const Expr* op2) @@ -196,7 +185,7 @@ bool Reducer::SameOp(const Expr* op1, const Expr* op2) if ( op1->Tag() == EXPR_NAME ) { // Needs to be both the same identifier and in contexts - // where the identifier has the same definition points. + // where the identifier has the same definitions. auto op1_n = op1->AsNameExpr(); auto op2_n = op2->AsNameExpr(); @@ -206,10 +195,13 @@ bool Reducer::SameOp(const Expr* op1, const Expr* op2) if ( op1_id != op2_id ) return false; - auto op1_dps = GetDefPoints(op1_n); - auto op2_dps = GetDefPoints(op2_n); + auto e_stmt_1 = op1->GetOptInfo()->stmt_num; + auto e_stmt_2 = op2->GetOptInfo()->stmt_num; - return same_DPs(op1_dps, op2_dps); + auto def_1 = op1_id->GetOptInfo()->DefinitionBefore(e_stmt_1); + auto def_2 = op2_id->GetOptInfo()->DefinitionBefore(e_stmt_2); + + return def_1 == def_2 && def_1 != NO_DEF; } else if ( op1->Tag() == EXPR_CONST ) @@ -391,11 +383,10 @@ IDPtr Reducer::FindExprTmp(const Expr* rhs, const Expr* a, // always makes it here. auto id = et_i->Id().get(); - // We use 'a' in the following rather than rhs - // because the RHS can get rewritten (for example, - // due to folding) after we generate RDs, and - // thus might not have any. - if ( ! mgr->HasSinglePreMinRD(a, id) ) + auto stmt_num = a->GetOptInfo()->stmt_num; + auto def = id->GetOptInfo()->DefinitionBefore(stmt_num); + + if ( def == NO_DEF ) // The temporary's value isn't guaranteed // to make it here. continue; @@ -470,8 +461,6 @@ void Reducer::CheckIDs(const Expr* e, std::vector& ids) const bool Reducer::IsCSE(const AssignExpr* a, const NameExpr* lhs, const Expr* rhs) { - auto a_max_rds = mgr->GetPostMaxRDs(GetRDLookupObj(a)); - auto lhs_id = lhs->Id(); auto lhs_tmp = FindTemporary(lhs_id); // nil if LHS not a temporary auto rhs_tmp = FindExprTmp(rhs, a, lhs_tmp); @@ -479,9 +468,7 @@ bool Reducer::IsCSE(const AssignExpr* a, const NameExpr* lhs, const Expr* rhs) ExprPtr new_rhs; if ( rhs_tmp ) { // We already have a temporary - auto tmp_di = mgr->GetConstID_DI(rhs_tmp.get()); - auto dps = a_max_rds->GetDefPoints(tmp_di); - new_rhs = NewVarUsage(rhs_tmp, dps, rhs); + new_rhs = NewVarUsage(rhs_tmp, rhs); rhs = new_rhs.get(); } @@ -507,104 +494,73 @@ bool Reducer::IsCSE(const AssignExpr* a, const NameExpr* lhs, const Expr* rhs) // Treat the LHS as either an alias for the RHS, // or as a constant if the RHS is a constant in // this context. - auto rhs_di = mgr->GetConstID_DI(rhs_id.get()); - auto dps = a_max_rds->GetDefPoints(rhs_di); + auto stmt_num = a->GetOptInfo()->stmt_num; + auto rhs_const = CheckForConst(rhs_id, stmt_num); - auto rhs_const = CheckForConst(rhs_id, dps); if ( rhs_const ) lhs_tmp->SetConst(rhs_const); else - lhs_tmp->SetAlias(rhs_id, dps); + lhs_tmp->SetAlias(rhs_id); return true; } - // Track where we define the temporary. - auto lhs_di = mgr->GetConstID_DI(lhs_id); - auto dps = a_max_rds->GetDefPoints(lhs_di); - if ( lhs_tmp->DPs() && ! same_DPs(lhs_tmp->DPs(), dps) ) - reporter->InternalError("double DPs for temporary"); - - lhs_tmp->SetDPs(dps); - SetDefPoints(lhs, dps); - expr_temps.emplace_back(lhs_tmp); } return false; } -const ConstExpr* Reducer::CheckForConst(const IDPtr& id, - const DefPoints* dps) const +const ConstExpr* Reducer::CheckForConst(const IDPtr& id, int stmt_num) const { - if ( ! dps || dps->length() == 0 ) - // This can happen for access to uninitialized values. + if ( id->GetType()->Tag() == TYPE_ANY ) + // Don't propagate identifiers of type "any" as constants. + // This is because the identifier might be used in some + // context that's dynamically unreachable due to the type + // of its value (such as via a type-switch), but for which + // constant propagation of the constant value to that + // context can result in compile-time errors when folding + // expressions in which the identifier appears (and is + // in that context presumed to have a different type). return nullptr; - if ( dps->length() != 1 ) - // Multiple definitions of the variable reach to this - // location. In theory we could check whether they *all* - // provide the same constant, but that hardly seems likely. - return nullptr; + auto oi = id->GetOptInfo(); + auto c = oi->Const(); - // Identifier has a unique definition. - auto dp = (*dps)[0]; - const Expr* e = nullptr; + if ( c ) + return c; - if ( dp.Tag() == STMT_DEF ) + auto e = id->GetOptInfo()->DefExprBefore(stmt_num); + if ( e ) { - auto s = dp.StmtVal(); + auto ce = constant_exprs.find(e.get()); + if ( ce != constant_exprs.end() ) + e = ce->second; - if ( s->Tag() == STMT_CATCH_RETURN ) - { - // Change 's' to refer to the associated assignment - // statement, if any. - auto cr = s->AsCatchReturnStmt(); - s = cr->AssignStmt().get(); + if ( e->Tag() == EXPR_CONST ) + return e->AsConstExpr(); - if ( ! s ) - return nullptr; - } - - if ( s->Tag() != STMT_EXPR ) - // Defined in a statement other than an assignment. + // Follow aliases. + if ( e->Tag() != EXPR_NAME ) return nullptr; - e = s->AsExprStmt()->StmtExpr(); + return CheckForConst(e->AsNameExpr()->IdPtr(), stmt_num); } - else if ( dp.Tag() == EXPR_DEF ) - e = dp.ExprVal(); - - else - return nullptr; - - if ( e->Tag() != EXPR_ASSIGN ) - // Not sure why this would happen, other than EXPR_APPEND_TO, - // but in any case not an expression we can mine for a - // constant. - return nullptr; - - auto rhs = e->GetOp2(); - - if ( rhs->Tag() != EXPR_CONST ) - return nullptr; - - return rhs->AsConstExpr(); + return nullptr; } -void Reducer::TrackExprReplacement(const Expr* orig, const Expr* e) +ConstExprPtr Reducer::Fold(ExprPtr e) { - new_expr_to_orig[e] = orig; + auto c = make_intrusive(e->Eval(nullptr)); + FoldedTo(e, c); + return c; } -const Obj* Reducer::GetRDLookupObj(const Expr* e) const +void Reducer::FoldedTo(ExprPtr e, ConstExprPtr c) { - auto orig_e = new_expr_to_orig.find(e); - if ( orig_e == new_expr_to_orig.end() ) - return e; - else - return orig_e->second; + constant_exprs[e.get()] = std::move(c); + folded_exprs.push_back(std::move(e)); } ExprPtr Reducer::OptExpr(Expr* e) @@ -635,13 +591,10 @@ ExprPtr Reducer::UpdateExpr(ExprPtr e) auto tmp_var = FindTemporary(id); if ( ! tmp_var ) { - auto max_rds = mgr->GetPreMaxRDs(GetRDLookupObj(n)); - IDPtr id_ptr = {NewRef{}, id}; - auto di = mgr->GetConstID_DI(id); - auto dps = max_rds->GetDefPoints(di); + auto stmt_num = e->GetOptInfo()->stmt_num; + auto is_const = CheckForConst(id_ptr, stmt_num); - auto is_const = CheckForConst(id_ptr, dps); if ( is_const ) { // Remember this variable as one whose value @@ -662,36 +615,33 @@ ExprPtr Reducer::UpdateExpr(ExprPtr e) auto alias = tmp_var->Alias(); if ( alias ) { - // Make sure that the definition points for the - // alias here are the same as when the alias - // was created. + // Make sure that the definitions for the alias here are + // the same as when the alias was created. auto alias_tmp = FindTemporary(alias.get()); - if ( alias_tmp ) + // Resolve any alias chains. + while ( alias_tmp && alias_tmp->Alias() ) { - while ( alias_tmp->Alias() ) - { - // Alias chains can occur due to - // re-reduction while optimizing. - auto a_id = alias_tmp->Id(); - if ( a_id == id ) - return e; + alias = alias_tmp->Alias(); + alias_tmp = FindTemporary(alias.get()); + } - alias_tmp = FindTemporary(alias_tmp->Id().get()); - } - - // Temporaries always have only one definition point, + if ( alias->GetOptInfo()->IsTemp() ) + { + // Temporaries always have only one definition, // so no need to check for consistency. - auto new_usage = NewVarUsage(alias, alias_tmp->DPs(), e.get()); + auto new_usage = NewVarUsage(alias, e.get()); return new_usage; } - auto e_max_rds = mgr->GetPreMaxRDs(GetRDLookupObj(e.get())); - auto alias_di = mgr->GetConstID_DI(alias.get()); - auto alias_dps = e_max_rds->GetDefPoints(alias_di); + auto e_stmt_1 = e->GetOptInfo()->stmt_num; + auto e_stmt_2 = tmp_var->RHS()->GetOptInfo()->stmt_num; - if ( same_DPs(alias_dps, tmp_var->DPs()) ) - return NewVarUsage(alias, alias_dps, e.get()); + auto def_1 = alias->GetOptInfo()->DefinitionBefore(e_stmt_1); + auto def_2 = tmp_var->Id()->GetOptInfo()->DefinitionBefore(e_stmt_2); + + if ( def_1 == def_2 && def_1 != NO_DEF ) + return NewVarUsage(alias, e.get()); else return e; } @@ -758,9 +708,17 @@ StmtPtr Reducer::MergeStmts(const NameExpr* lhs, ExprPtr rhs, Stmt* succ_stmt) lhs_tmp->Deactivate(); auto merge_e = make_intrusive(a_lhs_deref, rhs, false, nullptr, nullptr, false); - TrackExprReplacement(rhs.get(), merge_e.get()); + auto merge_e_stmt = make_intrusive(merge_e); - return make_intrusive(merge_e); + // Update the associated stmt_num's. For strict correctness, we + // want both of these bound to the earlier of the two statements + // we're merging (though in practice, either will work, since + // we're eliding the only difference between the two). Our + // caller ensures this. + BindExprToCurrStmt(merge_e); + BindStmtToCurrStmt(merge_e_stmt); + + return merge_e_stmt; } IDPtr Reducer::GenTemporary(const TypePtr& t, ExprPtr rhs) @@ -809,6 +767,9 @@ IDPtr Reducer::GenLocal(const IDPtr& orig) local_id->SetType(orig->GetType()); local_id->SetAttrs(orig->GetAttrs()); + if ( orig->GetOptInfo()->IsTemp() ) + local_id->GetOptInfo()->SetTemp(); + new_locals.insert(local_id.get()); orig_to_new_locals[orig.get()] = local_id; @@ -1040,27 +1001,6 @@ bool CSE_ValidityChecker::CheckAggrMod(const std::vector& ids, } -bool same_DPs(const DefPoints* dp1, const DefPoints* dp2) - { - if ( dp1 == dp2 ) - return true; - - if ( ! dp1 || ! dp2 ) - return false; - - // Given how we construct DPs, they should be element-by-element - // equivalent; we don't have to worry about reordering. - if ( dp1->length() != dp2->length() ) - return false; - - for ( auto i = 0; i < dp1->length(); ++i ) - if ( ! (*dp1)[i].SameAs((*dp2)[i]) ) - return false; - - return true; - } - - const Expr* non_reduced_perp; bool checking_reduction; diff --git a/src/script_opt/Reduce.h b/src/script_opt/Reduce.h index 3df88d2913..db51537ae7 100644 --- a/src/script_opt/Reduce.h +++ b/src/script_opt/Reduce.h @@ -6,13 +6,11 @@ #include "zeek/Expr.h" #include "zeek/Stmt.h" #include "zeek/Traverse.h" -#include "zeek/script_opt/DefSetsMgr.h" namespace zeek::detail { class Expr; class TempVar; -class ProfileFunc; class Reducer { public: @@ -20,8 +18,9 @@ public: StmtPtr Reduce(StmtPtr s); - const DefSetsMgr* GetDefSetsMgr() const { return mgr; } - void SetDefSetsMgr(const DefSetsMgr* _mgr) { mgr = _mgr; } + void SetReadyToOptimize() { opt_ready = true; } + + void SetCurrStmt(const Stmt* stmt) { curr_stmt = stmt; } ExprPtr GenTemporaryExpr(const TypePtr& t, ExprPtr rhs); @@ -76,7 +75,7 @@ public: // True if the Reducer is being used in the context of a second // pass over for AST optimization. bool Optimizing() const - { return ! IsPruning() && mgr != nullptr; } + { return ! IsPruning() && opt_ready; } // A predicate that indicates whether a given reduction pass // is being made to prune unused statements. @@ -126,6 +125,14 @@ public: // already been applied. bool IsCSE(const AssignExpr* a, const NameExpr* lhs, const Expr* rhs); + // Returns a constant representing folding of the given expression + // (which must have constant operands). + ConstExprPtr Fold(ExprPtr e); + + // Notes that the given expression has been folded to the + // given constant. + void FoldedTo(ExprPtr orig, ConstExprPtr c); + // Given an lhs=rhs statement followed by succ_stmt, returns // a (new) merge of the two if they're of the form tmp=rhs, var=tmp; // otherwise, nil. @@ -150,23 +157,13 @@ protected: // are in fact equivalent.) bool SameVal(const Val* v1, const Val* v2) const; - // Track that the variable "var", which has the given set of - // definition points, will be a replacement for the "orig" - // expression. Returns the replacement expression (which is - // is just a NameExpr referring to "var"). - ExprPtr NewVarUsage(IDPtr var, const DefPoints* dps, const Expr* orig); + // Track that the variable "var" will be a replacement for + // the "orig" expression. Returns the replacement expression + // (which is is just a NameExpr referring to "var"). + ExprPtr NewVarUsage(IDPtr var, const Expr* orig); - // Returns the definition points associated with "var". If none - // exist in our cache, then populates the cache. - const DefPoints* GetDefPoints(const NameExpr* var); - - // Retrieve the definition points associated in our cache with the - // given variable, if any. - const DefPoints* FindDefPoints(const NameExpr* var) const; - - // Adds a mapping in our cache of the given variable to the given - // set of definition points. - void SetDefPoints(const NameExpr* var, const DefPoints* dps); + void BindExprToCurrStmt(const ExprPtr& e); + void BindStmtToCurrStmt(const StmtPtr& s); // Returns true if op1 and op2 represent the same operand, given // the reaching definitions available at their usages (e1 and e2). @@ -216,23 +213,10 @@ protected: // for the current function. IDPtr GenLocal(const IDPtr& orig); - // This is the heart of constant propagation. Given an identifier - // and a set of definition points for it, if its value is constant - // then returns the corresponding ConstExpr with the value. - const ConstExpr* CheckForConst(const IDPtr& id, - const DefPoints* dps) const; - - // Track that we're replacing instances of "orig" with a new - // expression. This allows us to locate the RDs associated - // with "orig" in the context of the new expression, without - // requiring an additional RD propagation pass. - void TrackExprReplacement(const Expr* orig, const Expr* e); - - // Returns the object we should use to look up RD's associated - // with 'e'. (This isn't necessarily 'e' itself because we - // may have decided to replace it with a different expression, - // per TrackExprReplacement().) - const Obj* GetRDLookupObj(const Expr* e) const; + // This is the heart of constant propagation. Given an identifier, + // if its value is constant at the given location then returns + // the corresponding ConstExpr with the value. + const ConstExpr* CheckForConst(const IDPtr& id, int stmt_num) const; // Tracks the temporary variables created during the reduction/ // optimization process. @@ -253,6 +237,14 @@ protected: // rename local variables when inlining. std::unordered_map orig_to_new_locals; + // Tracks expressions we've folded, so that we can recognize them + // for constant propagation. + std::unordered_map constant_exprs; + + // Holds onto those expressions so they don't become invalid + // due to memory management. + std::vector folded_exprs; + // Which statements to elide from the AST (because optimization // has determined they're no longer needed). std::unordered_set omitted_stmts; @@ -270,25 +262,17 @@ protected: // exponentially. int bifurcation_level = 0; - // For a given usage of a variable's value, return the definition - // points associated with its use at that point. We use this - // both as a cache (populating it every time we do a more laborious - // lookup), and proactively when creating new references to variables. - std::unordered_map var_usage_to_DPs; - // Tracks which (non-temporary) variables had constant // values used for constant propagation. std::unordered_set constant_vars; - // For a new expression we've created, map it to the expression - // it's replacing. This allows us to locate the RDs associated - // with the usage. - std::unordered_map new_expr_to_orig; - // Statement at which the current reduction started. StmtPtr reduction_root = nullptr; - const DefSetsMgr* mgr = nullptr; + // Statement we're currently working on. + const Stmt* curr_stmt = nullptr; + + bool opt_ready = false; }; @@ -364,8 +348,6 @@ protected: }; -extern bool same_DPs(const DefPoints* dp1, const DefPoints* dp2); - // Used for debugging, to communicate which expression wasn't // reduced when we expected them all to be. extern const Expr* non_reduced_perp; diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index 47f0765b57..421eac50a8 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -6,15 +6,20 @@ #include "zeek/Desc.h" #include "zeek/EventHandler.h" #include "zeek/EventRegistry.h" + #include "zeek/script_opt/ScriptOpt.h" #include "zeek/script_opt/ProfileFunc.h" #include "zeek/script_opt/Inline.h" #include "zeek/script_opt/Reduce.h" +#include "zeek/script_opt/GenIDDefs.h" #include "zeek/script_opt/GenRDs.h" #include "zeek/script_opt/UseDefs.h" + #include "zeek/script_opt/CPP/Compile.h" #include "zeek/script_opt/CPP/Func.h" +#include "zeek/script_opt/ZAM/Compile.h" + namespace zeek::detail { @@ -31,21 +36,76 @@ static std::vector funcs; static ZAMCompiler* ZAM = nullptr; +static bool generating_CPP = false; +static std::string hash_dir; // for storing hashes of previous compilations -void optimize_func(ScriptFunc* f, std::shared_ptr pf, - ScopePtr scope, StmtPtr& body, - AnalyOpt& analysis_options) +static ScriptFuncPtr global_stmts; + +void analyze_func(ScriptFuncPtr f) + { + // Even if we're doing --optimize-only, we still track all functions + // here because the inliner will need the full list. + funcs.emplace_back(f, f->GetScope(), f->CurrentBody(), + f->CurrentPriority()); + } + +const FuncInfo* analyze_global_stmts(Stmt* stmts) + { + // We ignore analysis_options.only_func - if it's in use, later + // logic will keep this function from being compiled, but it's handy + // now to enter it into "funcs" so we have a FuncInfo to return. + + auto id = install_ID("", GLOBAL_MODULE_NAME, true, false); + auto empty_args_t = make_intrusive(nullptr); + auto func_t = make_intrusive(empty_args_t, nullptr, FUNC_FLAVOR_FUNCTION); + id->SetType(func_t); + + auto sc = current_scope(); + std::vector empty_inits; + StmtPtr stmts_p{NewRef{}, stmts}; + global_stmts = make_intrusive(id, stmts_p, empty_inits, + sc->Length(), 0); + + funcs.emplace_back(global_stmts, sc, stmts_p, 0); + + return &funcs.back(); + } + + +static bool optimize_AST(ScriptFunc* f, std::shared_ptr& pf, + std::shared_ptr& rc, ScopePtr scope, + StmtPtr& body) + { + pf = std::make_shared(f, body, true); + + // RD_Decorate reduced_rds(pf, f, scope, body); + GenIDDefs ID_defs(pf, f, scope, body); + + if ( reporter->Errors() > 0 ) + return false; + + rc->SetReadyToOptimize(); + + auto new_body = rc->Reduce(body); + + if ( reporter->Errors() > 0 ) + return false; + + if ( analysis_options.dump_xform ) + printf("Optimized: %s\n", obj_desc(new_body.get()).c_str()); + + f->ReplaceBody(body, new_body); + body = new_body; + + return true; + } + +static void optimize_func(ScriptFunc* f, std::shared_ptr pf, + ScopePtr scope, StmtPtr& body) { if ( reporter->Errors() > 0 ) return; - if ( ! analysis_options.activate ) - return; - - if ( analysis_options.only_func && - *analysis_options.only_func != f->Name() ) - return; - if ( analysis_options.only_func ) printf("Original: %s\n", obj_desc(body.get()).c_str()); @@ -53,10 +113,12 @@ void optimize_func(ScriptFunc* f, std::shared_ptr pf, // We're not able to optimize this. return; - if ( pf->NumWhenStmts() > 0 || pf->NumLambdas() > 0 ) + const char* reason; + if ( ! is_ZAM_compilable(pf.get(), &reason) ) { - if ( analysis_options.only_func ) - printf("Skipping analysis due to \"when\" statement or use of lambdas\n"); + if ( analysis_options.report_uncompilable ) + printf("Skipping compilation of %s due to %s\n", + f->Name(), reason); return; } @@ -77,59 +139,35 @@ void optimize_func(ScriptFunc* f, std::shared_ptr pf, if ( ! new_body->IsReduced(rc.get()) ) { if ( non_reduced_perp ) - printf("Reduction inconsistency for %s: %s\n", f->Name(), + reporter->InternalError("Reduction inconsistency for %s: %s\n", f->Name(), obj_desc(non_reduced_perp).c_str()); else - printf("Reduction inconsistency for %s\n", f->Name()); + reporter->InternalError("Reduction inconsistency for %s\n", f->Name()); } checking_reduction = false; - if ( analysis_options.only_func || analysis_options.dump_xform ) + if ( analysis_options.dump_xform ) printf("Transformed: %s\n", obj_desc(new_body.get()).c_str()); f->ReplaceBody(body, new_body); body = new_body; - if ( analysis_options.optimize_AST ) + if ( analysis_options.optimize_AST && + ! optimize_AST(f, pf, rc, scope, body) ) { - pf = std::make_shared(f, body, true); - - RD_Decorate reduced_rds(pf); - reduced_rds.TraverseFunction(f, scope, body); - - if ( reporter->Errors() > 0 ) - { - pop_scope(); - return; - } - - rc->SetDefSetsMgr(reduced_rds.GetDefSetsMgr()); - - new_body = rc->Reduce(body); - - if ( reporter->Errors() > 0 ) - { - pop_scope(); - return; - } - - if ( analysis_options.only_func || analysis_options.dump_xform ) - printf("Optimized: %s\n", obj_desc(new_body.get()).c_str()); - - f->ReplaceBody(body, new_body); - body = new_body; + pop_scope(); + return; } // Profile the new body. pf = std::make_shared(f, body, true); // Compute its reaching definitions. - RD_Decorate reduced_rds(pf); + // RD_Decorate reduced_rds(pf, f, scope, body); + GenIDDefs ID_defs(pf, f, scope, body); - reduced_rds.TraverseFunction(f, scope, body); - - rc->SetDefSetsMgr(reduced_rds.GetDefSetsMgr()); + rc->SetReadyToOptimize(); auto ud = std::make_shared(body, rc); ud->Analyze(); @@ -145,315 +183,290 @@ void optimize_func(ScriptFunc* f, std::shared_ptr pf, body = new_body; } - int new_frame_size = - scope->Length() + rc->NumTemps() + rc->NumNewLocals(); + int new_frame_size = scope->Length() + rc->NumTemps() + + rc->NumNewLocals(); if ( new_frame_size > f->FrameSize() ) f->SetFrameSize(new_frame_size); + if ( analysis_options.gen_ZAM_code ) + { + ZAM = new ZAMCompiler(f, pf, scope, new_body, ud, rc); + + new_body = ZAM->CompileBody(); + + if ( reporter->Errors() > 0 ) + return; + + if ( analysis_options.dump_ZAM ) + ZAM->Dump(); + + f->ReplaceBody(body, new_body); + body = new_body; + } + pop_scope(); } -FuncInfo::FuncInfo(ScriptFuncPtr _func, ScopePtr _scope, StmtPtr _body, - int _priority) - : func(std::move(_func)), scope(std::move(_scope)), - body(std::move(_body)), priority(_priority) - {} - -void FuncInfo::SetProfile(std::shared_ptr _pf) - { pf = std::move(_pf); } - -void analyze_func(ScriptFuncPtr f) - { - if ( analysis_options.only_func && - *analysis_options.only_func != f->Name() ) - return; - - funcs.emplace_back(f, f->GetScope(), f->CurrentBody(), - f->CurrentPriority()); - } - -const FuncInfo* analyze_global_stmts(Stmt* stmts) - { - // We ignore analysis_options.only_func - if it's in use, later - // logic will keep this function from being compiled, but it's handy - // now to enter it into "funcs" so we have a FuncInfo to return. - - auto id = install_ID("", GLOBAL_MODULE_NAME, true, false); - auto empty_args_t = make_intrusive(nullptr); - auto func_t = make_intrusive(empty_args_t, nullptr, FUNC_FLAVOR_FUNCTION); - id->SetType(func_t); - - auto sc = current_scope(); - std::vector empty_inits; - StmtPtr stmts_p{NewRef{}, stmts}; - auto sf = make_intrusive(id, stmts_p, empty_inits, sc->Length(), 0); - - funcs.emplace_back(sf, sc, stmts_p, 0); - - return &funcs.back(); - } - static void check_env_opt(const char* opt, bool& opt_flag) { if ( getenv(opt) ) opt_flag = true; } -void analyze_scripts() +static void init_options() { - static bool did_init = false; - static std::string hash_dir; + auto hd = getenv("ZEEK_HASH_DIR"); + if ( hd ) + hash_dir = std::string(hd) + "/"; - bool generating_CPP = false; + // ZAM-related options. + check_env_opt("ZEEK_DUMP_XFORM", analysis_options.dump_xform); + check_env_opt("ZEEK_DUMP_UDS", analysis_options.dump_uds); + check_env_opt("ZEEK_INLINE", analysis_options.inliner); + check_env_opt("ZEEK_OPT", analysis_options.optimize_AST); + check_env_opt("ZEEK_XFORM", analysis_options.activate); + check_env_opt("ZEEK_ZAM", analysis_options.gen_ZAM); + check_env_opt("ZEEK_COMPILE_ALL", analysis_options.compile_all); + check_env_opt("ZEEK_ZAM_CODE", analysis_options.gen_ZAM_code); + check_env_opt("ZEEK_NO_ZAM_OPT", analysis_options.no_ZAM_opt); + check_env_opt("ZEEK_DUMP_ZAM", analysis_options.dump_ZAM); + check_env_opt("ZEEK_PROFILE", analysis_options.profile_ZAM); - if ( ! did_init ) + // Compile-to-C++-related options. + check_env_opt("ZEEK_ADD_CPP", analysis_options.add_CPP); + check_env_opt("ZEEK_UPDATE_CPP", analysis_options.update_CPP); + check_env_opt("ZEEK_GEN_CPP", analysis_options.gen_CPP); + check_env_opt("ZEEK_GEN_STANDALONE_CPP", + analysis_options.gen_standalone_CPP); + check_env_opt("ZEEK_COMPILE_ALL", analysis_options.compile_all); + check_env_opt("ZEEK_REPORT_CPP", analysis_options.report_CPP); + check_env_opt("ZEEK_USE_CPP", analysis_options.use_CPP); + + if ( analysis_options.gen_standalone_CPP ) + analysis_options.gen_CPP = true; + + if ( analysis_options.gen_CPP ) { - auto hd = getenv("ZEEK_HASH_DIR"); - if ( hd ) - hash_dir = std::string(hd) + "/"; - - check_env_opt("ZEEK_DUMP_XFORM", analysis_options.dump_xform); - check_env_opt("ZEEK_DUMP_UDS", analysis_options.dump_uds); - check_env_opt("ZEEK_INLINE", analysis_options.inliner); - check_env_opt("ZEEK_OPT", analysis_options.optimize_AST); - check_env_opt("ZEEK_XFORM", analysis_options.activate); - check_env_opt("ZEEK_ADD_CPP", analysis_options.add_CPP); - check_env_opt("ZEEK_UPDATE_CPP", analysis_options.update_CPP); - check_env_opt("ZEEK_GEN_CPP", analysis_options.gen_CPP); - check_env_opt("ZEEK_GEN_STANDALONE_CPP", - analysis_options.gen_standalone_CPP); - check_env_opt("ZEEK_COMPILE_ALL", analysis_options.compile_all); - check_env_opt("ZEEK_REPORT_CPP", analysis_options.report_CPP); - check_env_opt("ZEEK_USE_CPP", analysis_options.use_CPP); - - if ( analysis_options.gen_standalone_CPP ) - analysis_options.gen_CPP = true; - - if ( analysis_options.gen_CPP ) + if ( analysis_options.add_CPP ) { - if ( analysis_options.add_CPP ) - { - reporter->Warning("gen-C++ incompatible with add-C++"); - analysis_options.add_CPP = false; - } - - if ( analysis_options.update_CPP ) - { - reporter->Warning("gen-C++ incompatible with update-C++"); - analysis_options.update_CPP = false; - } - - generating_CPP = true; + reporter->Warning("gen-C++ incompatible with add-C++"); + analysis_options.add_CPP = false; } - if ( analysis_options.update_CPP || analysis_options.add_CPP ) - generating_CPP = true; - - if ( analysis_options.use_CPP && generating_CPP ) + if ( analysis_options.update_CPP ) { - reporter->Error("generating C++ incompatible with using C++"); - exit(1); + reporter->Warning("gen-C++ incompatible with update-C++"); + analysis_options.update_CPP = false; } - if ( analysis_options.use_CPP && ! CPP_init_hook ) - { - reporter->Error("no C++ functions available to use"); - exit(1); - } - - auto usage = getenv("ZEEK_USAGE_ISSUES"); - - if ( usage ) - analysis_options.usage_issues = atoi(usage) > 1 ? 2 : 1; - - if ( ! analysis_options.only_func ) - { - auto zo = getenv("ZEEK_ONLY"); - if ( zo ) - analysis_options.only_func = zo; - } - - if ( analysis_options.only_func || - analysis_options.optimize_AST || - analysis_options.usage_issues > 0 ) - analysis_options.activate = true; - - did_init = true; + generating_CPP = true; } - if ( ! analysis_options.activate && ! analysis_options.inliner && - ! generating_CPP && ! analysis_options.report_CPP && - ! analysis_options.use_CPP ) - // Avoid profiling overhead. - return; + if ( analysis_options.update_CPP || analysis_options.add_CPP ) + generating_CPP = true; - // Now that everything's parsed and BiF's have been initialized, - // profile the functions. - auto pfs = std::make_unique(funcs, is_CPP_compilable, false); + if ( analysis_options.use_CPP && generating_CPP ) + reporter->FatalError("generating C++ incompatible with using C++"); - if ( CPP_init_hook ) - (*CPP_init_hook)(); + if ( analysis_options.use_CPP && ! CPP_init_hook ) + reporter->FatalError("no C++ functions available to use"); - if ( analysis_options.report_CPP ) + auto usage = getenv("ZEEK_USAGE_ISSUES"); + + if ( usage ) + analysis_options.usage_issues = atoi(usage) > 1 ? 2 : 1; + + if ( ! analysis_options.only_func ) { - if ( ! CPP_init_hook ) - { - printf("no C++ script bodies available\n"); - exit(0); - } + auto zo = getenv("ZEEK_ONLY"); + if ( zo ) + analysis_options.only_func = zo; + } - printf("C++ script bodies available that match loaded scripts:\n"); + if ( analysis_options.gen_ZAM ) + { + analysis_options.gen_ZAM_code = true; + analysis_options.inliner = true; + analysis_options.optimize_AST = true; + } - std::unordered_set already_reported; + if ( analysis_options.dump_ZAM ) + analysis_options.gen_ZAM_code = true; - for ( auto& f : funcs ) - { - auto name = f.Func()->Name(); - auto hash = f.Profile()->HashVal(); - bool have = compiled_scripts.count(hash) > 0; - auto specific = ""; + if ( analysis_options.only_func ) + { + // Note, this comes after the statement above because for + // --optimize-only we don't necessarily want to go all + // the way to *generating* ZAM code, though we'll want to + // dump it *if* we generate it. + analysis_options.dump_xform = analysis_options.dump_ZAM = true; - if ( ! have ) - { - hash = script_specific_hash(f.Body(), hash); - have = compiled_scripts.count(hash) > 0; - if ( have ) - specific = " - specific"; - } + if ( analysis_options.gen_ZAM_code || generating_CPP ) + analysis_options.report_uncompilable = true; + } - printf("script function %s (hash %llu%s): %s\n", - name, hash, specific, have ? "yes" : "no"); + if ( analysis_options.report_uncompilable && + ! analysis_options.gen_ZAM_code && ! generating_CPP ) + reporter->FatalError("report-uncompilable requires generation of ZAM or C++"); - if ( have ) - already_reported.insert(hash); - } - - printf("\nAdditional C++ script bodies available:\n"); - int addl = 0; - for ( const auto& s : compiled_scripts ) - if ( already_reported.count(s.first) == 0 ) - { - printf("%s body (hash %llu)\n", - s.second.body->Name().c_str(), s.first); - ++addl; - } - - if ( addl == 0 ) - printf("(none)\n"); + if ( analysis_options.only_func || + analysis_options.optimize_AST || + analysis_options.gen_ZAM_code || + analysis_options.usage_issues > 0 ) + analysis_options.activate = true; + } +static void report_CPP() + { + if ( ! CPP_init_hook ) + { + printf("no C++ script bodies available\n"); exit(0); } - if ( analysis_options.use_CPP ) + printf("C++ script bodies available that match loaded scripts:\n"); + + std::unordered_set already_reported; + + for ( auto& f : funcs ) { - for ( auto& f : funcs ) + auto name = f.Func()->Name(); + auto hash = f.Profile()->HashVal(); + bool have = compiled_scripts.count(hash) > 0; + auto specific = ""; + + if ( ! have ) { - auto hash = f.Profile()->HashVal(); - auto s = compiled_scripts.find(hash); - - if ( s == compiled_scripts.end() ) - { // Look for script-specific body. - hash = script_specific_hash(f.Body(), hash); - s = compiled_scripts.find(hash); - } - - if ( s != compiled_scripts.end() ) - { - auto b = s->second.body; - b->SetHash(hash); - - // We may have already updated the body if - // we're using code compiled for standalone. - if ( f.Body()->Tag() != STMT_CPP ) - { - auto func = f.Func(); - if ( added_bodies[func->Name()].count(hash) > 0 ) - // We've already added the - // replacement. Delete orig. - func->ReplaceBody(f.Body(), nullptr); - else - func->ReplaceBody(f.Body(), b); - - f.SetBody(b); - } - - for ( auto& e : s->second.events ) - { - auto h = event_registry->Register(e); - h->SetUsed(); - } - } + hash = script_specific_hash(f.Body(), hash); + have = compiled_scripts.count(hash) > 0; + if ( have ) + specific = " - specific"; } - // Now that we've loaded all of the compiled scripts - // relevant for the AST, activate standalone ones. - for ( auto cb : standalone_activations ) - (*cb)(); + printf("script function %s (hash %llu%s): %s\n", + name, hash, specific, have ? "yes" : "no"); + + if ( have ) + already_reported.insert(hash); } - if ( generating_CPP ) - { - const auto hash_name = hash_dir + "CPP-hashes"; + printf("\nAdditional C++ script bodies available:\n"); - auto hm = std::make_unique(hash_name.c_str(), - analysis_options.add_CPP); - - if ( ! analysis_options.gen_CPP ) + int addl = 0; + for ( const auto& s : compiled_scripts ) + if ( already_reported.count(s.first) == 0 ) { + printf("%s body (hash %llu)\n", + s.second.body->Name().c_str(), s.first); + ++addl; + } + + if ( addl == 0 ) + printf("(none)\n"); + } + +static void use_CPP() + { + for ( auto& f : funcs ) + { + auto hash = f.Profile()->HashVal(); + auto s = compiled_scripts.find(hash); + + if ( s == compiled_scripts.end() ) + { // Look for script-specific body. + hash = script_specific_hash(f.Body(), hash); + s = compiled_scripts.find(hash); + } + + if ( s != compiled_scripts.end() ) + { + auto b = s->second.body; + b->SetHash(hash); + + // We may have already updated the body if + // we're using code compiled for standalone. + if ( f.Body()->Tag() != STMT_CPP ) + { + auto func = f.Func(); + if ( added_bodies[func->Name()].count(hash) > 0 ) + // We've already added the + // replacement. Delete orig. + func->ReplaceBody(f.Body(), nullptr); + else + func->ReplaceBody(f.Body(), b); + + f.SetBody(b); + } + + for ( auto& e : s->second.events ) + { + auto h = event_registry->Register(e); + h->SetUsed(); + } + } + } + + // Now that we've loaded all of the compiled scripts + // relevant for the AST, activate standalone ones. + for ( auto cb : standalone_activations ) + (*cb)(); + } + +static void generate_CPP(std::unique_ptr& pfs) + { + const auto hash_name = hash_dir + "CPP-hashes"; + + auto hm = std::make_unique(hash_name.c_str(), + analysis_options.add_CPP); + + if ( analysis_options.gen_CPP ) + { + if ( analysis_options.only_func ) + { // deactivate all functions except the target one for ( auto& func : funcs ) { - auto hash = func.Profile()->HashVal(); - if ( compiled_scripts.count(hash) > 0 || - hm->HasHash(hash) ) + auto fn = func.Func()->Name(); + if ( *analysis_options.only_func != fn ) func.SetSkip(true); } - - // Now that we've presumably marked a lot of functions - // as skippable, recompute the global profile. - pfs = std::make_unique(funcs, is_CPP_compilable, false); + } + } + else + { // doing add-C++ instead, so look for previous compilations + for ( auto& func : funcs ) + { + auto hash = func.Profile()->HashVal(); + if ( compiled_scripts.count(hash) > 0 || + hm->HasHash(hash) ) + func.SetSkip(true); } - const auto gen_name = hash_dir + "CPP-gen.cc"; - const auto addl_name = hash_dir + "CPP-gen-addl.h"; - - CPPCompile cpp(funcs, *pfs, gen_name, addl_name, *hm, - analysis_options.gen_CPP || - analysis_options.update_CPP, - analysis_options.gen_standalone_CPP); - - exit(0); + // Now that we've presumably marked a lot of functions + // as skippable, recompute the global profile. + pfs = std::make_unique(funcs, is_CPP_compilable, false); } - if ( analysis_options.usage_issues > 0 && analysis_options.optimize_AST ) - { - fprintf(stderr, "warning: \"-O optimize-AST\" option is incompatible with -u option, deactivating optimization\n"); - analysis_options.optimize_AST = false; - } + const auto gen_name = hash_dir + "CPP-gen.cc"; + const auto addl_name = hash_dir + "CPP-gen-addl.h"; - // Re-profile the functions, this time without worrying about - // compatibility with compilation to C++. Note that the first - // profiling pass above may have marked some of the functions - // as to-skip, so first clear those markings. Once we have - // full compile-to-C++ and ZAM support for all Zeek language - // features, we can remove the re-profiling here. - for ( auto& f : funcs ) - f.SetSkip(false); - - pfs = std::make_unique(funcs, nullptr, true); + CPPCompile cpp(funcs, *pfs, gen_name, addl_name, *hm, + analysis_options.gen_CPP || analysis_options.update_CPP, + analysis_options.gen_standalone_CPP, + analysis_options.report_uncompilable); + } +static void find_when_funcs(std::unique_ptr& pfs, + std::unordered_set& when_funcs) + { // Figure out which functions either directly or indirectly // appear in "when" clauses. - // Final set of functions involved in "when" clauses. - std::unordered_set when_funcs; - // Which functions we still need to analyze. std::unordered_set when_funcs_to_do; for ( auto& f : funcs ) - { if ( f.Profile()->WhenCalls().size() > 0 ) { when_funcs.insert(f.Func()); @@ -463,18 +476,7 @@ void analyze_scripts() ASSERT(pfs->FuncProf(bf)); when_funcs_to_do.insert(bf); } - -#ifdef NOT_YET - if ( analysis_options.report_uncompilable ) - { - ODesc d; - f.ScriptFunc()->AddLocation(&d); - printf("%s cannot be compiled due to use of \"when\" statement (%s)\n", - f.ScriptFunc()->Name(), d.Description()); - } -#endif // NOT_YET } - } // Set of new functions to put on to-do list. Separate from // the to-do list itself so we don't modify it while iterating @@ -501,12 +503,41 @@ void analyze_scripts() when_funcs_to_do = new_to_do; new_to_do.clear(); } + } +static void analyze_scripts_for_ZAM(std::unique_ptr& pfs) + { + if ( analysis_options.usage_issues > 0 && + analysis_options.optimize_AST ) + { + fprintf(stderr, "warning: \"-O optimize-AST\" option is incompatible with -u option, deactivating optimization\n"); + analysis_options.optimize_AST = false; + } + + // Re-profile the functions, now without worrying about compatibility + // with compilation to C++. Note that the first profiling pass earlier + // may have marked some of the functions as to-skip, so first clear + // those markings. Once we have full compile-to-C++ and ZAM support + // for all Zeek language features, we can remove the re-profiling here. + for ( auto& f : funcs ) + f.SetSkip(false); + + pfs = std::make_unique(funcs, nullptr, true); + + // set of functions involved (directly or indirectly) in "when" + // clauses. + std::unordered_set when_funcs; + + find_when_funcs(pfs, when_funcs); + + bool report_recursive = analysis_options.report_recursive; std::unique_ptr inl; if ( analysis_options.inliner ) - inl = std::make_unique(funcs, analysis_options.report_recursive); + inl = std::make_unique(funcs, report_recursive); if ( ! analysis_options.activate ) + // Some --optimize options stop short of AST transformations, + // for development/debugging purposes. return; // The following tracks inlined functions that are also used @@ -515,6 +546,9 @@ void analyze_scripts() // since it won't be consulted in that case. std::unordered_set func_used_indirectly; + if ( global_stmts ) + func_used_indirectly.insert(global_stmts.get()); + if ( inl ) { for ( auto& f : funcs ) @@ -540,23 +574,67 @@ void analyze_scripts() { auto func = f.Func(); - if ( ! analysis_options.compile_all && - inl && inl->WasInlined(func) && - func_used_indirectly.count(func) == 0 ) - // No need to compile as it won't be - // called directly. - continue; + if ( analysis_options.only_func ) + { + if ( *analysis_options.only_func != func->Name() ) + continue; + } - if ( when_funcs.count(func) > 0 ) - // We don't try to compile these. + else if ( ! analysis_options.compile_all && + inl && inl->WasInlined(func) && + func_used_indirectly.count(func) == 0 ) + // No need to compile as it won't be called directly. continue; auto new_body = f.Body(); - optimize_func(func, f.ProfilePtr(), f.Scope(), new_body, - analysis_options); + optimize_func(func, f.ProfilePtr(), f.Scope(), new_body); f.SetBody(new_body); } } +void analyze_scripts() + { + static bool did_init = false; + + if ( ! did_init ) + { + init_options(); + did_init = true; + } + + if ( ! analysis_options.activate && ! analysis_options.inliner && + ! generating_CPP && ! analysis_options.report_CPP && + ! analysis_options.use_CPP ) + // No work to do, avoid profiling overhead. + return; + + // Now that everything's parsed and BiF's have been initialized, + // profile the functions. + auto pfs = std::make_unique(funcs, is_CPP_compilable, + false); + + if ( CPP_init_hook ) + (*CPP_init_hook)(); + + if ( analysis_options.report_CPP ) + { + report_CPP(); + exit(0); + } + + if ( analysis_options.use_CPP ) + use_CPP(); + + if ( generating_CPP ) + { + generate_CPP(pfs); + exit(0); + } + + // At this point we're done with C++ considerations, so instead + // are compiling to ZAM. + analyze_scripts_for_ZAM(pfs); + } + } // namespace zeek::detail diff --git a/src/script_opt/ScriptOpt.h b/src/script_opt/ScriptOpt.h index 5de3f0e507..f672044e1d 100644 --- a/src/script_opt/ScriptOpt.h +++ b/src/script_opt/ScriptOpt.h @@ -125,7 +125,11 @@ using ScriptFuncPtr = IntrusivePtr; // Info we need for tracking an instance of a function. class FuncInfo { public: - FuncInfo(ScriptFuncPtr func, ScopePtr scope, StmtPtr body, int priority); + FuncInfo(ScriptFuncPtr _func, ScopePtr _scope, StmtPtr _body, + int _priority) + : func(std::move(_func)), scope(std::move(_scope)), + body(std::move(_body)), priority(_priority) + {} ScriptFunc* Func() const { return func.get(); } const ScriptFuncPtr& FuncPtr() const { return func; } @@ -134,11 +138,11 @@ public: int Priority() const { return priority; } const ProfileFunc* Profile() const { return pf.get(); } std::shared_ptr ProfilePtr() const { return pf; } - const std::string& SaveFile() const { return save_file; } void SetBody(StmtPtr new_body) { body = std::move(new_body); } - void SetProfile(std::shared_ptr _pf); - void SetSaveFile(std::string _sf) { save_file = std::move(_sf); } + // void SetProfile(std::shared_ptr _pf); + void SetProfile(std::shared_ptr _pf) + { pf = std::move(_pf); } // The following provide a way of marking FuncInfo's as // should-be-skipped for script optimization, generally because @@ -156,10 +160,6 @@ protected: // Whether to skip optimizing this function. bool skip = false; - - // If we're saving this function in a file, this is the name - // of the file to use. - std::string save_file; }; diff --git a/src/script_opt/Stmt.cc b/src/script_opt/Stmt.cc index 892028f7fa..cb96bfa61f 100644 --- a/src/script_opt/Stmt.cc +++ b/src/script_opt/Stmt.cc @@ -8,6 +8,7 @@ #include "zeek/Reporter.h" #include "zeek/Desc.h" #include "zeek/Traverse.h" +#include "zeek/script_opt/IDOptInfo.h" #include "zeek/script_opt/Reduce.h" @@ -34,6 +35,8 @@ StmtPtr Stmt::Reduce(Reducer* c) return null; } + c->SetCurrStmt(this); + return DoReduce(c); } @@ -846,7 +849,9 @@ bool StmtList::ReduceStmt(int& s_i, StmtPList* f_stmts, Reducer* c) auto& s_i_succ = Stmts()[s_i + 1]; // Don't reduce s_i_succ. If it's what we're - // looking for, it's already reduced. + // looking for, it's already reduced. Plus + // that's what Reducer::MergeStmts (not that + // it really matters, per the comment there). auto merge = c->MergeStmts(var, rhs, s_i_succ); if ( merge ) { @@ -1014,10 +1019,19 @@ StmtPtr CatchReturnStmt::DoReduce(Reducer* c) return make_intrusive(); } - auto assign = make_intrusive(ret_var->Duplicate(), - ret_e->Duplicate(), + auto rv_dup = ret_var->Duplicate(); + auto ret_e_dup = ret_e->Duplicate(); + + auto assign = make_intrusive(rv_dup, ret_e_dup, false); assign_stmt = make_intrusive(assign); + + if ( ret_e_dup->Tag() == EXPR_CONST ) + { + auto c = ret_e_dup->AsConstExpr(); + rv_dup->AsNameExpr()->Id()->GetOptInfo()->SetConst(c); + } + return assign_stmt; } diff --git a/src/script_opt/TempVar.cc b/src/script_opt/TempVar.cc index b385bb432a..cd4924fed8 100644 --- a/src/script_opt/TempVar.cc +++ b/src/script_opt/TempVar.cc @@ -14,28 +14,15 @@ TempVar::TempVar(int num, const TypePtr& t, ExprPtr _rhs) : type(t) rhs = std::move(_rhs); } -void TempVar::SetAlias(IDPtr _alias, const DefPoints* _dps) +void TempVar::SetAlias(IDPtr _alias) { if ( alias ) reporter->InternalError("Re-aliasing a temporary"); - if ( ! _dps ) - { - printf("trying to alias %s to %s\n", name.c_str(), _alias->Name()); - reporter->InternalError("Empty dps for alias"); - } - if ( alias == id ) reporter->InternalError("Creating alias loop"); alias = std::move(_alias); - dps = _dps; - } - -void TempVar::SetDPs(const DefPoints* _dps) - { - ASSERT(_dps->length() == 1); - dps = _dps; } } // zeek::detail diff --git a/src/script_opt/TempVar.h b/src/script_opt/TempVar.h index 446327c1bb..4b6920eccc 100644 --- a/src/script_opt/TempVar.h +++ b/src/script_opt/TempVar.h @@ -9,6 +9,7 @@ #include "zeek/ID.h" #include "zeek/Expr.h" +#include "zeek/script_opt/IDOptInfo.h" #include "zeek/script_opt/ReachingDefs.h" namespace zeek::detail { @@ -22,21 +23,24 @@ public: const Expr* RHS() const { return rhs.get(); } IDPtr Id() const { return id; } - void SetID(IDPtr _id) { id = std::move(_id); } + void SetID(IDPtr _id) + { + id = std::move(_id); + id->GetOptInfo()->SetTemp(); + } void Deactivate() { active = false; } bool IsActive() const { return active; } // Associated constant expression, if any. - const ConstExpr* Const() const { return const_expr; } + const ConstExpr* Const() const { return id->GetOptInfo()->Const(); } // The most use of "const" in any single line in the Zeek // codebase :-P ... though only by one! - void SetConst(const ConstExpr* _const) { const_expr = _const; } + void SetConst(const ConstExpr* _const) + { id->GetOptInfo()->SetConst(_const); } IDPtr Alias() const { return alias; } - const DefPoints* DPs() const { return dps; } - void SetAlias(IDPtr id, const DefPoints* dps); - void SetDPs(const DefPoints* _dps); + void SetAlias(IDPtr id); const RDPtr& MaxRDs() const { return max_rds; } void SetMaxRDs(RDPtr rds) { max_rds = std::move(rds); } @@ -47,9 +51,7 @@ protected: const TypePtr& type; ExprPtr rhs; bool active = true; - const ConstExpr* const_expr = nullptr; IDPtr alias; - const DefPoints* dps = nullptr; RDPtr max_rds; };