diff --git a/CHANGES b/CHANGES index 17981a9140..1e03a3366c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +8.1.0-dev.570 | 2025-09-23 09:05:53 -0700 + + * BTests & baselines for testing selective skipping of script optimization (Vern Paxson, Corelight) + + * added &no_ZAM_opt/&no_CPP_opt attributes and --no-opt-files/--no-opt-funcs for controlling skipping script optimization (Vern Paxson, Corelight) + 8.1.0-dev.567 | 2025-09-23 13:07:24 +0200 * GH-4842: utils/decompose_uri: Support URIs containing IPv6 addresses (Arne Welzel, Corelight) diff --git a/VERSION b/VERSION index 4848eb42ce..9177c9a608 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.1.0-dev.567 +8.1.0-dev.570 diff --git a/src/Attr.cc b/src/Attr.cc index 9a8aa3396a..0eaa3748b9 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -41,6 +41,8 @@ const char* attr_name(AttrTag t) { "&is_assigned", "&is_used", "&ordered", + "&no_ZAM_opt", + "&no_CPP_opt", }; // clang-format on @@ -194,9 +196,19 @@ void Attributes::AddAttr(AttrPtr attr, bool is_redef) { return true; } - return new_tag == ATTR_LOG || new_tag == ATTR_OPTIONAL || new_tag == ATTR_REDEF || - new_tag == ATTR_BROKER_STORE_ALLOW_COMPLEX || new_tag == ATTR_RAW_OUTPUT || - new_tag == ATTR_ERROR_HANDLER || new_tag == ATTR_IS_USED; + static const std::set acceptable = { + ATTR_BROKER_STORE_ALLOW_COMPLEX, + ATTR_ERROR_HANDLER, + ATTR_IS_USED, + ATTR_LOG, + ATTR_NO_CPP_OPT, + ATTR_NO_ZAM_OPT, + ATTR_OPTIONAL, + ATTR_RAW_OUTPUT, + ATTR_REDEF, + }; + + return acceptable.contains(new_tag); }; // A `redef` is allowed to overwrite an existing attribute instead of @@ -217,7 +229,7 @@ void Attributes::AddAttr(AttrPtr attr, bool is_redef) { // that's a signal to skip the checking. If the type is error, // there's no point checking attributes either. if ( type && ! IsErrorType(type->Tag()) ) { - if ( ! CheckAttr(attr.get()) ) { + if ( ! CheckAttr(attr.get(), type) ) { // Get rid of it, so we don't get error cascades down the line. RemoveAttr(attr->Tag()); return; @@ -285,7 +297,7 @@ void Attributes::DescribeReST(ODesc* d, bool shorten) const { } } -bool Attributes::CheckAttr(Attr* a) { +bool Attributes::CheckAttr(Attr* a, const TypePtr& attrs_t) { switch ( a->Tag() ) { case ATTR_DEPRECATED: case ATTR_REDEF: @@ -541,6 +553,15 @@ bool Attributes::CheckAttr(Attr* a) { return AttrError("&ordered only applicable to tables"); break; + case ATTR_NO_ZAM_OPT: + case ATTR_NO_CPP_OPT: { + if ( attrs_t->Tag() != TYPE_FUNC ) { + bool is_no_zam = a->Tag() == ATTR_NO_ZAM_OPT; + Error(util::fmt("&no_%s_opt must apply to a function", is_no_zam ? "ZAM" : "CPP"), attrs_t.get()); + return false; + } + } break; + default: BadTag("Attributes::CheckAttr", attr_name(a->Tag())); } diff --git a/src/Attr.h b/src/Attr.h index d1fefa6f5a..744f68fb7b 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -50,6 +50,8 @@ enum AttrTag : uint8_t { ATTR_IS_ASSIGNED, // to suppress usage warnings ATTR_IS_USED, // to suppress usage warnings ATTR_ORDERED, // used to store tables in ordered mode + ATTR_NO_ZAM_OPT, // avoid ZAM optimization + ATTR_NO_CPP_OPT, // avoid -O gen-C++ optimization NUM_ATTRS // this item should always be last }; @@ -138,7 +140,7 @@ public: protected: // Returns true if the attribute is okay, false if not. - bool CheckAttr(Attr* attr); + bool CheckAttr(Attr* attr, const TypePtr& attrs_t); // Reports an attribute error and returns false (handy for CheckAttr()). bool AttrError(const char* msg); diff --git a/src/Func.cc b/src/Func.cc index ea86e31a68..e6047df71e 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -808,7 +808,10 @@ static int get_func_priority(const std::vector& attrs) { int priority = 0; for ( const auto& a : attrs ) { - if ( a->Tag() == ATTR_DEPRECATED || a->Tag() == ATTR_IS_USED || a->Tag() == ATTR_GROUP ) + static const std::set ok_for_func = { + ATTR_DEPRECATED, ATTR_GROUP, ATTR_IS_USED, ATTR_NO_CPP_OPT, ATTR_NO_ZAM_OPT, + }; + if ( ok_for_func.contains(a->Tag()) ) continue; if ( a->Tag() != ATTR_PRIORITY ) { @@ -879,12 +882,16 @@ FunctionIngredients::FunctionIngredients(ScopePtr _scope, StmtPtr _body, const s groups = get_func_groups(*attrs); - for ( const auto& a : *attrs ) - if ( a->Tag() == ATTR_IS_USED ) { + for ( const auto& a : *attrs ) { + static const std::set assoc_with_id = { + ATTR_IS_USED, + ATTR_NO_CPP_OPT, + ATTR_NO_ZAM_OPT, + }; + if ( assoc_with_id.contains(a->Tag()) ) // Associate this with the identifier, too. - id->AddAttr(make_intrusive(ATTR_IS_USED)); - break; - } + id->AddAttr(make_intrusive(a->Tag())); + } } else priority = 0; diff --git a/src/Options.cc b/src/Options.cc index 5f4494977c..ef771cc984 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -124,9 +124,15 @@ void usage(const char* prog) { printf( " -0|--optimize-files= | enable script optimization for all " "functions in files with names containing the given pattern\n"); + printf( + " --no-optimize-files= | skip script optimization for all " + "functions in files with names containing the given pattern\n"); printf( " -o|--optimize-funcs= | enable script optimization for " "functions with names fully matching the given pattern\n"); + printf( + " --no-optimize-funcs= | skip script optimization for " + "functions with names fully matching the given pattern\n"); printf(" -P|--prime-dns | prime DNS\n"); printf(" -Q|--time | print execution time summary to stderr\n"); printf(" -S|--debug-rules | enable rule debugging\n"); @@ -355,6 +361,8 @@ Options parse_cmdline(int argc, char** argv) { int profile_script_call_stacks = 0; std::string profile_filename; int no_unused_warnings = 0; + int no_optimize_files = 0; + int no_optimize_funcs = 0; bool enable_script_profile = false; bool enable_script_profile_call_stacks = false; @@ -393,6 +401,8 @@ Options parse_cmdline(int argc, char** argv) { {"re-level", required_argument, nullptr, 'T'}, {"watchdog", no_argument, nullptr, 'W'}, {"print-id", required_argument, nullptr, 'I'}, + {"no-optimize-funcs", required_argument, &no_optimize_funcs, 1}, + {"no-optimize-files", required_argument, &no_optimize_files, 1}, {"status-file", required_argument, nullptr, 'U'}, {"debug", required_argument, nullptr, 'B'}, @@ -512,8 +522,8 @@ Options parse_cmdline(int argc, char** argv) { case 'I': rval.identifier_to_print = optarg; break; case 'N': ++rval.print_plugins; break; case 'O': set_analysis_option(optarg, rval); break; - case 'o': add_func_analysis_pattern(rval.analysis_options, optarg); break; - case '0': add_file_analysis_pattern(rval.analysis_options, optarg); break; + case 'o': add_func_analysis_pattern(rval.analysis_options, optarg, true); break; + case '0': add_file_analysis_pattern(rval.analysis_options, optarg, true); break; case 'P': if ( rval.dns_mode != detail::DNS_DEFAULT ) { fprintf(stderr, "ERROR: can only change DNS manager mode once\n"); @@ -562,6 +572,13 @@ Options parse_cmdline(int argc, char** argv) { if ( no_unused_warnings ) rval.no_unused_warnings = true; + + if ( no_optimize_files ) + add_file_analysis_pattern(rval.analysis_options, optarg, false); + + if ( no_optimize_funcs ) + add_func_analysis_pattern(rval.analysis_options, optarg, false); + break; case '?': diff --git a/src/parse.y b/src/parse.y index 7f77e8484e..be4ad5c1d4 100644 --- a/src/parse.y +++ b/src/parse.y @@ -5,7 +5,7 @@ // Switching parser table type fixes ambiguity problems. %define lr.type ielr -%expect 217 +%expect 229 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY TOK_ASSERT %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -31,6 +31,7 @@ %token TOK_ATTR_PRIORITY TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER TOK_ATTR_GROUP %token TOK_ATTR_TYPE_COLUMN TOK_ATTR_DEPRECATED %token TOK_ATTR_IS_ASSIGNED TOK_ATTR_IS_USED TOK_ATTR_ORDERED +%token TOK_ATTR_NO_ZAM_OPT TOK_ATTR_NO_CPP_OPT %token TOK_DEBUG @@ -52,7 +53,7 @@ %left '*' '/' '%' %left TOK_INCR TOK_DECR %right '!' '~' -%left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR +%left '$' '[' ']' '(' ')' TOK_HAS_FIELD %nonassoc TOK_AS TOK_IS %type opt_no_test opt_no_test_block opt_deep when_flavor @@ -1793,6 +1794,10 @@ attr: } | TOK_ATTR_ORDERED { $$ = new Attr(ATTR_ORDERED); } + | TOK_ATTR_NO_ZAM_OPT + { $$ = new Attr(ATTR_NO_ZAM_OPT); } + | TOK_ATTR_NO_CPP_OPT + { $$ = new Attr(ATTR_NO_CPP_OPT); } ; stmt: diff --git a/src/scan.l b/src/scan.l index 2abe6a98b8..f93dfc5d79 100644 --- a/src/scan.l +++ b/src/scan.l @@ -394,6 +394,8 @@ when return TOK_WHEN; &broker_allow_complex_type return TOK_ATTR_BROKER_STORE_ALLOW_COMPLEX; &backend return TOK_ATTR_BACKEND; &ordered return TOK_ATTR_ORDERED; +&no_ZAM_opt return TOK_ATTR_NO_ZAM_OPT; +&no_CPP_opt return TOK_ATTR_NO_CPP_OPT; @deprecated.* { auto num_files = file_stack.length(); diff --git a/src/script_opt/CPP/Driver.cc b/src/script_opt/CPP/Driver.cc index cd9f82e3f7..b7cfd4a625 100644 --- a/src/script_opt/CPP/Driver.cc +++ b/src/script_opt/CPP/Driver.cc @@ -51,11 +51,11 @@ void CPPCompile::Compile(bool report_uncompilable) { reporter->FatalError("aborting standalone compilation to C++ due to having to skip some functions"); for ( auto& g : global_scope()->OrderedVars() ) { - bool compiled_global = obj_matches_opt_files(g); + bool compiled_global = obj_matches_opt_files(g) == AnalyzeDecision::SHOULD; if ( ! compiled_global ) for ( const auto& i_e : g->GetOptInfo()->GetInitExprs() ) - if ( obj_matches_opt_files(i_e) ) { + if ( obj_matches_opt_files(i_e) == AnalyzeDecision::SHOULD ) { compiled_global = true; break; } @@ -87,11 +87,11 @@ void CPPCompile::Compile(bool report_uncompilable) { } for ( auto& l : pfs->Lambdas() ) - if ( obj_matches_opt_files(l) ) + if ( obj_matches_opt_files(l) == AnalyzeDecision::SHOULD ) accessed_lambdas.insert(l); for ( auto& ea : pfs->ExprAttrs() ) - if ( obj_matches_opt_files(ea.first) ) { + if ( obj_matches_opt_files(ea.first) == AnalyzeDecision::SHOULD ) { auto& attr = ea.first; attrs.insert(attr); auto& t = attr->GetExpr()->GetType(); @@ -184,11 +184,11 @@ void CPPCompile::Compile(bool report_uncompilable) { // to generate, make sure we track their attributes. for ( const auto& fd : field_decls ) { auto td = fd.second; - if ( obj_matches_opt_files(td->type) ) { + if ( obj_matches_opt_files(td->type) == AnalyzeDecision::SHOULD ) { TypePtr tp = {NewRef{}, const_cast(TypeRep(td->type))}; RegisterType(tp); } - if ( obj_matches_opt_files(td->attrs) ) + if ( obj_matches_opt_files(td->attrs) == AnalyzeDecision::SHOULD ) RegisterAttributes(td->attrs); } @@ -202,6 +202,19 @@ bool CPPCompile::AnalyzeFuncBody(FuncInfo& fi, unordered_set& filenames_ string fn = body->GetLocationInfo()->FileName(); + if ( fi.ShouldAnalyze() && fi.ShouldSkip() ) { + const char* reason = nullptr; + auto is_compilable = is_CPP_compilable(fi.Profile(), &reason); + ASSERT(! is_compilable); + ASSERT(reason); + if ( standalone || report_uncompilable ) + reporter->Warning("%s cannot be compiled to C++ due to %s", f->GetName().c_str(), reason); + + fi.SetSkip(true); + if ( standalone ) + return false; + } + if ( ! analysis_options.allow_cond && ! fi.ShouldSkip() ) { if ( ! analysis_options.only_files.empty() && files_with_conditionals.contains(fn) ) { if ( report_uncompilable ) diff --git a/src/script_opt/CPP/Exprs.cc b/src/script_opt/CPP/Exprs.cc index 0f9fbc3cde..9cacd2506b 100644 --- a/src/script_opt/CPP/Exprs.cc +++ b/src/script_opt/CPP/Exprs.cc @@ -1278,7 +1278,7 @@ string CPPCompile::GenEnum(const TypePtr& t, const ValPtr& ev) { mapping_slot = num_ev_mappings++; string enum_name = et->Lookup(v); - bool create_if_missing = standalone && obj_matches_opt_files(ev); + bool create_if_missing = standalone && obj_matches_opt_files(ev) == AnalyzeDecision::SHOULD; enum_names.emplace_back(EnumMappingInfo{TypeOffset(t), std::move(enum_name), create_if_missing}); if ( evm != enum_val_mappings.end() ) { diff --git a/src/script_opt/CPP/Inits.cc b/src/script_opt/CPP/Inits.cc index ccf742a373..e828986beb 100644 --- a/src/script_opt/CPP/Inits.cc +++ b/src/script_opt/CPP/Inits.cc @@ -115,7 +115,7 @@ void CPPCompile::InitializeFieldMappings() { // We can assess whether this field is one we need to generate // because if it is, it will have an &optional attribute that // is local to one of the cmopiled source files. - if ( td->attrs && obj_matches_opt_files(td->attrs) ) { + if ( td->attrs && obj_matches_opt_files(td->attrs) == AnalyzeDecision::SHOULD ) { type_arg = Fmt(TypeOffset(td->type)); attrs_arg = Fmt(AttributesOffset(td->attrs)); } @@ -197,7 +197,7 @@ void CPPCompile::InitializeGlobals() { for ( const auto& ginit : IDOptInfo::GetGlobalInitExprs() ) { IDPtr g{NewRef{}, const_cast(ginit.Id())}; - if ( ! ofiles.empty() && ! obj_matches_opt_files(g) ) + if ( ! ofiles.empty() && obj_matches_opt_files(g) != AnalyzeDecision::SHOULD ) continue; if ( ! accessed_globals.contains(g) ) diff --git a/src/script_opt/CPP/InitsInfo.cc b/src/script_opt/CPP/InitsInfo.cc index 743bbf256c..8b14045c40 100644 --- a/src/script_opt/CPP/InitsInfo.cc +++ b/src/script_opt/CPP/InitsInfo.cc @@ -577,7 +577,7 @@ RecordTypeInfo::RecordTypeInfo(CPPCompile* _c, TypePtr _t, int _addl_fields) field_types.push_back(r_i->type); - if ( r_i->attrs && c->TargetingStandalone() && obj_matches_opt_files(r_i->attrs) ) { + if ( r_i->attrs && c->TargetingStandalone() && obj_matches_opt_files(r_i->attrs) == AnalyzeDecision::SHOULD ) { gi = c->RegisterAttributes(r_i->attrs); final_init_cohort = max(final_init_cohort, gi->InitCohort() + 1); field_attrs.push_back(gi->Offset()); diff --git a/src/script_opt/CPP/Types.cc b/src/script_opt/CPP/Types.cc index eb8ccc3f7c..10f2e7a838 100644 --- a/src/script_opt/CPP/Types.cc +++ b/src/script_opt/CPP/Types.cc @@ -198,14 +198,14 @@ shared_ptr CPPCompile::RegisterType(const TypePtr& tp) { // If any of those conditions don't hold, then this variable will remain 0. int addl_fields = 0; - bool type_init_needed = standalone && obj_matches_opt_files(tp); + bool type_init_needed = standalone && obj_matches_opt_files(tp) == AnalyzeDecision::SHOULD; if ( standalone && ! type_init_needed ) { if ( tp->Tag() == TYPE_RECORD ) { auto tr = tp->AsRecordType(); for ( auto i = tr->NumOrigFields(); i < tr->NumFields(); ++i ) { auto fd = tr->FieldDecl(i); - if ( filename_matches_opt_files(fd->GetLocationInfo()->FileName()) ) { + if ( filename_matches_opt_files(fd->GetLocationInfo()->FileName()) == AnalyzeDecision::SHOULD ) { if ( addl_fields == 0 ) addl_fields = i; } diff --git a/src/script_opt/CPP/Util.cc b/src/script_opt/CPP/Util.cc index 84e1846151..5b64b30544 100644 --- a/src/script_opt/CPP/Util.cc +++ b/src/script_opt/CPP/Util.cc @@ -39,6 +39,17 @@ string scope_prefix(const string& scope) { return "zeek::detail::CPP_" + scope; string scope_prefix(int scope) { return scope_prefix(to_string(scope)); } bool is_CPP_compilable(const ProfileFunc* pf, const char** reason) { + auto func = pf->ProfiledFunc(); // can be nil for lambdas + + if ( func ) { + auto& scope_id = pf->ProfiledScope()->GetID(); + if ( scope_id && scope_id->GetAttr(ATTR_NO_CPP_OPT) ) { + if ( reason ) + *reason = "&no_CPP_opt attribute"; + return false; + } + } + if ( has_AST_node_unknown_to_script_opt(pf, false) ) { if ( reason ) *reason = "unknown AST node type"; diff --git a/src/script_opt/CPP/Vars.cc b/src/script_opt/CPP/Vars.cc index cc3e9613b3..bc00b96970 100644 --- a/src/script_opt/CPP/Vars.cc +++ b/src/script_opt/CPP/Vars.cc @@ -76,7 +76,7 @@ std::shared_ptr CPPCompile::GenerateGlobalInit(IDPtr g) { if ( ! standalone ) return make_shared(this, g, globals[gn]); - if ( obj_matches_opt_files(g) ) + if ( obj_matches_opt_files(g) == AnalyzeDecision::SHOULD ) return make_shared(this, g, globals[gn]); // It's not a global that's created by the scripts we're compiling, @@ -86,7 +86,7 @@ std::shared_ptr CPPCompile::GenerateGlobalInit(IDPtr g) { bool needs_redef = false; for ( const auto& i_e : g->GetOptInfo()->GetInitExprs() ) - if ( obj_matches_opt_files(i_e) ) { + if ( obj_matches_opt_files(i_e) == AnalyzeDecision::SHOULD ) { needs_redef = true; break; } diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index 257cb5fbd9..16fcfdab52 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -57,7 +57,7 @@ bool is_lambda(const ScriptFunc* f) { return lambdas.contains(f); } bool is_when_lambda(const ScriptFunc* f) { return when_lambdas.contains(f); } void analyze_global_stmts(Stmt* stmts) { - if ( analysis_options.gen_standalone_CPP && obj_matches_opt_files(stmts) ) + if ( analysis_options.gen_standalone_CPP && obj_matches_opt_files(stmts) == AnalyzeDecision::SHOULD ) reporter->FatalError("cannot include global statements with -O gen-standalone-C++: %s", obj_desc(stmts).c_str()); @@ -88,19 +88,25 @@ std::pair get_global_stmts() { return std::pair{fi.Body(), fi.Scope()}; } -void add_func_analysis_pattern(AnalyOpt& opts, const char* pat) { +void add_func_analysis_pattern(AnalyOpt& opts, const char* pat, bool is_only) { try { std::string full_pat = std::string("^(") + pat + ")$"; - opts.only_funcs.emplace_back(std::move(full_pat)); + if ( is_only ) + opts.only_funcs.emplace_back(std::move(full_pat)); + else + opts.skip_funcs.emplace_back(std::move(full_pat)); } catch ( const std::regex_error& e ) { reporter->FatalError("bad file analysis pattern: %s", pat); } } -void add_file_analysis_pattern(AnalyOpt& opts, const char* pat) { +void add_file_analysis_pattern(AnalyOpt& opts, const char* pat, bool is_only) { try { std::string full_pat = std::string("^.*(") + pat + ").*$"; - opts.only_files.emplace_back(std::move(full_pat)); + if ( is_only ) + opts.only_files.emplace_back(std::move(full_pat)); + else + opts.skip_files.emplace_back(std::move(full_pat)); } catch ( const std::regex_error& e ) { reporter->FatalError("bad file analysis pattern: %s", pat); } @@ -108,39 +114,66 @@ void add_file_analysis_pattern(AnalyOpt& opts, const char* pat) { bool should_analyze(const ScriptFuncPtr& f, const StmtPtr& body) { auto& ofuncs = analysis_options.only_funcs; + auto& sfuncs = analysis_options.skip_funcs; auto& ofiles = analysis_options.only_files; + auto& sfiles = analysis_options.skip_files; - if ( ofiles.empty() && ofuncs.empty() ) + bool have_onlies = ! ofiles.empty() || ! ofuncs.empty(); + + if ( ! have_onlies && sfiles.empty() && sfuncs.empty() ) + // It's the default of compile-everything. return true; - if ( obj_matches_opt_files(body.get()) ) - return true; + auto file_decision = obj_matches_opt_files(body.get()); + if ( file_decision == AnalyzeDecision::SHOULD_NOT ) + return false; + + // Even if the file decision is SHOULD, that can be overridden by + // a function decision of "skip". const auto& fun = f->GetName(); + for ( auto& s : sfuncs ) + if ( std::regex_match(fun, s) ) + return false; // matches a "skip" function + + if ( file_decision == AnalyzeDecision::SHOULD ) + // It matches a specified file, and there's no "skip" for the function. + return true; for ( auto& o : ofuncs ) if ( std::regex_match(fun, o) ) - return true; + return true; // matches an "only" function - return false; + // If we get here, neither the file nor the function has an "only" + // or "skip" decision. If our sole directives were for skip's, then + // we should analyze this function. If we have any only's, then we + // shouldn't. + return ! have_onlies; } -bool filename_matches_opt_files(const char* filename) { +AnalyzeDecision filename_matches_opt_files(const char* filename) { auto& ofiles = analysis_options.only_files; + auto& sfiles = analysis_options.skip_files; - if ( ofiles.empty() ) - return false; + if ( ofiles.empty() && sfiles.empty() ) + return AnalyzeDecision::DEFAULT; auto fin = util::detail::normalize_path(filename); + for ( auto& s : analysis_options.skip_files ) + if ( std::regex_match(fin, s) ) + return AnalyzeDecision::SHOULD_NOT; + for ( auto& o : ofiles ) if ( std::regex_match(fin, o) ) - return true; + return AnalyzeDecision::SHOULD; - return false; + return AnalyzeDecision::DEFAULT; } -bool obj_matches_opt_files(const Obj* obj) { return filename_matches_opt_files(obj->GetLocationInfo()->FileName()); } +AnalyzeDecision obj_matches_opt_files(const Obj* obj) { + return filename_matches_opt_files(obj->GetLocationInfo()->FileName()); +} static bool optimize_AST(ScriptFuncPtr f, std::shared_ptr& pf, std::shared_ptr& rc, ScopePtr scope, StmtPtr& body) { @@ -329,13 +362,25 @@ static void init_options() { if ( analysis_options.only_funcs.empty() ) { auto zo = getenv("ZEEK_OPT_FUNCS"); if ( zo ) - add_func_analysis_pattern(analysis_options, zo); + add_func_analysis_pattern(analysis_options, zo, true); + } + + if ( analysis_options.skip_funcs.empty() ) { + auto zo = getenv("ZEEK_SKIP_FUNCS"); + if ( zo ) + add_func_analysis_pattern(analysis_options, zo, false); } if ( analysis_options.only_files.empty() ) { auto zo = getenv("ZEEK_OPT_FILES"); if ( zo ) - add_file_analysis_pattern(analysis_options, zo); + add_file_analysis_pattern(analysis_options, zo, true); + } + + if ( analysis_options.skip_files.empty() ) { + auto zo = getenv("ZEEK_SKIP_FILES"); + if ( zo ) + add_file_analysis_pattern(analysis_options, zo, false); } if ( analysis_options.profile_ZAM ) { diff --git a/src/script_opt/ScriptOpt.h b/src/script_opt/ScriptOpt.h index f7a6169cc2..013fa9f16d 100644 --- a/src/script_opt/ScriptOpt.h +++ b/src/script_opt/ScriptOpt.h @@ -34,6 +34,11 @@ struct AnalyOpt { // Same, but for the filenames where the function is found. std::vector only_files; + // The inverses of those - functions and files to skip. These + // have higher precedence than the only_'s. + std::vector skip_funcs; + std::vector skip_files; + // For a given compilation target, report functions that can't // be compiled. bool report_uncompilable = false; @@ -245,21 +250,23 @@ extern std::pair get_global_stmts(); // Used to associate module names with profiling information. extern void switch_to_module(const char* module); -// Add a pattern to the "only_funcs" list. -extern void add_func_analysis_pattern(AnalyOpt& opts, const char* pat); +// Add a pattern to the "only_funcs" (if is_only true) or "skip_funcs" list. +extern void add_func_analysis_pattern(AnalyOpt& opts, const char* pat, bool is_only); -// Add a pattern to the "only_files" list. -extern void add_file_analysis_pattern(AnalyOpt& opts, const char* pat); +// Add a pattern to the "only_files" / "skip_files" list. +extern void add_file_analysis_pattern(AnalyOpt& opts, const char* pat, bool is_only); // True if the given script function & body should be analyzed; otherwise // it should be skipped. extern bool should_analyze(const ScriptFuncPtr& f, const StmtPtr& body); -// True if the given filename or object location matches one specified by -// --optimize-files=... -extern bool filename_matches_opt_files(const char* filename); -extern bool obj_matches_opt_files(const Obj* obj); -inline bool obj_matches_opt_files(const ObjPtr& obj) { return obj_matches_opt_files(obj.get()); } +// SHOULD if the given filename or object location matches one specified by +// --optimize-files=..., SHOULD_NOT if it matches one specified by +// --no-opt-files=... (which takes precedence), DEFAULT if neither. +enum class AnalyzeDecision : uint8_t { SHOULD, SHOULD_NOT, DEFAULT }; +extern AnalyzeDecision filename_matches_opt_files(const char* filename); +extern AnalyzeDecision obj_matches_opt_files(const Obj* obj); +inline auto obj_matches_opt_files(const ObjPtr& obj) { return obj_matches_opt_files(obj.get()); } // Analyze all of the parsed scripts collectively for usage issues (unless // suppressed by the flag) and optimization. diff --git a/src/script_opt/ZAM/Support.cc b/src/script_opt/ZAM/Support.cc index 3c606335cb..6dee876bfc 100644 --- a/src/script_opt/ZAM/Support.cc +++ b/src/script_opt/ZAM/Support.cc @@ -117,6 +117,17 @@ bool file_mgr_set_reassembly_buffer(StringVal* file_id, uint64_t max) { bool ZAM_error = false; bool is_ZAM_compilable(const ProfileFunc* pf, const char** reason) { + auto func = pf->ProfiledFunc(); // can be nil for lambdas + + if ( func ) { + auto& scope_id = pf->ProfiledScope()->GetID(); + if ( scope_id && scope_id->GetAttr(ATTR_NO_ZAM_OPT) ) { + if ( reason ) + *reason = "&no_ZAM_opt attribute"; + return false; + } + } + if ( has_AST_node_unknown_to_script_opt(pf, true) ) { if ( reason ) *reason = "unknown AST node type"; diff --git a/testing/btest/Baseline.cpp/language.skip-opt1/out b/testing/btest/Baseline.cpp/language.skip-opt1/out new file mode 100644 index 0000000000..03947e1784 --- /dev/null +++ b/testing/btest/Baseline.cpp/language.skip-opt1/out @@ -0,0 +1,9 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +always: function() : void +compiled-C++ +no_ZAM: function() : void +compiled-C++ +no_CPP: function() : void +{ +print no CPP; +} diff --git a/testing/btest/Baseline.cpp/language.skip-opt2/out b/testing/btest/Baseline.cpp/language.skip-opt2/out new file mode 100644 index 0000000000..b44c826a88 --- /dev/null +++ b/testing/btest/Baseline.cpp/language.skip-opt2/out @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +always: function() : void +{ +print always; +} +no_ZAM: function() : void +compiled-C++ +no_CPP: function() : void +{ +print no CPP; +} diff --git a/testing/btest/Baseline.zam/language.skip-opt1/out b/testing/btest/Baseline.zam/language.skip-opt1/out new file mode 100644 index 0000000000..aa133df5ad --- /dev/null +++ b/testing/btest/Baseline.zam/language.skip-opt1/out @@ -0,0 +1,9 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +always: function() : void +ZAM-code always +no_ZAM: function() : void +{ +print no ZAM; +} +no_CPP: function() : void +ZAM-code no_CPP diff --git a/testing/btest/Baseline.zam/language.skip-opt2/out b/testing/btest/Baseline.zam/language.skip-opt2/out new file mode 100644 index 0000000000..90533a197a --- /dev/null +++ b/testing/btest/Baseline.zam/language.skip-opt2/out @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +always: function() : void +{ +print always; +} +no_ZAM: function() : void +{ +print no ZAM; +} +no_CPP: function() : void +ZAM-code no_CPP diff --git a/testing/btest/Baseline/language.skip-opt1/out b/testing/btest/Baseline/language.skip-opt1/out new file mode 100644 index 0000000000..5425ec1afc --- /dev/null +++ b/testing/btest/Baseline/language.skip-opt1/out @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +always: function() : void +{ +print always; +} +no_ZAM: function() : void +{ +print no ZAM; +} +no_CPP: function() : void +{ +print no CPP; +} diff --git a/testing/btest/Baseline/language.skip-opt2/out b/testing/btest/Baseline/language.skip-opt2/out new file mode 100644 index 0000000000..5425ec1afc --- /dev/null +++ b/testing/btest/Baseline/language.skip-opt2/out @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +always: function() : void +{ +print always; +} +no_ZAM: function() : void +{ +print no ZAM; +} +no_CPP: function() : void +{ +print no CPP; +} diff --git a/testing/btest/Baseline/language.skip-opt3/out b/testing/btest/Baseline/language.skip-opt3/out new file mode 100644 index 0000000000..5425ec1afc --- /dev/null +++ b/testing/btest/Baseline/language.skip-opt3/out @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +always: function() : void +{ +print always; +} +no_ZAM: function() : void +{ +print no ZAM; +} +no_CPP: function() : void +{ +print no CPP; +} diff --git a/testing/btest/Baseline/opt.opt-skip-files/output b/testing/btest/Baseline/opt.opt-skip-files/output new file mode 100644 index 0000000000..6a70430a5a --- /dev/null +++ b/testing/btest/Baseline/opt.opt-skip-files/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +I shouldn't be ZAM code! +my_test: function() : void +{ +print I shouldn't be ZAM code!; +} diff --git a/testing/btest/Baseline/opt.opt-skip-funcs/out1 b/testing/btest/Baseline/opt.opt-skip-funcs/out1 new file mode 100644 index 0000000000..6a70430a5a --- /dev/null +++ b/testing/btest/Baseline/opt.opt-skip-funcs/out1 @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +I shouldn't be ZAM code! +my_test: function() : void +{ +print I shouldn't be ZAM code!; +} diff --git a/testing/btest/Baseline/opt.opt-skip-funcs/out2 b/testing/btest/Baseline/opt.opt-skip-funcs/out2 new file mode 100644 index 0000000000..6a70430a5a --- /dev/null +++ b/testing/btest/Baseline/opt.opt-skip-funcs/out2 @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +I shouldn't be ZAM code! +my_test: function() : void +{ +print I shouldn't be ZAM code!; +} diff --git a/testing/btest/Baseline/opt.opt-skip-other/out1 b/testing/btest/Baseline/opt.opt-skip-other/out1 new file mode 100644 index 0000000000..d2e95de646 --- /dev/null +++ b/testing/btest/Baseline/opt.opt-skip-other/out1 @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +I should be ZAM code! +my_test: function() : void +ZAM-code my_test diff --git a/testing/btest/Baseline/opt.opt-skip-other/out2 b/testing/btest/Baseline/opt.opt-skip-other/out2 new file mode 100644 index 0000000000..d2e95de646 --- /dev/null +++ b/testing/btest/Baseline/opt.opt-skip-other/out2 @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +I should be ZAM code! +my_test: function() : void +ZAM-code my_test diff --git a/testing/btest/language/skip-opt1.zeek b/testing/btest/language/skip-opt1.zeek new file mode 100644 index 0000000000..4d3866048a --- /dev/null +++ b/testing/btest/language/skip-opt1.zeek @@ -0,0 +1,24 @@ +# @TEST-EXEC: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: btest-diff out + +function always() + { + print "always"; + } + +function no_ZAM() &no_ZAM_opt + { + print "no ZAM"; + } + +function no_CPP() &no_CPP_opt + { + print "no CPP"; + } + +event zeek_init() + { + print always; + print no_ZAM; + print no_CPP; + } diff --git a/testing/btest/language/skip-opt2.zeek b/testing/btest/language/skip-opt2.zeek new file mode 100644 index 0000000000..e7f9ecf158 --- /dev/null +++ b/testing/btest/language/skip-opt2.zeek @@ -0,0 +1,24 @@ +# @TEST-EXEC: zeek -b %INPUT --no-optimize-func=always >out 2>&1 +# @TEST-EXEC: btest-diff out + +function always() + { + print "always"; + } + +function no_ZAM() &no_ZAM_opt + { + print "no ZAM"; + } + +function no_CPP() &no_CPP_opt + { + print "no CPP"; + } + +event zeek_init() + { + print always; + print no_ZAM; + print no_CPP; + } diff --git a/testing/btest/language/skip-opt3.zeek b/testing/btest/language/skip-opt3.zeek new file mode 100644 index 0000000000..f1884cda21 --- /dev/null +++ b/testing/btest/language/skip-opt3.zeek @@ -0,0 +1,24 @@ +# @TEST-EXEC: zeek -b %INPUT --no-optimize-file=skip-opt >out 2>&1 +# @TEST-EXEC: btest-diff out + +function always() + { + print "always"; + } + +function no_ZAM() &no_ZAM_opt + { + print "no ZAM"; + } + +function no_CPP() &no_CPP_opt + { + print "no CPP"; + } + +event zeek_init() + { + print always; + print no_ZAM; + print no_CPP; + } diff --git a/testing/btest/opt/opt-files.zeek b/testing/btest/opt/opt-files.zeek index c2413bf59c..9f06cf7e68 100644 --- a/testing/btest/opt/opt-files.zeek +++ b/testing/btest/opt/opt-files.zeek @@ -1,5 +1,5 @@ # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" -# @TEST-EXEC: zeek -b -O ZAM --optimize-files='opt-files' %INPUT >output +# @TEST-EXEC: zeek -b -O ZAM --optimize-files=opt-files %INPUT >output # @TEST-EXEC: btest-diff output # Tests that we can selectively pick this file. diff --git a/testing/btest/opt/opt-files2.zeek b/testing/btest/opt/opt-files2.zeek index c2cb5852f9..43ea8bb5de 100644 --- a/testing/btest/opt/opt-files2.zeek +++ b/testing/btest/opt/opt-files2.zeek @@ -1,5 +1,5 @@ # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" -# @TEST-EXEC: zeek -b -O ZAM --optimize-files='base/utils' %INPUT >output +# @TEST-EXEC: zeek -b -O ZAM --optimize-files=base/utils %INPUT >output # @TEST-EXEC: btest-diff output # Tests that we can selectively pick a group of files but not this one. diff --git a/testing/btest/opt/opt-files3.zeek b/testing/btest/opt/opt-files3.zeek index 647d791f19..7c60595dc5 100644 --- a/testing/btest/opt/opt-files3.zeek +++ b/testing/btest/opt/opt-files3.zeek @@ -1,8 +1,8 @@ # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" -# @TEST-EXEC: zeek -b -O ZAM --optimize-files='base/utils' --optimize-files='opt-files3' %INPUT >output +# @TEST-EXEC: zeek -b -O ZAM --optimize-files=base/utils --optimize-files=opt-files3 %INPUT >output # @TEST-EXEC: btest-diff output -# Tests that we can selectively pick a group of files *and* this one. +# Tests that we can selectively pick a group of (other) files *and* this one. function my_test() { diff --git a/testing/btest/opt/opt-func.zeek b/testing/btest/opt/opt-func.zeek index 01713b0676..61b967086e 100644 --- a/testing/btest/opt/opt-func.zeek +++ b/testing/btest/opt/opt-func.zeek @@ -1,5 +1,5 @@ # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" -# @TEST-EXEC: zeek -b -O ZAM --optimize-func='my_test' %INPUT >output +# @TEST-EXEC: zeek -b -O ZAM --optimize-func=my_test %INPUT >output # @TEST-EXEC: btest-diff output # Tests that we can selectively pick a given function. diff --git a/testing/btest/opt/opt-func2.zeek b/testing/btest/opt/opt-func2.zeek index 6ec477da0b..dc29a99445 100644 --- a/testing/btest/opt/opt-func2.zeek +++ b/testing/btest/opt/opt-func2.zeek @@ -1,5 +1,5 @@ # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" -# @TEST-EXEC: zeek -b -O ZAM --optimize-func='another_test' %INPUT >output +# @TEST-EXEC: zeek -b -O ZAM --optimize-func=another_test %INPUT >output # @TEST-EXEC: btest-diff output # Tests that we can selectively pick a bunch of functions (event handlers), diff --git a/testing/btest/opt/opt-func3.zeek b/testing/btest/opt/opt-func3.zeek index 4d1f44ce3d..0a095ed107 100644 --- a/testing/btest/opt/opt-func3.zeek +++ b/testing/btest/opt/opt-func3.zeek @@ -1,5 +1,5 @@ # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" -# @TEST-EXEC: zeek -b -O ZAM --optimize-func='my_test' --optimize-func='another_test' %INPUT >output +# @TEST-EXEC: zeek -b -O ZAM --optimize-func=my_test --optimize-func=another_test %INPUT >output # @TEST-EXEC: btest-diff output # Tests that we can selectively pick a bunch of functions (event handlers), diff --git a/testing/btest/opt/opt-no-files.zeek b/testing/btest/opt/opt-no-files.zeek index 36a22c4c5e..dfa904fdfe 100644 --- a/testing/btest/opt/opt-no-files.zeek +++ b/testing/btest/opt/opt-no-files.zeek @@ -1,4 +1,4 @@ -# @TEST-EXEC-FAIL: zeek -b -O ZAM --optimize-files='Xopt-files' %INPUT +# @TEST-EXEC-FAIL: zeek -b -O ZAM --optimize-files=Xopt-files %INPUT # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr # Make sure that if --optimize-files is specified but there are no matching diff --git a/testing/btest/opt/opt-no-func.zeek b/testing/btest/opt/opt-no-func.zeek index 88b8d6de47..06e6009471 100644 --- a/testing/btest/opt/opt-no-func.zeek +++ b/testing/btest/opt/opt-no-func.zeek @@ -1,4 +1,4 @@ -# @TEST-EXEC-FAIL: zeek -b -O ZAM --optimize-files='my_func' %INPUT +# @TEST-EXEC-FAIL: zeek -b -O ZAM --optimize-files=my_func %INPUT # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr # Make sure that if --optimize-func is specified but there are no matching diff --git a/testing/btest/opt/opt-no-func2.zeek b/testing/btest/opt/opt-no-func2.zeek index 84d09b90fe..8b9f08afdb 100644 --- a/testing/btest/opt/opt-no-func2.zeek +++ b/testing/btest/opt/opt-no-func2.zeek @@ -1,4 +1,4 @@ -# @TEST-EXEC-FAIL: zeek -b -O ZAM --optimize-files='my_func' %INPUT +# @TEST-EXEC-FAIL: zeek -b -O ZAM --optimize-files=my_func %INPUT # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr # Make sure that --optimize-func anchors the regex. diff --git a/testing/btest/opt/opt-skip-files.zeek b/testing/btest/opt/opt-skip-files.zeek new file mode 100644 index 0000000000..95e63f20d5 --- /dev/null +++ b/testing/btest/opt/opt-skip-files.zeek @@ -0,0 +1,20 @@ +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" +# @TEST-EXEC: zeek -b -O ZAM --no-optimize-files=opt-skip-files %INPUT >output +# @TEST-EXEC-FAIL: zeek -b -O ZAM --no-optimize-files=opt-skip-files --optimize-files=opt-skip-files %INPUT +# @TEST-EXEC: btest-diff output + +# The first run tests that we can selectively exclude this file. +# +# The second run tests that skipping overrides including. This should +# result in an error because there are no functions to compile. + +function my_test() + { + print "I shouldn't be ZAM code!"; + } + +event zeek_init() + { + my_test(); + print my_test; + } diff --git a/testing/btest/opt/opt-skip-funcs.zeek b/testing/btest/opt/opt-skip-funcs.zeek new file mode 100644 index 0000000000..14d7b814f6 --- /dev/null +++ b/testing/btest/opt/opt-skip-funcs.zeek @@ -0,0 +1,19 @@ +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" +# @TEST-EXEC: zeek -b -O ZAM --no-optimize-func=my_test %INPUT >out1 +# @TEST-EXEC: zeek -b -O ZAM --no-optimize-func=my_test --optimize-func=my_test %INPUT --optimize-func=zeek_init >out2 +# @TEST-EXEC: btest-diff out1 +# @TEST-EXEC: btest-diff out2 + +# The first test checks that we can selectively exclude a function. +# The second tests that skipping overrides including. + +function my_test() + { + print "I shouldn't be ZAM code!"; + } + +event zeek_init() + { + my_test(); + print my_test; + } diff --git a/testing/btest/opt/opt-skip-other.zeek b/testing/btest/opt/opt-skip-other.zeek new file mode 100644 index 0000000000..55b4833f61 --- /dev/null +++ b/testing/btest/opt/opt-skip-other.zeek @@ -0,0 +1,20 @@ +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" +# @TEST-EXEC: zeek -b -O ZAM --no-optimize-files=Xopt-skip-other-file %INPUT >out1 +# @TEST-EXEC: zeek -b -O ZAM --no-optimize-func=my_tes %INPUT >out2 +# @TEST-EXEC: btest-diff out1 +# @TEST-EXEC: btest-diff out2 + +# The first test checks that if we exclude a different file, we still compile +# this one. The second tests that if we exclude a different function (since +# function matches have to be full), we still compile this one. + +function my_test() + { + print "I should be ZAM code!"; + } + +event zeek_init() + { + my_test(); + print my_test; + }