diff --git a/CHANGES b/CHANGES index fd7a229fc1..ab7f748865 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,35 @@ +5.0.0-dev.427 | 2022-05-16 16:59:31 -0700 + + * NEWS items for script profiling, "footprint" BiFs, and Trigger method deprecations (Vern Paxson, Corelight) + + * make deprecation advice a bit more helpful (Vern Paxson, Corelight) + + * add deprecation for Trigger Cache/Lookup interfaces (Vern Paxson, Corelight) + + * helper scripts for -O C++ maintenance (Vern Paxson, Corelight) + + * "-O use-C++" now fails if no C++ bodies wound up being applicable (Vern Paxson, Corelight) + + * update environemnt variable names to match command-line arguments (Vern Paxson, Corelight) + + * final changes for -O C++ feature completeness, mainly "when" statement support (Vern Paxson, Corelight) + + * Change "when" statements that don't require closures to use new implementation. (Vern Paxson, Corelight) + Provide hooks for script optimization access to "when" statements. + Regularize treatment of naming and timeouts for Triggers. + + * switch cached Trigger values to be opaque-and-generic, rather than assuming CallExpr's (Vern Paxson, Corelight) + + * tracking of when statements/expressions occur in a "when" context (Vern Paxson, Corelight) + + * fix bug that sometimes associated the wrong location with global statements (Vern Paxson, Corelight) + + * fix incomplete AST traversal of "when" statements (Vern Paxson, Corelight) + + * add some missing "when" closures (Vern Paxson, Corelight) + + * regularize test suite names to avoid custom/outdated suffices (Vern Paxson, Corelight) + 5.0.0-dev.410 | 2022-05-16 13:14:31 -0700 * Call pip using python -m flag during FreeBSD CI prep (Tim Wojtulewicz, Corelight) diff --git a/NEWS b/NEWS index 0b9eb52929..fa1f7f3711 100644 --- a/NEWS +++ b/NEWS @@ -12,6 +12,19 @@ Breaking Changes New Functionality ----------------- +- A new feature, ``--profile-scripts[=file]`` instructs Zeek to write (upon + termination) a profile of every function body to the given file (default: + stdout), along with some aggregate profiles. A function body is associated + with a function or a BiF (they can only have one), or a hook or event + handler (they can have multiple). The profile is in TSV, quasi-Zeek-log + format. It includes (1) the name of the script body, (2) its location, + (3) its type (e.g., "BiF" or "function"), (4) how many times it was called, + (5) the total CPU time that accumulated during its execution, (6) how much + of that time was due to execution of its "children" (other functions it + called), (7) the total memory allocated during its execution (approximately), + and (8) how much of that memory was due to its children. Note that profiling + is expensive and may not be suitable for execution on live traffic. + - Zeek now supports generation and replay of event traces via the new ``--event-trace`` / ``-E`` command-line options. For details, see: https://docs.zeek.org/en/master/quickstart.html#tracing-events @@ -37,6 +50,12 @@ New Functionality to this library simplifies the DNS code, adds support for IPv6 lookups, and adds the ability to support more DNS request types in the future. +- Two new BiFs, val_footprint() and global_container_footprints(), offer + substitutes for the deprecated val_size() and global_sizes() BiFs. + A value's "footprint" is the number of elements it includes either + directly or indirectly. The number is not meant to be precise, but + rather comparable: larger footprint correlates with more memory consumption. + Changed Functionality --------------------- @@ -55,6 +74,12 @@ Changed Functionality Deprecated Functionality ------------------------ +- For developers of asynchronous BiFs, or other uses of ``when`` statement + triggers, the versions of ``Trigger::Cache`` and ``Trigger::Lookup`` that + take ``const CallExpr*`` parameters have been deprecated. The replacements + take ``const void*`` parameters. Usually all you'll need to do is replace + previous uses of ``Frame::GetCall`` with ``Frame::GetTriggerAssoc``. + Zeek 4.2.0 ========== diff --git a/VERSION b/VERSION index 6f25309aa6..0b2da318ac 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.0-dev.410 +5.0.0-dev.427 diff --git a/src/Expr.cc b/src/Expr.cc index 997c1509e9..fa3dfaafdf 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2830,8 +2830,10 @@ ValPtr IndexSliceAssignExpr::Eval(Frame* f) const return nullptr; } -IndexExpr::IndexExpr(ExprPtr arg_op1, ListExprPtr arg_op2, bool arg_is_slice) - : BinaryExpr(EXPR_INDEX, std::move(arg_op1), std::move(arg_op2)), is_slice(arg_is_slice) +IndexExpr::IndexExpr(ExprPtr arg_op1, ListExprPtr arg_op2, bool arg_is_slice, + bool arg_is_inside_when) + : BinaryExpr(EXPR_INDEX, std::move(arg_op1), std::move(arg_op2)), is_slice(arg_is_slice), + is_inside_when(arg_is_inside_when) { if ( IsError() ) return; @@ -4467,8 +4469,8 @@ ValPtr InExpr::Fold(Val* v1, Val* v2) const return val_mgr->Bool(res); } -CallExpr::CallExpr(ExprPtr arg_func, ListExprPtr arg_args, bool in_hook) - : Expr(EXPR_CALL), func(std::move(arg_func)), args(std::move(arg_args)) +CallExpr::CallExpr(ExprPtr arg_func, ListExprPtr arg_args, bool in_hook, bool _in_when) + : Expr(EXPR_CALL), func(std::move(arg_func)), args(std::move(arg_args)), in_when(_in_when) { if ( func->IsError() || args->IsError() ) { @@ -4588,7 +4590,7 @@ ValPtr CallExpr::Eval(Frame* f) const { if ( trigger::Trigger* trigger = f->GetTrigger() ) { - if ( Val* v = trigger->Lookup(this) ) + if ( Val* v = trigger->Lookup((void*)this) ) { DBG_LOG(DBG_NOTIFIERS, "%s: provides cached function result", trigger->Name()); return {NewRef{}, v}; @@ -4603,7 +4605,7 @@ ValPtr CallExpr::Eval(Frame* f) const if ( func_val && v ) { const zeek::Func* funcv = func_val->AsFunc(); - const CallExpr* current_call = f ? f->GetCall() : nullptr; + auto current_assoc = f ? f->GetTriggerAssoc() : nullptr; if ( f ) f->SetCall(this); @@ -4612,7 +4614,7 @@ ValPtr CallExpr::Eval(Frame* f) const ret = funcv->Invoke(&args, f); if ( f ) - f->SetCall(current_call); + f->SetTriggerAssoc(current_assoc); } return ret; diff --git a/src/Expr.h b/src/Expr.h index a8b90d42dc..f6e45561a7 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -1005,7 +1005,7 @@ public: class IndexExpr : public BinaryExpr { public: - IndexExpr(ExprPtr op1, ListExprPtr op2, bool is_slice = false); + IndexExpr(ExprPtr op1, ListExprPtr op2, bool is_slice = false, bool is_inside_when = false); bool CanAdd() const override; bool CanDel() const override; @@ -1021,6 +1021,7 @@ public: ValPtr Eval(Frame* f) const override; bool IsSlice() const { return is_slice; } + bool IsInsideWhen() const { return is_inside_when; } // Optimization-related: ExprPtr Duplicate() override; @@ -1034,6 +1035,7 @@ protected: void ExprDescribe(ODesc* d) const override; bool is_slice; + bool is_inside_when; }; // The following execute the heart of IndexExpr functionality for @@ -1059,6 +1061,13 @@ extern VectorValPtr vector_bool_select(VectorTypePtr vt, const VectorVal* v1, co // indices to select). extern VectorValPtr vector_int_select(VectorTypePtr vt, const VectorVal* v1, const VectorVal* v2); +// The following is used for index expressions that occur inside "when" +// clauses. It tracks all the results produced by evaluating indexing +// aggregates, so that if any of them are Modifiable(), the associated +// Trigger can register interest in changes to them. +// +// TODO: One Fine Day we should do the equivalent for accessing fields +// in records, too. class IndexExprWhen final : public IndexExpr { public: @@ -1077,7 +1086,7 @@ public: } IndexExprWhen(ExprPtr op1, ListExprPtr op2, bool is_slice = false) - : IndexExpr(std::move(op1), std::move(op2), is_slice) + : IndexExpr(std::move(op1), std::move(op2), is_slice, true) { } @@ -1398,13 +1407,14 @@ protected: class CallExpr final : public Expr { public: - CallExpr(ExprPtr func, ListExprPtr args, bool in_hook = false); + CallExpr(ExprPtr func, ListExprPtr args, bool in_hook = false, bool in_when = false); Expr* Func() const { return func.get(); } ListExpr* Args() const { return args.get(); } ListExprPtr ArgsPtr() const { return args; } bool IsPure() const override; + bool IsInWhen() const { return in_when; } ValPtr Eval(Frame* f) const override; @@ -1424,6 +1434,7 @@ protected: ExprPtr func; ListExprPtr args; + bool in_when; }; /** diff --git a/src/Frame.cc b/src/Frame.cc index e0165d0b52..a3539157e9 100644 --- a/src/Frame.cc +++ b/src/Frame.cc @@ -201,6 +201,7 @@ Frame* Frame::Clone() const other->CaptureClosure(closure, outer_ids); other->call = call; + other->assoc = assoc; other->trigger = trigger; for ( int i = 0; i < size; i++ ) diff --git a/src/Frame.h b/src/Frame.h index d67929fb2b..64944b7e3e 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -258,10 +258,16 @@ public: void ClearTrigger(); trigger::Trigger* GetTrigger() const { return trigger.get(); } - void SetCall(const CallExpr* arg_call) { call = arg_call; } - void ClearCall() { call = nullptr; } + void SetCall(const CallExpr* arg_call) + { + call = arg_call; + SetTriggerAssoc((void*)call); + } const CallExpr* GetCall() const { return call; } + void SetTriggerAssoc(const void* arg_assoc) { assoc = arg_assoc; } + const void* GetTriggerAssoc() const { return assoc; } + void SetCallLoc(const Location* loc) { call_loc = loc; } const detail::Location* GetCallLocation() const; @@ -388,6 +394,7 @@ private: trigger::TriggerPtr trigger; const CallExpr* call = nullptr; + const void* assoc = nullptr; const Location* call_loc = nullptr; // only needed if call is nil std::unique_ptr> functions_with_closure_frame_reference; diff --git a/src/Func.cc b/src/Func.cc index 3c44d247c5..432c1b5b70 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -365,7 +365,7 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const if ( parent ) { f->SetTrigger({NewRef{}, parent->GetTrigger()}); - f->SetCall(parent->GetCall()); + f->SetTriggerAssoc(parent->GetTriggerAssoc()); } g_frame_stack.push_back(f.get()); // used for backtracing diff --git a/src/Stmt.cc b/src/Stmt.cc index 0edca90435..00d29e8eb6 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -1801,40 +1801,37 @@ TraversalCode NullStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -WhenInfo::WhenInfo(ExprPtr _cond, FuncType::CaptureList* _cl, bool _is_return) - : cond(std::move(_cond)), cl(_cl), is_return(_is_return) +WhenInfo::WhenInfo(ExprPtr arg_cond, FuncType::CaptureList* arg_cl, bool arg_is_return) + : cond(std::move(arg_cond)), cl(arg_cl), is_return(arg_is_return) { prior_vars = current_scope()->Vars(); ProfileFunc cond_pf(cond.get()); - if ( ! cl ) - { - for ( auto& wl : cond_pf.WhenLocals() ) - prior_vars.erase(wl->Name()); - return; - } - when_expr_locals = cond_pf.Locals(); - when_expr_globals = cond_pf.Globals(); + when_expr_globals = cond_pf.AllGlobals(); + when_new_locals = cond_pf.WhenLocals(); // Make any when-locals part of our captures, if not already present, // to enable sharing between the condition and the body/timeout code. - for ( auto& wl : cond_pf.WhenLocals() ) + for ( auto& wl : when_new_locals ) { bool is_present = false; - for ( auto& c : *cl ) - if ( c.id == wl ) - { - is_present = true; - break; - } - - if ( ! is_present ) + if ( cl ) { - IDPtr wl_ptr = {NewRef{}, const_cast(wl)}; - cl->emplace_back(FuncType::Capture{wl_ptr, false}); + for ( auto& c : *cl ) + if ( c.id == wl ) + { + is_present = true; + break; + } + + if ( ! is_present ) + { + IDPtr wl_ptr = {NewRef{}, const_cast(wl)}; + cl->emplace_back(FuncType::Capture{wl_ptr, false}); + } } // In addition, don't treat them as external locals that @@ -1851,62 +1848,50 @@ WhenInfo::WhenInfo(ExprPtr _cond, FuncType::CaptureList* _cl, bool _is_return) param_list->push_back(new TypeDecl(util::copy_string(lambda_param_id.c_str()), count_t)); auto params = make_intrusive(param_list); - auto ft = make_intrusive(params, base_type(TYPE_ANY), FUNC_FLAVOR_FUNCTION); - ft->SetCaptures(*cl); + lambda_ft = make_intrusive(params, base_type(TYPE_ANY), FUNC_FLAVOR_FUNCTION); if ( ! is_return ) - ft->SetExpressionlessReturnOkay(true); + lambda_ft->SetExpressionlessReturnOkay(true); auto id = current_scope()->GenerateTemporary("when-internal"); + id->SetType(lambda_ft); + push_scope(std::move(id), nullptr); - // This begin_func will be completed by WhenInfo::Build(). - begin_func(id, current_module.c_str(), FUNC_FLAVOR_FUNCTION, false, ft); + auto arg_id = install_ID(lambda_param_id.c_str(), current_module.c_str(), false, false); + arg_id->SetType(count_t); + } + +WhenInfo::WhenInfo(bool arg_is_return) : is_return(arg_is_return) + { + // This won't be needed once we remove the deprecated semantics. + cl = new zeek::FuncType::CaptureList; + BuildInvokeElems(); } void WhenInfo::Build(StmtPtr ws) { - if ( ! cl ) + if ( IsDeprecatedSemantics(ws) ) { - // Old-style semantics. - auto locals = when_expr_locals; - - ProfileFunc cond_pf(cond.get()); - for ( auto& bl : cond_pf.Locals() ) - if ( prior_vars.count(bl->Name()) > 0 ) - locals.insert(bl); - - ProfileFunc body_pf(s.get()); - for ( auto& bl : body_pf.Locals() ) - if ( prior_vars.count(bl->Name()) > 0 ) - locals.insert(bl); - - if ( timeout_s ) - { - ProfileFunc to_pf(timeout_s.get()); - for ( auto& tl : to_pf.Locals() ) - if ( prior_vars.count(tl->Name()) > 0 ) - locals.insert(tl); - } - - if ( ! locals.empty() ) - { - std::string vars; - for ( auto& l : locals ) - { - if ( ! vars.empty() ) - vars += ", "; - vars += l->Name(); - } - - std::string msg = util::fmt("\"when\" statement referring to locals without an " - "explicit [] capture is deprecated: %s", - vars.c_str()); - ws->Warn(msg.c_str()); - } - + pop_scope(); return; } + if ( ! cl ) + { + // This instance is compatible with new-style semantics, + // so create a capture list for it and populate with any + // when-locals. + cl = new zeek::FuncType::CaptureList; + + for ( auto& wl : when_new_locals ) + { + IDPtr wl_ptr = {NewRef{}, const_cast(wl)}; + cl->emplace_back(FuncType::Capture{wl_ptr, false}); + } + } + + lambda_ft->SetCaptures(*cl); + // Our general strategy is to construct a single lambda (so that // the values of captures are shared across all of its elements) // that's used for all three of the "when" components: condition, @@ -1924,14 +1909,9 @@ void WhenInfo::Build(StmtPtr ws) // Build the AST elements of the lambda. // First, the constants we'll need. - auto true_const = make_intrusive(val_mgr->True()); - auto one_const = make_intrusive(val_mgr->Count(1)); - auto two_const = make_intrusive(val_mgr->Count(2)); - auto three_const = make_intrusive(val_mgr->Count(3)); + BuildInvokeElems(); - invoke_cond = make_intrusive(one_const); - invoke_s = make_intrusive(two_const); - invoke_timeout = make_intrusive(three_const); + auto true_const = make_intrusive(val_mgr->True()); // Access to the parameter that selects which action we're doing. auto param_id = lookup_ID(lambda_param_id.c_str(), current_module.c_str()); @@ -1963,7 +1943,13 @@ void WhenInfo::Build(StmtPtr ws) void WhenInfo::Instantiate(Frame* f) { if ( cl ) - curr_lambda = make_intrusive(lambda->Eval(f)); + Instantiate(lambda->Eval(f)); + } + +void WhenInfo::Instantiate(ValPtr func) + { + if ( cl ) + curr_lambda = make_intrusive(std::move(func)); } ExprPtr WhenInfo::Cond() @@ -1983,6 +1969,18 @@ StmtPtr WhenInfo::WhenBody() return make_intrusive(invoke, true); } +double WhenInfo::TimeoutVal(Frame* f) + { + if ( timeout ) + { + auto t = timeout->Eval(f); + if ( t ) + return t->AsDouble(); + } + + return -1.0; // signals "no timeout" + } + StmtPtr WhenInfo::TimeoutStmt() { if ( ! curr_lambda ) @@ -1992,7 +1990,66 @@ StmtPtr WhenInfo::TimeoutStmt() return make_intrusive(invoke, true); } -WhenStmt::WhenStmt(WhenInfo* _wi) : Stmt(STMT_WHEN), wi(_wi) +bool WhenInfo::IsDeprecatedSemantics(StmtPtr ws) + { + if ( cl ) + return false; + + // Which locals of the outer function are used in any of the "when" + // elements. + IDSet locals; + + for ( auto& wl : when_new_locals ) + prior_vars.erase(wl->Name()); + + for ( auto& bl : when_expr_locals ) + if ( prior_vars.count(bl->Name()) > 0 ) + locals.insert(bl); + + ProfileFunc body_pf(s.get()); + for ( auto& bl : body_pf.Locals() ) + if ( prior_vars.count(bl->Name()) > 0 ) + locals.insert(bl); + + if ( timeout_s ) + { + ProfileFunc to_pf(timeout_s.get()); + for ( auto& tl : to_pf.Locals() ) + if ( prior_vars.count(tl->Name()) > 0 ) + locals.insert(tl); + } + + if ( locals.empty() ) + return false; + + std::string vars; + for ( auto& l : locals ) + { + if ( ! vars.empty() ) + vars += ", "; + vars += l->Name(); + } + + std::string msg = util::fmt("\"when\" statement referring to locals without an " + "explicit [] capture is deprecated: %s", + vars.c_str()); + ws->Warn(msg.c_str()); + + return true; + } + +void WhenInfo::BuildInvokeElems() + { + one_const = make_intrusive(val_mgr->Count(1)); + two_const = make_intrusive(val_mgr->Count(2)); + three_const = make_intrusive(val_mgr->Count(3)); + + invoke_cond = make_intrusive(one_const); + invoke_s = make_intrusive(two_const); + invoke_timeout = make_intrusive(three_const); + } + +WhenStmt::WhenStmt(WhenInfo* arg_wi) : Stmt(STMT_WHEN), wi(arg_wi) { wi->Build(ThisPtr()); @@ -2026,6 +2083,8 @@ ValPtr WhenStmt::Exec(Frame* f, StmtFlowType& flow) wi->Instantiate(f); + auto timeout = wi->TimeoutVal(f); + if ( wi->Captures() ) { std::vector local_aggrs; @@ -2037,12 +2096,12 @@ ValPtr WhenStmt::Exec(Frame* f, StmtFlowType& flow) local_aggrs.emplace_back(std::move(v)); } - new trigger::Trigger(wi, wi->WhenExprGlobals(), local_aggrs, f, location); + new trigger::Trigger(wi, timeout, wi->WhenExprGlobals(), local_aggrs, f, location); } else // The new trigger object will take care of its own deletion. - new trigger::Trigger(wi->Cond(), wi->WhenBody(), wi->TimeoutStmt(), wi->TimeoutExpr(), f, + new trigger::Trigger(wi->Cond(), wi->WhenBody(), wi->TimeoutStmt(), timeout, f, wi->IsReturn(), location); return nullptr; @@ -2105,6 +2164,12 @@ TraversalCode WhenStmt::Traverse(TraversalCallback* cb) const tc = wi->WhenBody()->Traverse(cb); HANDLE_TC_STMT_PRE(tc); + if ( wi->TimeoutExpr() ) + { + tc = wi->TimeoutExpr()->Traverse(cb); + HANDLE_TC_STMT_PRE(tc); + } + if ( wi->TimeoutStmt() ) { tc = wi->TimeoutStmt()->Traverse(cb); diff --git a/src/Stmt.h b/src/Stmt.h index f61ffb28b1..2fd18d225d 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -546,24 +546,35 @@ class WhenInfo public: // Takes ownership of the CaptureList, which if nil signifies // old-style frame semantics. - WhenInfo(ExprPtr _cond, FuncType::CaptureList* _cl, bool _is_return); + WhenInfo(ExprPtr cond, FuncType::CaptureList* cl, bool is_return); + + // Constructor used by script optimization to create a stub. + WhenInfo(bool is_return); + ~WhenInfo() { delete cl; } - void AddBody(StmtPtr _s) { s = std::move(_s); } + void AddBody(StmtPtr arg_s) { s = std::move(arg_s); } - void AddTimeout(ExprPtr _timeout, StmtPtr _timeout_s) + void AddTimeout(ExprPtr arg_timeout, StmtPtr arg_timeout_s) { - timeout = std::move(_timeout); - timeout_s = std::move(_timeout_s); + timeout = std::move(arg_timeout); + timeout_s = std::move(arg_timeout_s); } // Complete construction of the associated internals, including // the (complex) lambda used to access the different elements of - // the statement. - void Build(StmtPtr ws); + // the statement. The optional argument is only for generating + // error messages. + void Build(StmtPtr ws = nullptr); - // Instantiate a new instance. + // This is available after a call to Build(). + const LambdaExprPtr& Lambda() const { return lambda; } + + // Instantiate a new instance, either by evaluating the associated + // lambda, or directly using the given function value (for compiled + // code). void Instantiate(Frame* f); + void Instantiate(ValPtr func); // For old-style semantics, the following simply return the // individual "when" components. For capture semantics, however, @@ -572,15 +583,15 @@ public: ExprPtr Cond(); StmtPtr WhenBody(); - ExprPtr TimeoutExpr() { return timeout; } + ExprPtr TimeoutExpr() const { return timeout; } + double TimeoutVal(Frame* f); + StmtPtr TimeoutStmt(); FuncType::CaptureList* Captures() { return cl; } bool IsReturn() const { return is_return; } - const LambdaExprPtr& Lambda() const { return lambda; } - // The locals and globals used in the conditional expression // (other than newly introduced locals), necessary for registering // the associated triggers for when their values change. @@ -588,6 +599,15 @@ public: const IDSet& WhenExprGlobals() const { return when_expr_globals; } private: + // True if the "when" statement corresponds to old-style deprecated + // semantics (no captures, but needing captures). Also generates + // the corresponding deprecation warnings, which are associated + // with "ws". + bool IsDeprecatedSemantics(StmtPtr ws); + + // Build those elements we'll need for invoking our lambda. + void BuildInvokeElems(); + ExprPtr cond; StmtPtr s; ExprPtr timeout; @@ -599,8 +619,9 @@ private: // The name of parameter passed ot the lambda. std::string lambda_param_id; - // The expression for constructing the lambda. + // The expression for constructing the lambda, and its type. LambdaExprPtr lambda; + FuncTypePtr lambda_ft; // The current instance of the lambda. Created by Instantiate(), // for immediate use via calls to Cond() etc. @@ -612,9 +633,17 @@ private: ListExprPtr invoke_s; ListExprPtr invoke_timeout; + // Helper expressions for calling the lambda / testing within it. + ConstExprPtr one_const; + ConstExprPtr two_const; + ConstExprPtr three_const; + IDSet when_expr_locals; IDSet when_expr_globals; + // Locals introduced via "local" in the "when" clause itself. + IDSet when_new_locals; + // Used for identifying deprecated instances. Holds all of the local // variables in the scope prior to parsing the "when" statement. std::map> prior_vars; diff --git a/src/Trigger.cc b/src/Trigger.cc index 9be2bed8d0..36e2a7acd4 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -99,13 +99,6 @@ protected: double time; }; -Trigger::Trigger(ExprPtr cond, StmtPtr body, StmtPtr timeout_stmts, ExprPtr timeout_expr, - Frame* frame, bool is_return, const Location* location) - { - GetTimeout(timeout_expr, frame); - Init(cond, body, timeout_stmts, frame, is_return, location); - } - Trigger::Trigger(ExprPtr cond, StmtPtr body, StmtPtr timeout_stmts, double timeout, Frame* frame, bool is_return, const Location* location) { @@ -113,41 +106,19 @@ Trigger::Trigger(ExprPtr cond, StmtPtr body, StmtPtr timeout_stmts, double timeo Init(cond, body, timeout_stmts, frame, is_return, location); } -Trigger::Trigger(WhenInfo* wi, const IDSet& _globals, std::vector _local_aggrs, Frame* f, - const Location* loc) +Trigger::Trigger(WhenInfo* wi, double timeout, const IDSet& _globals, + std::vector _local_aggrs, Frame* f, const Location* loc) { + timeout_value = timeout; globals = _globals; local_aggrs = std::move(_local_aggrs); have_trigger_elems = true; - GetTimeout(wi->TimeoutExpr(), f); - Init(wi->Cond(), wi->WhenBody(), wi->TimeoutStmt(), f, wi->IsReturn(), loc); } -void Trigger::GetTimeout(const ExprPtr& timeout_expr, Frame* f) - { - timeout_value = -1.0; - - if ( timeout_expr ) - { - ValPtr timeout_val; - - try - { - timeout_val = timeout_expr->Eval(f); - } - catch ( InterpreterException& ) - { /* Already reported */ - } - - if ( timeout_val ) - timeout_value = timeout_val->AsInterval(); - } - } - void Trigger::Init(ExprPtr arg_cond, StmtPtr arg_body, StmtPtr arg_timeout_stmts, Frame* arg_frame, - bool arg_is_return, const Location* arg_location) + bool arg_is_return, const Location* location) { cond = arg_cond; body = arg_body; @@ -157,7 +128,11 @@ void Trigger::Init(ExprPtr arg_cond, StmtPtr arg_body, StmtPtr arg_timeout_stmts disabled = false; attached = nullptr; is_return = arg_is_return; - location = arg_location; + + if ( location ) + name = util::fmt("%s:%d-%d", location->filename, location->first_line, location->last_line); + else + name = ""; if ( arg_frame ) frame = arg_frame->Clone(); @@ -350,7 +325,7 @@ bool Trigger::Eval() { Trigger* trigger = frame->GetTrigger(); assert(trigger); - assert(frame->GetCall()); + assert(frame->GetTriggerAssoc()); assert(trigger->attached == this); #ifdef DEBUG @@ -359,7 +334,7 @@ bool Trigger::Eval() delete[] pname; #endif - auto queued = trigger->Cache(frame->GetCall(), v.get()); + auto queued = trigger->Cache(frame->GetTriggerAssoc(), v.get()); trigger->Release(); frame->ClearTrigger(); @@ -404,7 +379,7 @@ void Trigger::Timeout() { Trigger* trigger = frame->GetTrigger(); assert(trigger); - assert(frame->GetCall()); + assert(frame->GetTriggerAssoc()); assert(trigger->attached == this); #ifdef DEBUG @@ -413,7 +388,7 @@ void Trigger::Timeout() pname); delete[] pname; #endif - auto queued = trigger->Cache(frame->GetCall(), v.get()); + auto queued = trigger->Cache(frame->GetTriggerAssoc(), v.get()); trigger->Release(); frame->ClearTrigger(); @@ -480,12 +455,12 @@ void Trigger::Attach(Trigger* trigger) Hold(); } -bool Trigger::Cache(const CallExpr* expr, Val* v) +bool Trigger::Cache(const void* obj, Val* v) { if ( disabled || ! v ) return false; - ValCache::iterator i = cache.find(expr); + ValCache::iterator i = cache.find(obj); if ( i != cache.end() ) { @@ -494,7 +469,7 @@ bool Trigger::Cache(const CallExpr* expr, Val* v) } else - cache.insert(ValCache::value_type(expr, v)); + cache.insert(ValCache::value_type(obj, v)); Ref(v); @@ -502,11 +477,11 @@ bool Trigger::Cache(const CallExpr* expr, Val* v) return true; } -Val* Trigger::Lookup(const CallExpr* expr) +Val* Trigger::Lookup(const void* obj) { assert(! disabled); - ValCache::iterator i = cache.find(expr); + ValCache::iterator i = cache.find(obj); return (i != cache.end()) ? i->second : 0; } @@ -526,12 +501,6 @@ void Trigger::Modified(notifier::detail::Modifiable* m) trigger_mgr->Queue(this); } -const char* Trigger::Name() const - { - assert(location); - return util::fmt("%s:%d-%d", location->filename, location->first_line, location->last_line); - } - Manager::Manager() : iosource::IOSource() { pending = new TriggerList(); diff --git a/src/Trigger.h b/src/Trigger.h index 833b3f05bc..99e1bf3433 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -48,16 +48,13 @@ public: // statements are executed immediately and the object is deleted // right away. - // These first two constructors are for the deprecated deep-copy - // semantics. - Trigger(ExprPtr cond, StmtPtr body, StmtPtr timeout_stmts, ExprPtr timeout, Frame* f, - bool is_return, const Location* loc); + // These first constructor is for the deprecated deep-copy semantics. Trigger(ExprPtr cond, StmtPtr body, StmtPtr timeout_stmts, double timeout, Frame* f, bool is_return, const Location* loc); // Used for capture-list semantics. - Trigger(WhenInfo* wi, const IDSet& globals, std::vector local_aggrs, Frame* f, - const Location* loc); + Trigger(WhenInfo* wi, double timeout, const IDSet& globals, std::vector local_aggrs, + Frame* f, const Location* loc); ~Trigger() override; @@ -86,9 +83,24 @@ public: // Cache for return values of delayed function calls. Returns whether // the trigger is queued for later evaluation -- it may not be queued - // if the Val is null or it's disabled. - bool Cache(const CallExpr* expr, Val* val); - Val* Lookup(const CallExpr*); + // if the Val is null or it's disabled. The cache is managed using + // void*'s so that the value can be associated with either a CallExpr + // (for interpreted execution) or a C++ function (for compiled-to-C++). + bool Cache(const void* obj, Val* val); + Val* Lookup(const void* obj); + + [[deprecated( + "Remove in v5.1. Use Frame::GetTriggerAssoc() / const void* interface instead.")]] bool + Cache(const CallExpr* call, Val* val) + { + return Cache((const void*)call, val); + } + [[deprecated( + "Remove in v5.1. Use Frame::GetTriggerAssoc() / const void* interface instead.")]] Val* + Lookup(const CallExpr* call) + { + return Lookup((const void*)call); + } // Disable this trigger completely. Needed because Unref'ing the trigger // may not immediately delete it as other references may still exist. @@ -107,13 +119,12 @@ public: // for any further progress to be made, so just Unref ourselves. void Terminate() override; - const char* Name() const; + const char* Name() const { return name.c_str(); } + void SetName(const char* new_name) { name = new_name; } private: friend class TriggerTimer; - void GetTimeout(const ExprPtr& timeout_expr, Frame* f); - void Init(ExprPtr cond, StmtPtr body, StmtPtr timeout_stmts, Frame* frame, bool is_return, const Location* location); @@ -130,7 +141,8 @@ private: double timeout_value; Frame* frame; bool is_return; - const Location* location; + + std::string name; TriggerTimer* timer; Trigger* attached; @@ -153,7 +165,7 @@ private: std::vector> objs; - using ValCache = std::map; + using ValCache = std::map; ValCache cache; }; diff --git a/src/broker/Store.h b/src/broker/Store.h index 10ebad95e5..dd6f2c14af 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -5,6 +5,7 @@ #include #include +#include "zeek/Expr.h" #include "zeek/OpaqueVal.h" #include "zeek/Trigger.h" #include "zeek/broker/data.bif.h" @@ -72,9 +73,9 @@ static std::optional convert_expiry(double e) class StoreQueryCallback { public: - StoreQueryCallback(zeek::detail::trigger::Trigger* arg_trigger, - const zeek::detail::CallExpr* arg_call, broker::store store) - : trigger(arg_trigger), call(arg_call), store(std::move(store)) + StoreQueryCallback(zeek::detail::trigger::Trigger* arg_trigger, const void* arg_assoc, + broker::store store) + : trigger(arg_trigger), assoc(arg_assoc), store(std::move(store)) { Ref(trigger); } @@ -83,14 +84,14 @@ public: void Result(const RecordValPtr& result) { - trigger->Cache(call, result.get()); + trigger->Cache(assoc, result.get()); trigger->Release(); } void Abort() { auto result = query_result(); - trigger->Cache(call, result.get()); + trigger->Cache(assoc, result.get()); trigger->Release(); } @@ -100,7 +101,7 @@ public: private: zeek::detail::trigger::Trigger* trigger; - const zeek::detail::CallExpr* call; + const void* assoc; broker::store store; }; diff --git a/src/broker/store.bif b/src/broker/store.bif index d29a44191d..8d4bf79dc3 100644 --- a/src/broker/store.bif +++ b/src/broker/store.bif @@ -156,7 +156,7 @@ function Broker::__exists%(h: opaque of Broker::Store, frame->SetDelayed(); trigger->Hold(); - auto cb = new zeek::Broker::detail::StoreQueryCallback(trigger, frame->GetCall(), + auto cb = new zeek::Broker::detail::StoreQueryCallback(trigger, frame->GetTriggerAssoc(), handle->store); auto req_id = handle->proxy.exists(std::move(*key)); broker_mgr->TrackStoreQuery(handle, req_id, cb); @@ -202,7 +202,7 @@ function Broker::__get%(h: opaque of Broker::Store, frame->SetDelayed(); trigger->Hold(); - auto cb = new zeek::Broker::detail::StoreQueryCallback(trigger, frame->GetCall(), + auto cb = new zeek::Broker::detail::StoreQueryCallback(trigger, frame->GetTriggerAssoc(), handle->store); auto req_id = handle->proxy.get(std::move(*key)); broker_mgr->TrackStoreQuery(handle, req_id, cb); @@ -255,7 +255,7 @@ function Broker::__put_unique%(h: opaque of Broker::Store, frame->SetDelayed(); trigger->Hold(); - auto cb = new zeek::Broker::detail::StoreQueryCallback(trigger, frame->GetCall(), + auto cb = new zeek::Broker::detail::StoreQueryCallback(trigger, frame->GetTriggerAssoc(), handle->store); auto req_id = handle->proxy.put_unique(std::move(*key), std::move(*val), @@ -311,7 +311,7 @@ function Broker::__get_index_from_value%(h: opaque of Broker::Store, frame->SetDelayed(); trigger->Hold(); - auto cb = new zeek::Broker::detail::StoreQueryCallback(trigger, frame->GetCall(), + auto cb = new zeek::Broker::detail::StoreQueryCallback(trigger, frame->GetTriggerAssoc(), handle->store); auto req_id = handle->proxy.get_index_from_value(std::move(*key), std::move(*index)); @@ -355,7 +355,7 @@ function Broker::__keys%(h: opaque of Broker::Store%): Broker::QueryResult frame->SetDelayed(); trigger->Hold(); - auto cb = new zeek::Broker::detail::StoreQueryCallback(trigger, frame->GetCall(), + auto cb = new zeek::Broker::detail::StoreQueryCallback(trigger, frame->GetTriggerAssoc(), handle->store); auto req_id = handle->proxy.keys(); broker_mgr->TrackStoreQuery(handle, req_id, cb); diff --git a/src/parse.y b/src/parse.y index ced76e6269..d2bb078322 100644 --- a/src/parse.y +++ b/src/parse.y @@ -5,7 +5,7 @@ // Switching parser table type fixes ambiguity problems. %define lr.type ielr -%expect 196 +%expect 195 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -13,7 +13,8 @@ %token TOK_CONSTANT TOK_COPY TOK_COUNT TOK_DEFAULT TOK_DELETE %token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FALLTHROUGH %token TOK_FILE TOK_FOR TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT -%token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE +%token TOK_INTERVAL TOK_LIST TOK_MODULE +%token TOK_LOCAL TOK_WHEN_LOCAL %token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_END TOK_PATTERN_TEXT %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET @@ -111,7 +112,7 @@ extern int conditional_epoch; // let's us track embedded conditionals // Whether the file we're currently parsing includes @if conditionals. extern bool current_file_has_conditionals; -YYLTYPE GetCurrentLocation(); +extern YYLTYPE GetCurrentLocation(); extern int yyerror(const char[]); extern int brolex(); @@ -133,7 +134,8 @@ extern Expr* g_curr_debug_expr; extern bool in_debug; extern const char* g_curr_debug_error; -static int in_when_cond = 0; +extern int in_when_cond; + static int in_hook = 0; int in_init = 0; int in_record = 0; @@ -317,12 +319,22 @@ static StmtPtr build_local(ID* id, Type* t, InitClass ic, Expr* e, %% zeek: - decl_list stmt_list + decl_list + { + // Without the following, in some scenarios the + // location associated with global statements gets + // associated with the last @load'd file rather than + // the script that includes the global statements. + auto loc = zeek::detail::GetCurrentLocation(); + if ( loc.filename ) + set_location(loc); + } + stmt_list { if ( stmts ) - stmts->AsStmtList()->Stmts().push_back($2); + stmts->AsStmtList()->Stmts().push_back($3); else - stmts = $2; + stmts = $3; // Any objects creates from here on out should not // have file positions associated with them. @@ -608,7 +620,7 @@ expr: $$ = get_assign_expr({AdoptRef{}, $1}, {AdoptRef{}, $3}, in_init).release(); } - | TOK_LOCAL local_id '=' rhs + | TOK_WHEN_LOCAL local_id '=' rhs { set_location(@2, @4); if ( ! locals_at_this_scope.empty() ) @@ -770,7 +782,7 @@ expr: } else - $$ = new CallExpr({AdoptRef{}, $1}, {AdoptRef{}, $4}, in_hook > 0); + $$ = new CallExpr({AdoptRef{}, $1}, {AdoptRef{}, $4}, in_hook > 0, in_when_cond); } | TOK_HOOK { ++in_hook; } expr diff --git a/src/scan.l b/src/scan.l index 52d2896692..cca6ddd0f5 100644 --- a/src/scan.l +++ b/src/scan.l @@ -68,6 +68,10 @@ std::unordered_set files_with_conditionals; zeek::detail::int_list if_stack; +// Whether we're parsing a "when" conditional, for which we treat +// the "local" keyword differently. +int in_when_cond = 0; + int line_number = 1; const char* filename = 0; // Absolute path of file currently being parsed. const char* last_filename = 0; // Absolute path of last file parsed. @@ -278,7 +282,7 @@ int return TOK_INT; interval return TOK_INTERVAL; is return TOK_IS; list return TOK_LIST; -local return TOK_LOCAL; +local return in_when_cond ? TOK_WHEN_LOCAL : TOK_LOCAL; module return TOK_MODULE; next return TOK_NEXT; of return TOK_OF; @@ -586,10 +590,11 @@ YYLTYPE zeek::detail::GetCurrentLocation() return currloc; } -void zeek::detail::SetCurrentLocation(YYLTYPE currloc) { +void zeek::detail::SetCurrentLocation(YYLTYPE currloc) + { ::filename = currloc.filename; line_number = currloc.first_line; -} + } static int load_files(const char* orig_file) { diff --git a/src/script_opt/CPP/Compile.h b/src/script_opt/CPP/Compile.h index 447427204e..01f5923bfd 100644 --- a/src/script_opt/CPP/Compile.h +++ b/src/script_opt/CPP/Compile.h @@ -724,6 +724,7 @@ private: void GenTypeSwitchCase(const ID* id, int case_offset, bool is_multi); void GenValueSwitchStmt(const Expr* e, const case_list* cases); + void GenWhenStmt(const WhenStmt* w); void GenForStmt(const ForStmt* f); void GenForOverTable(const ExprPtr& tbl, const IDPtr& value_var, const IDPList* loop_vars); void GenForOverVector(const ExprPtr& tbl, const IDPList* loop_vars); diff --git a/src/script_opt/CPP/Exprs.cc b/src/script_opt/CPP/Exprs.cc index fed8ddcbe9..58120b7cd7 100644 --- a/src/script_opt/CPP/Exprs.cc +++ b/src/script_opt/CPP/Exprs.cc @@ -292,6 +292,7 @@ string CPPCompile::GenCallExpr(const CallExpr* c, GenType gt) const auto& t = c->GetType(); auto f = c->Func(); auto args_l = c->Args(); + bool is_async = c->IsInWhen(); auto gen = GenExpr(f, GEN_DONT_CARE); @@ -304,8 +305,8 @@ string CPPCompile::GenCallExpr(const CallExpr* c, GenType gt) bool is_compiled = compiled_simple_funcs.count(id_name) > 0; bool was_compiled = hashed_funcs.count(id_name) > 0; - if ( is_compiled || was_compiled ) - { + if ( ! is_async && (is_compiled || was_compiled) ) + { // Can call directly. string fname; if ( was_compiled ) @@ -340,8 +341,14 @@ string CPPCompile::GenCallExpr(const CallExpr* c, GenType gt) // Indirect call. gen = string("(") + gen + ")->AsFunc()"; + string invoke_func = is_async ? "when_invoke__CPP" : "invoke__CPP"; auto args_list = string(", {") + GenExpr(args_l, GEN_VAL_PTR) + "}"; - auto invoker = string("invoke__CPP(") + gen + args_list + ", f__CPP)"; + auto invoker = invoke_func + "(" + gen + args_list + ", f__CPP"; + + if ( is_async ) + invoker += ", (void*) &" + body_name; + + invoker += ")"; if ( IsNativeType(t) && gt != GEN_VAL_PTR ) return invoker + NativeAccessor(t); @@ -408,12 +415,17 @@ string CPPCompile::GenIndexExpr(const Expr* e, GenType gt) { auto aggr = e->GetOp1(); const auto& aggr_t = aggr->GetType(); + bool inside_when = e->AsIndexExpr()->IsInsideWhen(); string gen; + string func; if ( aggr_t->Tag() == TYPE_TABLE ) - gen = string("index_table__CPP(") + GenExpr(aggr, GEN_NATIVE) + ", {" + - GenExpr(e->GetOp2(), GEN_VAL_PTR) + "})"; + { + func = inside_when ? "when_index_table__CPP" : "index_table__CPP"; + gen = func + "(" + GenExpr(aggr, GEN_NATIVE) + ", {" + GenExpr(e->GetOp2(), GEN_VAL_PTR) + + "})"; + } else if ( aggr_t->Tag() == TYPE_VECTOR ) { @@ -426,12 +438,16 @@ string CPPCompile::GenIndexExpr(const Expr* e, GenType gt) auto& inds = op2->AsListExpr()->Exprs(); auto first = inds[0]; auto last = inds[1]; - gen = string("index_slice(") + GenExpr(aggr, GEN_VAL_PTR) + ".get(), " + + func = inside_when ? "when_index_slice__CPP" : "index_slice"; + gen = func + "(" + GenExpr(aggr, GEN_VAL_PTR) + ".get(), " + GenExpr(first, GEN_NATIVE) + ", " + GenExpr(last, GEN_NATIVE) + ")"; } else - gen = string("index_vec__CPP(") + GenExpr(aggr, GEN_NATIVE) + ", " + - GenExpr(e->GetOp2(), GEN_NATIVE) + ")"; + { + func = inside_when ? "when_index_vec__CPP" : "index_vec__CPP"; + gen = func + "(" + GenExpr(aggr, GEN_NATIVE) + ", " + GenExpr(e->GetOp2(), GEN_NATIVE) + + ")"; + } } else if ( aggr_t->Tag() == TYPE_STRING ) diff --git a/src/script_opt/CPP/Runtime.h b/src/script_opt/CPP/Runtime.h index d872c1d9e3..7763e95229 100644 --- a/src/script_opt/CPP/Runtime.h +++ b/src/script_opt/CPP/Runtime.h @@ -13,6 +13,7 @@ #include "zeek/RE.h" #include "zeek/RunState.h" #include "zeek/Scope.h" +#include "zeek/Trigger.h" #include "zeek/Val.h" #include "zeek/ZeekString.h" #include "zeek/module_util.h" diff --git a/src/script_opt/CPP/RuntimeInitSupport.cc b/src/script_opt/CPP/RuntimeInitSupport.cc index eea927d192..4758659faf 100644 --- a/src/script_opt/CPP/RuntimeInitSupport.cc +++ b/src/script_opt/CPP/RuntimeInitSupport.cc @@ -245,6 +245,16 @@ FuncValPtr lookup_func__CPP(string name, int num_bodies, vector has return make_intrusive(move(sf)); } +IDPtr find_global__CPP(const char* g) + { + auto gl = lookup_ID(g, GLOBAL_MODULE_NAME, false, false, false); + + if ( ! gl ) + reporter->CPPRuntimeError("global %s is missing", g); + + return gl; + } + RecordTypePtr get_record_type__CPP(const char* record_type_name) { IDPtr existing_type; diff --git a/src/script_opt/CPP/RuntimeInitSupport.h b/src/script_opt/CPP/RuntimeInitSupport.h index 6b5ea46014..e7998716e4 100644 --- a/src/script_opt/CPP/RuntimeInitSupport.h +++ b/src/script_opt/CPP/RuntimeInitSupport.h @@ -70,6 +70,10 @@ extern Func* lookup_bif__CPP(const char* bif); extern FuncValPtr lookup_func__CPP(std::string name, int num_bodies, std::vector h, const TypePtr& t); +// Looks for a global with the given name, generating a run-time error +// if not present. +extern IDPtr find_global__CPP(const char* g); + // Returns the record corresponding to the given name, as long as the // name is indeed a record type. Otherwise (or if the name is nil) // creates a new empty record. diff --git a/src/script_opt/CPP/RuntimeOps.cc b/src/script_opt/CPP/RuntimeOps.cc index 35dd89dc6f..a6af5cad3e 100644 --- a/src/script_opt/CPP/RuntimeOps.cc +++ b/src/script_opt/CPP/RuntimeOps.cc @@ -3,8 +3,10 @@ #include "zeek/script_opt/CPP/RuntimeOps.h" #include "zeek/EventRegistry.h" +#include "zeek/Frame.h" #include "zeek/IPAddr.h" #include "zeek/RunState.h" +#include "zeek/Trigger.h" #include "zeek/ZeekString.h" namespace zeek::detail @@ -60,6 +62,50 @@ ValPtr index_string__CPP(const StringValPtr& svp, vector indices) return index_string(svp->AsString(), index_val__CPP(move(indices)).get()); } +ValPtr when_index_table__CPP(const TableValPtr& t, vector indices) + { + auto v = index_table__CPP(t, std::move(indices)); + if ( v && IndexExprWhen::evaluating > 0 ) + IndexExprWhen::results.emplace_back(v); + return v; + } + +ValPtr when_index_vec__CPP(const VectorValPtr& vec, int index) + { + auto v = index_vec__CPP(vec, index); + if ( v && IndexExprWhen::evaluating > 0 ) + IndexExprWhen::results.emplace_back(v); + return v; + } + +ValPtr when_index_slice__CPP(VectorVal* vec, const ListVal* lv) + { + auto v = index_slice(vec, lv); + if ( v && IndexExprWhen::evaluating > 0 ) + IndexExprWhen::results.emplace_back(v); + return v; + } + +ValPtr when_invoke__CPP(Func* f, std::vector args, Frame* frame, void* caller_addr) + { + auto trigger = frame->GetTrigger(); + + if ( trigger ) + { + Val* v = trigger->Lookup(caller_addr); + if ( v ) + return {NewRef{}, v}; + } + + frame->SetTriggerAssoc(caller_addr); + + auto res = f->Invoke(&args, frame); + if ( ! res ) + throw DelayedCallException(); + + return res; + } + ValPtr set_event__CPP(IDPtr g, ValPtr v, EventHandlerPtr& gh) { g->SetVal(v); diff --git a/src/script_opt/CPP/RuntimeOps.h b/src/script_opt/CPP/RuntimeOps.h index 02df049eb9..d4d8ba2034 100644 --- a/src/script_opt/CPP/RuntimeOps.h +++ b/src/script_opt/CPP/RuntimeOps.h @@ -33,6 +33,14 @@ extern ValPtr index_table__CPP(const TableValPtr& t, std::vector indices extern ValPtr index_vec__CPP(const VectorValPtr& vec, int index); extern ValPtr index_string__CPP(const StringValPtr& svp, std::vector indices); +// The same, but for indexing happening inside a "when" clause. +extern ValPtr when_index_table__CPP(const TableValPtr& t, std::vector indices); +extern ValPtr when_index_vec__CPP(const VectorValPtr& vec, int index); + +// For vector slices, we use the existing index_slice(), but we need a +// custom one for those occurring inside a "when" clause. +extern ValPtr when_index_slice__CPP(VectorVal* vec, const ListVal* lv); + // Calls out to the given script or BiF function. A separate function because // of the need to (1) construct the "args" vector using {} initializers, // but (2) needing to have the address of that vector. @@ -41,6 +49,18 @@ inline ValPtr invoke__CPP(Func* f, std::vector args, Frame* frame) return f->Invoke(&args, frame); } +// The same, but raises an interpreter exception if the function does +// not return a value. Used for calls inside "when" conditions. The +// last argument is the address of the calling function; we just need +// it to be distinct to the call, so we can associate a Trigger cache +// with it. +extern ValPtr when_invoke__CPP(Func* f, std::vector args, Frame* frame, void* caller_addr); + +// Thrown when a call inside a "when" delays. +class DelayedCallException : public InterpreterException + { + }; + // Assigns the given value to the given global. A separate function because // we also need to return the value, for use in assignment cascades. inline ValPtr set_global__CPP(IDPtr g, ValPtr v) diff --git a/src/script_opt/CPP/Stmts.cc b/src/script_opt/CPP/Stmts.cc index 1e3c89c460..8cc5a5f39c 100644 --- a/src/script_opt/CPP/Stmts.cc +++ b/src/script_opt/CPP/Stmts.cc @@ -66,6 +66,10 @@ void CPPCompile::GenStmt(const Stmt* s) GenSwitchStmt(static_cast(s)); break; + case STMT_WHEN: + GenWhenStmt(static_cast(s)); + break; + case STMT_FOR: GenForStmt(s->AsForStmt()); break; @@ -91,10 +95,6 @@ void CPPCompile::GenStmt(const Stmt* s) case STMT_FALLTHROUGH: break; - case STMT_WHEN: - ASSERT(0); - break; - default: reporter->InternalError("bad statement type in CPPCompile::GenStmt"); } @@ -163,23 +163,26 @@ void CPPCompile::GenReturnStmt(const ReturnStmt* r) { auto e = r->StmtExpr(); - if ( ! ret_type || ! e || e->GetType()->Tag() == TYPE_VOID || in_hook ) + if ( in_hook ) + Emit("return true;"); + + else if ( ! e && ret_type && ret_type->Tag() != TYPE_VOID ) + // This occurs for ExpressionlessReturnOkay() functions. + Emit("return nullptr;"); + + else if ( ! ret_type || ! e || e->GetType()->Tag() == TYPE_VOID ) + Emit("return;"); + + else { - if ( in_hook ) - Emit("return true;"); - else - Emit("return;"); + auto gt = ret_type->Tag() == TYPE_ANY ? GEN_VAL_PTR : GEN_NATIVE; + auto ret = GenExpr(e, gt); - return; + if ( e->GetType()->Tag() == TYPE_ANY ) + ret = GenericValPtrToGT(ret, ret_type, gt); + + Emit("return %s;", ret); } - - auto gt = ret_type->Tag() == TYPE_ANY ? GEN_VAL_PTR : GEN_NATIVE; - auto ret = GenExpr(e, gt); - - if ( e->GetType()->Tag() == TYPE_ANY ) - ret = GenericValPtrToGT(ret, ret_type, gt); - - Emit("return %s;", ret); } void CPPCompile::GenAddStmt(const ExprStmt* es) @@ -384,6 +387,68 @@ void CPPCompile::GenValueSwitchStmt(const Expr* e, const case_list* cases) Emit("}"); } +void CPPCompile::GenWhenStmt(const WhenStmt* w) + { + auto wi = w->Info(); + auto wl = wi->Lambda(); + + if ( ! wl ) + reporter->FatalError("cannot compile deprecated \"when\" statement"); + + auto is_return = wi->IsReturn() ? "true" : "false"; + auto timeout = wi->TimeoutExpr(); + auto timeout_val = timeout ? GenExpr(timeout, GEN_NATIVE) : "-1.0"; + auto loc = w->GetLocationInfo(); + + Emit("{ // begin a new scope for internal variables"); + + Emit("static WhenInfo* CPP__wi = nullptr;"); + Emit("static IDSet CPP__w_globals;"); + + NL(); + + Emit("if ( ! CPP__wi )"); + StartBlock(); + Emit("CPP__wi = new WhenInfo(%s);", is_return); + for ( auto& wg : wi->WhenExprGlobals() ) + Emit("CPP__w_globals.insert(find_global__CPP(\"%s\").get());", wg->Name()); + EndBlock(); + NL(); + + Emit("std::vector CPP__local_aggrs;"); + for ( auto l : wi->WhenExprLocals() ) + if ( IsAggr(l->GetType()) ) + Emit("CPP__local_aggrs.emplace_back(%s);", IDNameStr(l)); + + Emit("CPP__wi->Instantiate(%s);", GenExpr(wi->Lambda(), GEN_NATIVE)); + + // We need a new frame for the trigger to unambiguously associate + // with, in case we're called multiple times with our existing frame. + Emit("auto new_frame = make_intrusive(0, nullptr, nullptr);"); + Emit("auto curr_t = f__CPP->GetTrigger();"); + Emit("auto curr_assoc = f__CPP->GetTriggerAssoc();"); + Emit("new_frame->SetTrigger({NewRef{}, curr_t});"); + Emit("new_frame->SetTriggerAssoc(curr_assoc);"); + + Emit("auto t = new trigger::Trigger(CPP__wi, %s, CPP__w_globals, CPP__local_aggrs, " + "new_frame.get(), " + "nullptr);", + timeout_val); + + auto loc_str = util::fmt("%s:%d-%d", loc->filename, loc->first_line, loc->last_line); + Emit("t->SetName(\"%s\");", loc_str); + + if ( ret_type && ret_type->Tag() != TYPE_VOID ) + { + Emit("ValPtr retval = {NewRef{}, curr_t->Lookup(curr_assoc)};"); + Emit("if ( ! retval )"); + Emit("\tthrow DelayedCallException();"); + Emit("return %s;", GenericValPtrToGT("retval", ret_type, GEN_NATIVE)); + } + + Emit("}"); + } + void CPPCompile::GenForStmt(const ForStmt* f) { Emit("{ // begin a new scope for the internal loop vars"); diff --git a/src/script_opt/CPP/Util.cc b/src/script_opt/CPP/Util.cc index 1afbb0c4a3..510cfd7a00 100644 --- a/src/script_opt/CPP/Util.cc +++ b/src/script_opt/CPP/Util.cc @@ -38,13 +38,6 @@ string scope_prefix(int scope) bool is_CPP_compilable(const ProfileFunc* pf, const char** reason) { - if ( pf->NumWhenStmts() > 0 ) - { - if ( reason ) - *reason = "use of \"when\""; - return false; - } - auto body = pf->ProfiledBody(); if ( body && ! body->GetOptInfo()->is_free_of_conditionals ) { diff --git a/src/script_opt/CPP/maint/README b/src/script_opt/CPP/maint/README new file mode 100644 index 0000000000..f1383be5f0 --- /dev/null +++ b/src/script_opt/CPP/maint/README @@ -0,0 +1,85 @@ +This is a collection of scripts to support maintenance of -O gen-C++ +(and friends). They're oriented around running against the BTest test +suite, and are currently tailored for the lead maintainer's own environment. +The scripts all assume you're running them from build/ . + +If you make changes to the scripts, format them using + + shfmt -w -i 4 -ci *.sh + +(or set up "pre-commit" and use "pre-commit run -a") + +The maintenance workflow: + +1. Update this timestamp, so this file will be changed and you'll remember + to check in updates to the list of how the compiler currently fares + on various btests (see end of this doc): + + Thu May 12 12:54:10 PDT 2022 + +2. Run "find-test-files.sh" to generate a list (to stdout) of all of the + possible Zeek source files found in the test suite. + +3. For each such Zeek file, run "check-zeek.sh" to see whether Zeek can + parse it. This helps remove from further consideration difficult + tests (like those that have embedded input files, or multiple separate + scripts). + +4. "mkdir CPP-test" - a directory for holding results relating to C++ testing + +5. Run "check-CPP-gen.sh" for each Zeek file that passed "check-zeek.sh". + This will generate a corresponding file in CPP-test/out* indicating whether + "-O gen-C++" can successfully run on the input. Presently, it should + be able to do so for all of them. + +6. Copy ./src/zeek to ./zeek.HOLD. This is used to speed up recompilation used + in the next step. However, it's also a headache to do development to + fix a bug and then forget to update zeek.HOLD, which means you wind up + running the old version. You can combat that by removing ./zeek.HOLD + every time you start working on fixing a bug. + +7. Use the appended database to remove inputs that have known issues. + +8. For every input that survives that pruning, run "do-CPP-btest.sh". + This will generate C++ for the BTest, compile it, and run the result + to see if it succeeds. It populates CPP-test/diag* with the Btest + diagnostic output (empty means success). For non-empty output, + either fix the problem or update the database if it's not fixable. + +-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= + +Database Of Known Issues (be sure to keep sorted) + +These BTests won't successfully run due to the indicated issue: + + @if - has conditional code + bad-when - deliberately has old-style "when" without captures + command-line-error - a deliberate command-line error + complex-to-debug - hard-to-figure-out failure + deprecated - uses features deprecated for -O C++ + no-script - there's no actual script to compile + ZAM - meant specifically for -O ZAM + +../testing/btest/core/negative-time.test no-script +../testing/btest/core/pcap/dumper.zeek no-script +../testing/btest/core/pcap/input-error.zeek command-line-error +../testing/btest/core/proc-status-file.zeek no-script +../testing/btest/language/at-if-event.zeek @if +../testing/btest/language/at-if.zeek @if +../testing/btest/language/at-ifdef.zeek @if +../testing/btest/language/at-ifndef.zeek @if +../testing/btest/language/vector-in-operator.zeek deprecated +../testing/btest/language/when-aggregates.zeek bad-when +../testing/btest/opt/opt-files.zeek ZAM +../testing/btest/opt/opt-files2.zeek ZAM +../testing/btest/opt/opt-files3.zeek ZAM +../testing/btest/opt/opt-func.zeek ZAM +../testing/btest/opt/opt-func2.zeek ZAM +../testing/btest/opt/opt-func3.zeek ZAM +../testing/btest/scripts/base/protocols/dhcp/dhcp-ack-msg-types.zeek no-script +../testing/btest/scripts/base/protocols/dhcp/dhcp-all-msg-types.zeek no-script +../testing/btest/scripts/base/protocols/dhcp/dhcp-discover-msg-types.zeek no-script +../testing/btest/scripts/base/protocols/dhcp/inform.test no-script +../testing/btest/scripts/base/utils/active-http.test complex-to-debug +../testing/btest/scripts/policy/protocols/ssl/validate-certs.zeek no-script +../testing/btest/supervisor/config-bare-mode.zeek @if diff --git a/src/script_opt/CPP/maint/check-CPP-gen.sh b/src/script_opt/CPP/maint/check-CPP-gen.sh new file mode 100644 index 0000000000..e8c56fbf91 --- /dev/null +++ b/src/script_opt/CPP/maint/check-CPP-gen.sh @@ -0,0 +1,8 @@ +#! /bin/sh + +out=out.$(echo $1 | sed 's,\.\./,,;s,/,#,g') + +( + /bin/echo -n $1" " + (src/zeek -O gen-C++ --optimize-files=testing/btest --optimize-func="" $1 >&/dev/null && echo "success") || echo "fail" +) >CPP-test/$out 2>&1 diff --git a/src/script_opt/CPP/maint/check-zeek.sh b/src/script_opt/CPP/maint/check-zeek.sh new file mode 100644 index 0000000000..a61bd7ecb0 --- /dev/null +++ b/src/script_opt/CPP/maint/check-zeek.sh @@ -0,0 +1,4 @@ +#! /bin/sh + +/bin/echo -n $1" " +(src/zeek --parse-only $1 >&/dev/null && echo "success") || echo "fail" diff --git a/src/script_opt/CPP/maint/do-CPP-btest.sh b/src/script_opt/CPP/maint/do-CPP-btest.sh new file mode 100644 index 0000000000..3146db4997 --- /dev/null +++ b/src/script_opt/CPP/maint/do-CPP-btest.sh @@ -0,0 +1,30 @@ +#! /bin/sh + +rm -f CPP-gen.cc +cp zeek.HOLD src/zeek || ( + echo Need to create clean zeek.HOLD + exit 1 +) || exit 1 + +base=$(echo $1 | sed 's,\.\./,,;s,/,#,g') +rel_test=$(echo $1 | sed 's,.*testing/btest/,,') + +export ZEEK_GEN_CPP=1 +export ZEEK_CPP_DIR=$(pwd) +# export ZEEK_OPT_FUNCS="" +export ZEEK_OPT_FILES="testing/btest" + +( + cd ../testing/btest + ../../auxil/btest/btest $rel_test +) + +# export -n ZEEK_GEN_CPP ZEEK_CPP_DIR ZEEK_OPT_FUNCS ZEEK_OPT_FILES +export -n ZEEK_GEN_CPP ZEEK_CPP_DIR ZEEK_OPT_FILES + +ninja + +( + cd ../testing/btest + ../../auxil/btest/btest -a cpp -d -f ../../build/CPP-test/diag.$base $rel_test +) diff --git a/src/script_opt/CPP/maint/find-test-files.sh b/src/script_opt/CPP/maint/find-test-files.sh new file mode 100644 index 0000000000..1e35288517 --- /dev/null +++ b/src/script_opt/CPP/maint/find-test-files.sh @@ -0,0 +1,6 @@ +#! /bin/sh + +find ../testing/btest -type f | + egrep -v 'Baseline|\.tmp' | + egrep '\.(zeek|test)$' | + sort diff --git a/src/script_opt/Expr.cc b/src/script_opt/Expr.cc index 483b93efe9..b76f4b16bd 100644 --- a/src/script_opt/Expr.cc +++ b/src/script_opt/Expr.cc @@ -1863,7 +1863,7 @@ ExprPtr IndexExpr::Duplicate() { auto op1_d = op1->Duplicate(); auto op2_l = op2->Duplicate()->AsListExprPtr(); - return SetSucc(new IndexExpr(op1_d, op2_l, is_slice)); + return SetSucc(new IndexExpr(op1_d, op2_l, is_slice, is_inside_when)); } bool IndexExpr::HasReducedOps(Reducer* c) const diff --git a/src/script_opt/ProfileFunc.cc b/src/script_opt/ProfileFunc.cc index 1b8c31b019..a6057eb4e3 100644 --- a/src/script_opt/ProfileFunc.cc +++ b/src/script_opt/ProfileFunc.cc @@ -129,7 +129,15 @@ TraversalCode ProfileFunc::PreStmt(const Stmt* s) return TC_ABORTSTMT; case STMT_WHEN: + { ++num_when_stmts; + + auto w = s->AsWhenStmt(); + auto wi = w->Info(); + auto wl = wi ? wi->Lambda() : nullptr; + if ( wl ) + lambdas.push_back(wl.get()); + } break; case STMT_FOR: diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index 347b0d3348..18a5b9d33a 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -293,14 +293,14 @@ static void init_options() if ( analysis_options.only_funcs.empty() ) { - auto zo = getenv("ZEEK_FUNC_ONLY"); + auto zo = getenv("ZEEK_OPT_FUNCS"); if ( zo ) add_func_analysis_pattern(analysis_options, zo); } if ( analysis_options.only_files.empty() ) { - auto zo = getenv("ZEEK_FILE_ONLY"); + auto zo = getenv("ZEEK_OPT_FILES"); if ( zo ) add_file_analysis_pattern(analysis_options, zo); } @@ -370,6 +370,8 @@ static void use_CPP() if ( ! CPP_init_hook ) reporter->FatalError("no C++ functions available to use"); + int num_used = 0; + for ( auto& f : funcs ) { auto hash = f.Profile()->HashVal(); @@ -377,6 +379,8 @@ static void use_CPP() if ( s != compiled_scripts.end() ) { + ++num_used; + auto b = s->second.body; b->SetHash(hash); @@ -407,6 +411,9 @@ static void use_CPP() } } + if ( num_used == 0 ) + reporter->FatalError("no C++ functions found to use"); + // Now that we've loaded all of the compiled scripts // relevant for the AST, activate standalone ones. for ( auto cb : standalone_activations ) diff --git a/src/zeek.bif b/src/zeek.bif index a1fc14b26c..785f606973 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -3704,11 +3704,11 @@ function dump_packet%(pkt: pcap_packet, file_name: string%) : bool class LookupHostCallback : public zeek::detail::DNS_Mgr::LookupCallback { public: LookupHostCallback(zeek::detail::trigger::Trigger* arg_trigger, - const zeek::detail::CallExpr* arg_call, bool arg_lookup_name) + const void* arg_assoc, bool arg_lookup_name) { Ref(arg_trigger); trigger = arg_trigger; - call = arg_call; + assoc = arg_assoc; lookup_name = arg_lookup_name; } @@ -3721,7 +3721,7 @@ public: void Resolved(const std::string& name) override { zeek::Val* result = new zeek::StringVal(name); - trigger->Cache(call, result); + trigger->Cache(assoc, result); Unref(result); trigger->Release(); } @@ -3729,7 +3729,7 @@ public: void Resolved(zeek::TableValPtr addrs) override { // No Ref() for addrs. - trigger->Cache(call, addrs.get()); + trigger->Cache(assoc, addrs.get()); trigger->Release(); } @@ -3738,7 +3738,7 @@ public: if ( lookup_name ) { zeek::Val* result = new zeek::StringVal("<\?\?\?>"); - trigger->Cache(call, result); + trigger->Cache(assoc, result); Unref(result); } @@ -3747,7 +3747,7 @@ public: auto* lv = new zeek::ListVal(zeek::TYPE_ADDR); lv->Append(zeek::make_intrusive("0.0.0.0")); auto result = lv->ToSetVal(); - trigger->Cache(call, result.get()); + trigger->Cache(assoc, result.get()); Unref(lv); } @@ -3756,7 +3756,7 @@ public: private: zeek::detail::trigger::Trigger* trigger; - const zeek::detail::CallExpr* call; + const void* assoc; bool lookup_name; }; %%} @@ -3786,7 +3786,7 @@ function lookup_addr%(host: addr%) : string trigger->Hold(); zeek::detail::dns_mgr->LookupAddr(host->AsAddr(), - new LookupHostCallback(trigger, frame->GetCall(), true)); + new LookupHostCallback(trigger, frame->GetTriggerAssoc(), true)); return nullptr; %} @@ -3815,7 +3815,7 @@ function lookup_hostname_txt%(host: string%) : string trigger->Hold(); zeek::detail::dns_mgr->Lookup(host->CheckString(), T_TXT, - new LookupHostCallback(trigger, frame->GetCall(), true)); + new LookupHostCallback(trigger, frame->GetTriggerAssoc(), true)); return nullptr; %} @@ -3844,7 +3844,7 @@ function lookup_hostname%(host: string%) : addr_set trigger->Hold(); zeek::detail::dns_mgr->LookupHost(host->CheckString(), - new LookupHostCallback(trigger, frame->GetCall(), false)); + new LookupHostCallback(trigger, frame->GetTriggerAssoc(), false)); return nullptr; %} diff --git a/testing/btest/broker/store/handle-copy.zeek b/testing/btest/broker/store/handle-copy.zeek index cdb3b25f4c..d2d8f41e45 100644 --- a/testing/btest/broker/store/handle-copy.zeek +++ b/testing/btest/broker/store/handle-copy.zeek @@ -10,7 +10,7 @@ global result_count = 0; function print_keys(a: opaque of Broker::Store) { - when ( local s = Broker::keys(a) ) + when [a] ( local s = Broker::keys(a) ) { print "keys", s; ++result_count; diff --git a/testing/btest/broker/store/ops.zeek b/testing/btest/broker/store/ops.zeek index 30aa4f284f..6f5ee21cd6 100644 --- a/testing/btest/broker/store/ops.zeek +++ b/testing/btest/broker/store/ops.zeek @@ -12,7 +12,7 @@ global step: count = 0; function print_index(k: any) { - when ( local r = Broker::get(h, k) ) + when [k] ( local r = Broker::get(h, k) ) { step += 1; print fmt("[%d]", step), k, r$status, r$result; @@ -26,7 +26,7 @@ function print_index(k: any) function print_exists(k: any) { - when ( local r = Broker::exists(h, k) ) + when [k] ( local r = Broker::exists(h, k) ) { step += 1; print fmt("[%d]", step), k, r; @@ -40,7 +40,7 @@ function print_exists(k: any) function print_index_from_value(k: any, i: any) { - when ( local r = Broker::get_index_from_value(h, k, i) ) + when [k, i] ( local r = Broker::get_index_from_value(h, k, i) ) { step += 1; print fmt("[%d]", step), k, r$status, r$result; diff --git a/testing/btest/broker/store/sqlite.zeek b/testing/btest/broker/store/sqlite.zeek index b9b1225fce..76fe3fd3a6 100644 --- a/testing/btest/broker/store/sqlite.zeek +++ b/testing/btest/broker/store/sqlite.zeek @@ -12,7 +12,7 @@ global h: opaque of Broker::Store; function print_index(k: any) { - when ( local r = Broker::get(h, k) ) + when [k] ( local r = Broker::get(h, k) ) { print k, r$status, r$result; } diff --git a/testing/btest/language/more-closure-tests.zeek b/testing/btest/language/more-closure-tests.zeek index 2f6d6fbd91..cf0125f259 100644 --- a/testing/btest/language/more-closure-tests.zeek +++ b/testing/btest/language/more-closure-tests.zeek @@ -52,7 +52,7 @@ function die() { local h: addr = 127.0.0.1; - when ( local hname = lookup_addr(h) ) + when [h] ( local hname = lookup_addr(h) ) { print "lookup successful"; terminate(); diff --git a/testing/btest/language/returnwhen.zeek b/testing/btest/language/returnwhen.zeek index 8eddd4a30b..5fe60d44a1 100644 --- a/testing/btest/language/returnwhen.zeek +++ b/testing/btest/language/returnwhen.zeek @@ -43,7 +43,7 @@ event do_another() if ( ! done ) schedule 1sec { set_flag() }; - when ( local result = async_func("from do_another()") ) + when [local_dummy, anon] ( local result = async_func("from do_another()") ) { print "async_func() return result in do_another()", result; print local_dummy("from do_another() when block"); @@ -66,7 +66,7 @@ event zeek_init() schedule 1sec { set_flag() }; - when ( local result = async_func("from zeek_init()") ) + when [local_dummy, anon] ( local result = async_func("from zeek_init()") ) { print "async_func() return result in zeek_init()", result; print local_dummy("from zeek_init() when block"); diff --git a/testing/btest/language/when.zeek b/testing/btest/language/when.zeek index 38367dd8fb..d0d07b44fa 100644 --- a/testing/btest/language/when.zeek +++ b/testing/btest/language/when.zeek @@ -9,7 +9,7 @@ event zeek_init() { local h: addr = 127.0.0.1; - when ( local hname = lookup_addr(h) ) + when [h] ( local hname = lookup_addr(h) ) { print "lookup successful"; terminate(); @@ -21,9 +21,9 @@ event zeek_init() local to = 5sec; # Just checking that timeouts can use arbitrary expressions... - when ( local hname2 = lookup_addr(h) ) {} + when [h] ( local hname2 = lookup_addr(h) ) {} timeout to {} - when ( local hname3 = lookup_addr(h) ) {} + when [h] ( local hname3 = lookup_addr(h) ) {} timeout to + 2sec {} print "done"; diff --git a/testing/btest/scripts/base/protocols/dhcp/dhcp-ack-msg-types.btest b/testing/btest/scripts/base/protocols/dhcp/dhcp-ack-msg-types.zeek similarity index 100% rename from testing/btest/scripts/base/protocols/dhcp/dhcp-ack-msg-types.btest rename to testing/btest/scripts/base/protocols/dhcp/dhcp-ack-msg-types.zeek diff --git a/testing/btest/scripts/base/protocols/dhcp/dhcp-all-msg-types.btest b/testing/btest/scripts/base/protocols/dhcp/dhcp-all-msg-types.zeek similarity index 100% rename from testing/btest/scripts/base/protocols/dhcp/dhcp-all-msg-types.btest rename to testing/btest/scripts/base/protocols/dhcp/dhcp-all-msg-types.zeek diff --git a/testing/btest/scripts/base/protocols/dhcp/dhcp-discover-msg-types.btest b/testing/btest/scripts/base/protocols/dhcp/dhcp-discover-msg-types.zeek similarity index 100% rename from testing/btest/scripts/base/protocols/dhcp/dhcp-discover-msg-types.btest rename to testing/btest/scripts/base/protocols/dhcp/dhcp-discover-msg-types.zeek diff --git a/testing/btest/scripts/base/protocols/dhcp/dhcp-sub-opts.btest b/testing/btest/scripts/base/protocols/dhcp/dhcp-sub-opts.zeek similarity index 100% rename from testing/btest/scripts/base/protocols/dhcp/dhcp-sub-opts.btest rename to testing/btest/scripts/base/protocols/dhcp/dhcp-sub-opts.zeek diff --git a/testing/btest/scripts/base/protocols/rfb/vnc-scanner.bro b/testing/btest/scripts/base/protocols/rfb/vnc-scanner.zeek similarity index 100% rename from testing/btest/scripts/base/protocols/rfb/vnc-scanner.bro rename to testing/btest/scripts/base/protocols/rfb/vnc-scanner.zeek