diff --git a/CHANGES b/CHANGES index e60c581437..b154311b56 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,53 @@ +6.1.0-dev.152 | 2023-06-30 10:19:19 +0200 + + * documentation updates (Vern Paxson, Corelight) + + * updates to ZAM-specific BTest baseline (Vern Paxson, Corelight) + + * Address PR review feedback on zam-feature-complete (Vern Paxson, Corelight) + + * Updates to BTest baselines due to previous BTest tweaks (Vern Paxson, Corelight) + + * tweaks to BTests for ZAM feature completeness; new ZAM-only btest (Vern Paxson, Corelight) + + * removal of unused functionality and some follow-on simplifications (Vern Paxson, Corelight) + + * feature completeness for ZAM (Vern Paxson, Corelight) + + * -O gen-C++ tweaks to be compatible with ZAM changes (Vern Paxson, Corelight) + + * ZAM support for "when" statements (Vern Paxson, Corelight) + + * ZAM changes intermixed with lambda and "when" support (Vern Paxson, Corelight) + + * WhenStmt/WhenInfo restructuring in support of ZAM "when" statements (Vern Paxson, Corelight) + + * ZAM support for lambdas (Vern Paxson, Corelight) + + * ZAM internals have a notion of "captures" as global-like variables (Vern Paxson, Corelight) + + * AST profiling enhnacements in support of script optimization for lambdas/whens (Vern Paxson, Corelight) + + * refinements to LambdaExpr's to provide flexibility, support for ZVal captures (Vern Paxson, Corelight) + + * support in ScriptFunc class for ZVal-oriented vector of captures (Vern Paxson, Corelight) + + * simplifications to the Frame class now that it no longer has to support old-style captures (Vern Paxson, Corelight) + + * use Ingredients directly for constructing functions (Vern Paxson, Corelight) + + * the "Capture" struct is now a class (Vern Paxson, Corelight) + + * more debugging information when dumping script optimization data structures (Vern Paxson, Corelight) + + * bug fixes for script optimization intermediate forms (Vern Paxson, Corelight) + + * clarifying comments, interface tightening (Vern Paxson, Corelight) + + * added some class accessors/set-ers (Vern Paxson, Corelight) + + * Update doc submodule [nomail] [skip ci] (zeek-bot) + 6.1.0-dev.127 | 2023-06-29 11:22:58 -0700 * Move CMake template files to separate directory (Tim Wojtulewicz, Corelight) diff --git a/VERSION b/VERSION index 83c5d583cb..f7b11912b1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.1.0-dev.127 +6.1.0-dev.152 diff --git a/auxil/gen-zam b/auxil/gen-zam index 16841c9584..f62a0215f0 160000 --- a/auxil/gen-zam +++ b/auxil/gen-zam @@ -1 +1 @@ -Subproject commit 16841c95849d4d82f239bfb0c46bc217af368da2 +Subproject commit f62a0215f063bc768d7e85b4a49faeb07ca3cb14 diff --git a/src/Expr.cc b/src/Expr.cc index bcd9ac96ae..1d490c22f8 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -25,6 +25,7 @@ #include "zeek/digest.h" #include "zeek/module_util.h" #include "zeek/script_opt/ExprOptInfo.h" +#include "zeek/script_opt/ScriptOpt.h" namespace zeek::detail { @@ -4644,14 +4645,15 @@ void CallExpr::ExprDescribe(ODesc* d) const args->Describe(d); } -LambdaExpr::LambdaExpr(std::unique_ptr arg_ing, IDPList arg_outer_ids, +LambdaExpr::LambdaExpr(FunctionIngredientsPtr arg_ing, IDPList arg_outer_ids, std::string name, StmtPtr when_parent) : Expr(EXPR_LAMBDA) { ingredients = std::move(arg_ing); outer_ids = std::move(arg_outer_ids); - SetType(ingredients->GetID()->GetType()); + auto ingr_t = ingredients->GetID()->GetType(); + SetType(ingr_t); if ( ! CheckCaptures(when_parent) ) { @@ -4659,47 +4661,61 @@ LambdaExpr::LambdaExpr(std::unique_ptr arg_ing, IDPList arg return; } - // Install a dummy version of the function globally for use only - // when broker provides a closure. - auto dummy_func = make_intrusive(ingredients->GetID()); - dummy_func->AddBody(ingredients->Body(), ingredients->Inits(), ingredients->FrameSize(), - ingredients->Priority()); + // Install a primary version of the function globally. This is used + // by both broker (for transmitting closures) and script optimization + // (replacing its AST body with a compiled one). + primary_func = make_intrusive(ingredients->GetID()); + primary_func->SetOuterIDs(outer_ids); - dummy_func->SetOuterIDs(outer_ids); + // When we build the body, it will get updated with initialization + // statements. Update the ingredients to reflect the new body, + // and no more need for initializers. + primary_func->AddBody(*ingredients); + primary_func->SetScope(ingredients->Scope()); + ingredients->ClearInits(); - // Get the body's "string" representation. - ODesc d; - dummy_func->Describe(&d); - - for ( ;; ) - { - hash128_t h; - KeyedHash::Hash128(d.Bytes(), d.Len(), &h); - - my_name = "lambda_<" + std::to_string(h[0]) + ">"; - auto fullname = make_full_var_name(current_module.data(), my_name.data()); - const auto& id = global_scope()->Find(fullname); - - if ( id ) - // Just try again to make a unique lambda name. - // If two peer processes need to agree on the same - // lambda name, this assumes they're loading the same - // scripts and thus have the same hash collisions. - d.Add(" "); - else - break; - } + if ( name.empty() ) + BuildName(); + else + my_name = name; // Install that in the global_scope lambda_id = install_ID(my_name.c_str(), current_module.c_str(), true, false); // Update lamb's name - dummy_func->SetName(my_name.c_str()); + primary_func->SetName(my_name.c_str()); - auto v = make_intrusive(std::move(dummy_func)); + auto v = make_intrusive(primary_func); lambda_id->SetVal(std::move(v)); - lambda_id->SetType(ingredients->GetID()->GetType()); + lambda_id->SetType(ingr_t); lambda_id->SetConst(); + + captures = ingr_t->GetCaptures(); + + analyze_lambda(this); + } + +LambdaExpr::LambdaExpr(LambdaExpr* orig) : Expr(EXPR_LAMBDA) + { + primary_func = orig->primary_func; + ingredients = orig->ingredients; + lambda_id = orig->lambda_id; + my_name = orig->my_name; + private_captures = orig->private_captures; + + // We need to have our own copies of the outer IDs and captures so + // we can rename them when inlined. + for ( auto i : orig->outer_ids ) + outer_ids.append(i); + + if ( orig->captures ) + { + captures = std::vector{}; + for ( auto& c : *orig->captures ) + captures->push_back(c); + } + + SetType(orig->GetType()); } bool LambdaExpr::CheckCaptures(StmtPtr when_parent) @@ -4726,7 +4742,7 @@ bool LambdaExpr::CheckCaptures(StmtPtr when_parent) for ( const auto& c : *captures ) { - auto cid = c.id.get(); + auto cid = c.Id().get(); if ( ! cid ) // This happens for undefined/inappropriate @@ -4768,7 +4784,7 @@ bool LambdaExpr::CheckCaptures(StmtPtr when_parent) for ( const auto& c : *captures ) { - auto cid = c.id.get(); + auto cid = c.Id().get(); if ( cid && capture_is_matched.count(cid) == 0 ) { auto msg = util::fmt("%s is captured but not used inside %s", cid->Name(), desc); @@ -4784,6 +4800,32 @@ bool LambdaExpr::CheckCaptures(StmtPtr when_parent) return true; } +void LambdaExpr::BuildName() + { + // Get the body's "string" representation. + ODesc d; + primary_func->Describe(&d); + + for ( ;; ) + { + hash128_t h; + KeyedHash::Hash128(d.Bytes(), d.Len(), &h); + + my_name = "lambda_<" + std::to_string(h[0]) + ">"; + auto fullname = make_full_var_name(current_module.data(), my_name.data()); + const auto& id = global_scope()->Find(fullname); + + if ( id ) + // Just try again to make a unique lambda name. + // If two peer processes need to agree on the same + // lambda name, this assumes they're loading the same + // scripts and thus have the same hash collisions. + d.Add(" "); + else + break; + } + } + ScopePtr LambdaExpr::GetScope() const { return ingredients->Scope(); @@ -4792,12 +4834,23 @@ ScopePtr LambdaExpr::GetScope() const ValPtr LambdaExpr::Eval(Frame* f) const { auto lamb = make_intrusive(ingredients->GetID()); - lamb->AddBody(ingredients->Body(), ingredients->Inits(), ingredients->FrameSize(), - ingredients->Priority()); + StmtPtr body = primary_func->GetBodies()[0].stmts; + + if ( run_state::is_parsing ) + // We're evaluating this lambda at parse time, which happens + // for initializations. If we're doing script optimization + // then the current version of the body might be left in an + // inconsistent state (e.g., if it's replaced with ZAM code) + // causing problems if we execute this lambda subsequently. + // To avoid that problem, we duplicate the AST so it's + // distinct. + body = body->Duplicate(); + + lamb->AddBody(*ingredients, body); lamb->CreateCaptures(f); - // Set name to corresponding dummy func. + // Set name to corresponding master func. // Allows for lookups by the receiver. lamb->SetName(my_name.c_str()); @@ -4807,6 +4860,22 @@ ValPtr LambdaExpr::Eval(Frame* f) const void LambdaExpr::ExprDescribe(ODesc* d) const { d->Add(expr_name(Tag())); + + if ( captures && d->IsReadable() ) + { + d->Add("["); + + for ( auto& c : *captures ) + { + if ( &c != &(*captures)[0] ) + d->AddSP(", "); + + d->Add(c.Id()->Name()); + } + + d->Add("]"); + } + ingredients->Body()->Describe(d); } diff --git a/src/Expr.h b/src/Expr.h index 97127629b0..5b3f5ec6ef 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -28,8 +28,11 @@ namespace detail class Frame; class Scope; class FunctionIngredients; +class WhenInfo; using IDPtr = IntrusivePtr; using ScopePtr = IntrusivePtr; +using ScriptFuncPtr = IntrusivePtr; +using FunctionIngredientsPtr = std::shared_ptr; enum ExprTag : int { @@ -1452,12 +1455,17 @@ protected: class LambdaExpr final : public Expr { public: - LambdaExpr(std::unique_ptr ingredients, IDPList outer_ids, + LambdaExpr(FunctionIngredientsPtr ingredients, IDPList outer_ids, std::string name = "", StmtPtr when_parent = nullptr); const std::string& Name() const { return my_name; } + const IDPList& OuterIDs() const { return outer_ids; } - const FunctionIngredients& Ingredients() const { return *ingredients; } + + // Lambda's potentially have their own private copy of captures, + // to enable updates to the set during script optimization. + using CaptureList = std::vector; + const std::optional& GetCaptures() const { return captures; } ValPtr Eval(Frame* f) const override; TraversalCode Traverse(TraversalCallback* cb) const override; @@ -1466,19 +1474,43 @@ public: // Optimization-related: ExprPtr Duplicate() override; - ExprPtr Inline(Inliner* inl) override; + const ScriptFuncPtr& PrimaryFunc() const { return primary_func; } + + const FunctionIngredientsPtr& Ingredients() const { return ingredients; } + + bool IsReduced(Reducer* c) const override; + bool HasReducedOps(Reducer* c) const override; ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + StmtPtr ReduceToSingletons(Reducer* c) override; protected: + // Constructor used for script optimization. + LambdaExpr(LambdaExpr* orig); + void ExprDescribe(ODesc* d) const override; private: - bool CheckCaptures(StmtPtr when_parent); + friend class WhenInfo; - std::unique_ptr ingredients; + // "Private" captures are captures that correspond to "when" + // condition locals. These aren't true captures in that they + // don't come from the outer frame when the lambda is constructed, + // but they otherwise behave like captures in that they persist + // across function invocations. + void SetPrivateCaptures(const IDSet& pcaps) { private_captures = pcaps; } + + bool CheckCaptures(StmtPtr when_parent); + void BuildName(); + + void UpdateCaptures(Reducer* c); + + FunctionIngredientsPtr ingredients; + ScriptFuncPtr primary_func; IDPtr lambda_id; IDPList outer_ids; + std::optional captures; + IDSet private_captures; std::string my_name; }; diff --git a/src/Frame.cc b/src/Frame.cc index 642440d2c2..6d60a34401 100644 --- a/src/Frame.cc +++ b/src/Frame.cc @@ -139,11 +139,8 @@ static bool val_is_func(const ValPtr& v, ScriptFunc* func) return v->AsFunc() == func; } -broker::expected Frame::SerializeCopyFrame() +broker::expected Frame::Serialize() { - broker::vector rval; - rval.emplace_back(std::string("CopyFrame")); - broker::vector body; for ( int i = 0; i < size; ++i ) @@ -158,28 +155,18 @@ broker::expected Frame::SerializeCopyFrame() body.emplace_back(std::move(val_tuple)); } + broker::vector rval; rval.emplace_back(std::move(body)); return {std::move(rval)}; } -std::pair Frame::Unserialize(const broker::vector& data, - const std::optional& captures) +std::pair Frame::Unserialize(const broker::vector& data) { if ( data.size() == 0 ) return std::make_pair(true, nullptr); auto where = data.begin(); - - auto has_name = broker::get_if(*where); - if ( ! has_name ) - return std::make_pair(false, nullptr); - - std::advance(where, 1); - - if ( captures ) - ASSERT(*has_name == "CopyFrame"); - auto has_body = broker::get_if(*where); if ( ! has_body ) return std::make_pair(false, nullptr); diff --git a/src/Frame.h b/src/Frame.h index 7c081939c1..8b169b42df 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -53,6 +53,13 @@ public: */ Frame(int size, const ScriptFunc* func, const zeek::Args* fn_args); + /** + * Returns the size of the frame. + * + * @return the number of elements in the frame. + */ + int FrameSize() const { return size; } + /** * @param n the index to get. * @return the value at index *n* of the underlying array. @@ -165,16 +172,11 @@ public: Frame* CloneForTrigger() const; /** - * Serializes the frame in support of copy semantics for lambdas: - * - * [ "CopyFrame", serialized_values ] - * - * where serialized_values are two-element vectors. A serialized_value - * has the result of calling broker::data_to_val on the value in the - * first index, and an integer representing that value's type in the - * second index. + * Serializes the frame (only done for lambda/when captures) as a + * sequence of two-element vectors, the first element reflecting + * the frame value, the second its type. */ - broker::expected SerializeCopyFrame(); + broker::expected Serialize(); /** * Instantiates a Frame from a serialized one. @@ -182,13 +184,8 @@ public: * @return a pair in which the first item is the status of the serialization; * and the second is the unserialized frame with reference count +1, or * null if the serialization wasn't successful. - * - * The *captures* argument, if non-nil, specifies that the frame - * reflects captures with copy-semantics rather than deprecated - * reference semantics. */ - static std::pair - Unserialize(const broker::vector& data, const std::optional& captures); + static std::pair Unserialize(const broker::vector& data); // If the frame is run in the context of a trigger condition evaluation, // the trigger needs to be registered. @@ -239,7 +236,7 @@ private: */ int current_offset; - /** Frame used for captures (if any) with copy semantics. */ + /** Frame used for lambda/when captures. */ Frame* captures; /** Maps IDs to offsets into the "captures" frame. If the ID diff --git a/src/Func.cc b/src/Func.cc index fa08d81709..feca7bbdeb 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -123,6 +123,14 @@ std::string render_call_stack() return rval; } +void Func::AddBody(const detail::FunctionIngredients& ingr, detail::StmtPtr new_body) + { + if ( ! new_body ) + new_body = ingr.Body(); + + AddBody(new_body, ingr.Inits(), ingr.FrameSize(), ingr.Priority(), ingr.Groups()); + } + void Func::AddBody(detail::StmtPtr new_body, const std::vector& new_inits, size_t new_frame_size, int priority) { @@ -130,6 +138,13 @@ void Func::AddBody(detail::StmtPtr new_body, const std::vector& n AddBody(new_body, new_inits, new_frame_size, priority, groups); } +void Func::AddBody(detail::StmtPtr new_body, size_t new_frame_size) + { + std::vector no_inits; + std::set no_groups; + AddBody(new_body, no_inits, new_frame_size, 0, no_groups); + } + void Func::AddBody(detail::StmtPtr /* new_body */, const std::vector& /* new_inits */, size_t /* new_frame_size */, int /* priority */, const std::set& /* groups */) @@ -316,6 +331,15 @@ ScriptFunc::ScriptFunc(std::string _name, FuncTypePtr ft, std::vector b ScriptFunc::~ScriptFunc() { + if ( captures_vec ) + { + auto& cvec = *captures_vec; + auto& captures = *type->GetCaptures(); + for ( auto i = 0u; i < captures.size(); ++i ) + if ( captures[i].IsManaged() ) + ZVal::DeleteManagedType(cvec[i]); + } + delete captures_frame; delete captures_offset_mapping; } @@ -498,31 +522,77 @@ void ScriptFunc::CreateCaptures(Frame* f) if ( ! captures ) return; - // Create a private Frame to hold the values of captured variables, - // and a mapping from those variables to their offsets in the Frame. - delete captures_frame; - delete captures_offset_mapping; - captures_frame = new Frame(captures->size(), this, nullptr); - captures_offset_mapping = new OffsetMap; + // Create *either* a private Frame to hold the values of captured + // variables, and a mapping from those variables to their offsets + // in the Frame; *or* a ZVal frame if this script has a ZAM-compiled + // body. + ASSERT(bodies.size() == 1); + + if ( bodies[0].stmts->Tag() == STMT_ZAM ) + captures_vec = std::make_unique>(); + else + { + delete captures_frame; + delete captures_offset_mapping; + captures_frame = new Frame(captures->size(), this, nullptr); + captures_offset_mapping = new OffsetMap; + } int offset = 0; for ( const auto& c : *captures ) { - auto v = f->GetElementByID(c.id); + auto v = f->GetElementByID(c.Id()); if ( v ) { - if ( c.deep_copy || ! v->Modifiable() ) + if ( c.IsDeepCopy() || ! v->Modifiable() ) v = v->Clone(); - captures_frame->SetElement(offset, std::move(v)); + if ( captures_vec ) + // Don't use v->GetType() here, as that might + // be "any" and we need to convert. + captures_vec->push_back(ZVal(v, c.Id()->GetType())); + else + captures_frame->SetElement(offset, std::move(v)); } - (*captures_offset_mapping)[c.id->Name()] = offset; + else if ( captures_vec ) + captures_vec->push_back(ZVal()); + + if ( ! captures_vec ) + captures_offset_mapping->insert_or_assign(c.Id()->Name(), offset); + ++offset; } } +void ScriptFunc::CreateCaptures(std::unique_ptr> cvec) + { + const auto& captures = *type->GetCaptures(); + + ASSERT(cvec->size() == captures.size()); + ASSERT(bodies.size() == 1 && bodies[0].stmts->Tag() == STMT_ZAM); + + captures_vec = std::move(cvec); + + auto n = captures.size(); + for ( auto i = 0U; i < n; ++i ) + { + auto& c_i = captures[i]; + auto& cv_i = (*captures_vec)[i]; + + if ( c_i.IsDeepCopy() ) + { + auto& t = c_i.Id()->GetType(); + auto new_cv_i = cv_i.ToVal(t)->Clone(); + if ( c_i.IsManaged() ) + ZVal::DeleteManagedType(cv_i); + + cv_i = ZVal(new_cv_i, t); + } + } + } + void ScriptFunc::SetCaptures(Frame* f) { const auto& captures = type->GetCaptures(); @@ -536,7 +606,7 @@ void ScriptFunc::SetCaptures(Frame* f) int offset = 0; for ( const auto& c : *captures ) { - (*captures_offset_mapping)[c.id->Name()] = offset; + captures_offset_mapping->insert_or_assign(c.Id()->Name(), offset); ++offset; } } @@ -595,11 +665,32 @@ void ScriptFunc::ReplaceBody(const StmtPtr& old_body, StmtPtr new_body) bool ScriptFunc::DeserializeCaptures(const broker::vector& data) { - auto result = Frame::Unserialize(data, GetType()->GetCaptures()); + auto result = Frame::Unserialize(data); ASSERT(result.first); - SetCaptures(result.second.release()); + auto& f = result.second; + + if ( bodies[0].stmts->Tag() == STMT_ZAM ) + { + auto& captures = *type->GetCaptures(); + int n = f->FrameSize(); + + ASSERT(captures.size() == static_cast(n)); + + auto cvec = std::make_unique>(); + + for ( int i = 0; i < n; ++i ) + { + auto& f_i = f->GetElement(i); + cvec->push_back(ZVal(f_i, captures[i].Id()->GetType())); + } + + CreateCaptures(std::move(cvec)); + } + + else + SetCaptures(f.release()); return true; } @@ -608,6 +699,9 @@ FuncPtr ScriptFunc::DoClone() { // ScriptFunc could hold a closure. In this case a clone of it must // store a copy of this closure. + // + // We don't use make_intrusive<> directly because we're accessing + // a protected constructor. auto other = IntrusivePtr{AdoptRef{}, new ScriptFunc()}; CopyStateInto(other.get()); @@ -622,13 +716,43 @@ FuncPtr ScriptFunc::DoClone() *other->captures_offset_mapping = *captures_offset_mapping; } + if ( captures_vec ) + { + auto cv_i = captures_vec->begin(); + other->captures_vec = std::make_unique>(); + for ( auto& c : *type->GetCaptures() ) + { + // Need to clone cv_i. + auto& t_i = c.Id()->GetType(); + auto cv_i_val = cv_i->ToVal(t_i)->Clone(); + other->captures_vec->push_back(ZVal(cv_i_val, t_i)); + ++cv_i; + } + } + return other; } broker::expected ScriptFunc::SerializeCaptures() const { + if ( captures_vec ) + { + auto& cv = *captures_vec; + auto& captures = *type->GetCaptures(); + int n = captures_vec->size(); + auto temp_frame = make_intrusive(n, this, nullptr); + + for ( int i = 0; i < n; ++i ) + { + auto c_i = cv[i].ToVal(captures[i].Id()->GetType()); + temp_frame->SetElement(i, c_i); + } + + return temp_frame->Serialize(); + } + if ( captures_frame ) - return captures_frame->SerializeCopyFrame(); + return captures_frame->Serialize(); // No captures, return an empty vector. return broker::vector{}; diff --git a/src/Func.h b/src/Func.h index 06f2d26306..e1b8f7995b 100644 --- a/src/Func.h +++ b/src/Func.h @@ -44,6 +44,7 @@ using IDPtr = IntrusivePtr; using StmtPtr = IntrusivePtr; class ScriptFunc; +class FunctionIngredients; } // namespace detail @@ -114,14 +115,18 @@ public: return Invoke(&zargs); } - // Add a new event handler to an existing function (event). + // Various ways to add a new event handler to an existing function + // (event). The usual version to use is the first with its default + // parameter. All of the others are for use by script optimization, + // as is a non-default second parameter to the first method, which + // overrides the function body in "ingr". + void AddBody(const detail::FunctionIngredients& ingr, detail::StmtPtr new_body = nullptr); virtual void AddBody(detail::StmtPtr new_body, const std::vector& new_inits, size_t new_frame_size, int priority, const std::set& groups); - - // Add a new event handler to an existing function (event). - virtual void AddBody(detail::StmtPtr new_body, const std::vector& new_inits, - size_t new_frame_size, int priority = 0); + void AddBody(detail::StmtPtr new_body, const std::vector& new_inits, + size_t new_frame_size, int priority = 0); + void AddBody(detail::StmtPtr new_body, size_t new_frame_size); virtual void SetScope(detail::ScopePtr newscope); virtual detail::ScopePtr GetScope() const { return scope; } @@ -190,6 +195,17 @@ public: */ void CreateCaptures(Frame* f); + /** + * Uses the given set of ZVal's for captures. Note that this is + * different from the method above, which uses its argument to + * compute the captures, rather than here where they are pre-computed. + * + * Makes deep copies if required. + * + * @param cvec a vector of ZVal's corresponding to the captures. + */ + void CreateCaptures(std::unique_ptr> cvec); + /** * Returns the frame associated with this function for tracking * captures, or nil if there isn't one. @@ -198,6 +214,18 @@ public: */ Frame* GetCapturesFrame() const { return captures_frame; } + /** + * Returns the set of ZVal's used for captures. It's okay to modify + * these as long as memory-management is done for managed entries. + * + * @return internal vector of ZVal's kept for persisting captures + */ + auto& GetCapturesVec() const + { + ASSERT(captures_vec); + return *captures_vec; + } + // Same definition as in Frame.h. using OffsetMap = std::unordered_map; @@ -273,6 +301,7 @@ protected: /** * Uses the given frame for captures, and generates the * mapping from captured variables to offsets in the frame. + * Virtual so it can be modified for script optimization uses. * * @param f the frame holding the values of capture variables */ @@ -291,6 +320,9 @@ private: OffsetMap* captures_offset_mapping = nullptr; + // Captures when using ZVal block instead of a Frame. + std::unique_ptr> captures_vec; + // The most recently added/updated body ... StmtPtr current_body; @@ -332,8 +364,7 @@ struct CallInfo const zeek::Args& args; }; -// Struct that collects all the specifics defining a Func. Used for ScriptFuncs -// with closures. +// Class that collects all the specifics defining a Func. class FunctionIngredients { public: @@ -344,14 +375,19 @@ public: const IDPtr& GetID() const { return id; } const StmtPtr& Body() const { return body; } - void SetBody(StmtPtr _body) { body = std::move(_body); } const auto& Inits() const { return inits; } + void ClearInits() { inits.clear(); } + size_t FrameSize() const { return frame_size; } int Priority() const { return priority; } const ScopePtr& Scope() const { return scope; } const auto& Groups() const { return groups; } + // Used by script optimization to update lambda ingredients + // after compilation. + void SetFrameSize(size_t _frame_size) { frame_size = _frame_size; } + private: IDPtr id; StmtPtr body; @@ -362,6 +398,8 @@ private: std::set groups; }; +using FunctionIngredientsPtr = std::shared_ptr; + extern std::vector call_stack; /** diff --git a/src/Stmt.cc b/src/Stmt.cc index 5cfb7bb914..3b59f6aec6 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -52,7 +52,6 @@ const char* stmt_name(StmtTag t) "check-any-length", "compiled-C++", "ZAM", - "ZAM-resumption", "null", "assert", }; @@ -2005,35 +2004,7 @@ WhenInfo::WhenInfo(ExprPtr arg_cond, FuncType::CaptureList* arg_cl, bool arg_is_ if ( ! cl ) cl = new zeek::FuncType::CaptureList; - ProfileFunc cond_pf(cond.get()); - - when_expr_locals = cond_pf.Locals(); - when_expr_globals = cond_pf.AllGlobals(); - when_new_locals = cond_pf.WhenLocals(); - - // Make any when-locals part of our captures, if not already present, - // to enable sharing between the condition and the body/timeout code. - for ( auto& wl : when_new_locals ) - { - bool is_present = false; - - for ( auto& c : *cl ) - if ( c.id == wl ) - { - is_present = true; - break; - } - - if ( ! is_present ) - { - IDPtr wl_ptr = {NewRef{}, const_cast(wl)}; - cl->emplace_back(FuncType::Capture{wl_ptr, false}); - } - - // In addition, don't treat them as external locals that - // existed at the onset. - when_expr_locals.erase(wl); - } + BuildProfile(); // Create the internal lambda we'll use to manage the captures. static int num_params = 0; // to ensure each is distinct @@ -2049,12 +2020,39 @@ WhenInfo::WhenInfo(ExprPtr arg_cond, FuncType::CaptureList* arg_cl, bool arg_is_ if ( ! is_return ) lambda_ft->SetExpressionlessReturnOkay(true); + lambda_ft->SetCaptures(*cl); + auto id = current_scope()->GenerateTemporary("when-internal"); id->SetType(lambda_ft); push_scope(std::move(id), nullptr); - auto arg_id = install_ID(lambda_param_id.c_str(), current_module.c_str(), false, false); - arg_id->SetType(count_t); + param_id = install_ID(lambda_param_id.c_str(), current_module.c_str(), false, false); + param_id->SetType(count_t); + } + +WhenInfo::WhenInfo(const WhenInfo* orig) + { + if ( orig->cl ) + { + cl = new FuncType::CaptureList; + *cl = *orig->cl; + } + + cond = orig->OrigCond()->Duplicate(); + + // We don't duplicate these, as they'll be compiled separately. + s = orig->OrigBody(); + timeout_s = orig->OrigBody(); + + timeout = orig->OrigTimeout(); + if ( timeout ) + timeout = timeout->Duplicate(); + + lambda = cast_intrusive(orig->Lambda()->Duplicate()); + + is_return = orig->IsReturn(); + + BuildProfile(); } WhenInfo::WhenInfo(bool arg_is_return) : is_return(arg_is_return) @@ -2063,10 +2061,49 @@ WhenInfo::WhenInfo(bool arg_is_return) : is_return(arg_is_return) BuildInvokeElems(); } +void WhenInfo::BuildProfile() + { + ProfileFunc cond_pf(cond.get()); + + auto when_expr_locals_set = cond_pf.Locals(); + when_expr_globals = cond_pf.AllGlobals(); + when_new_locals = cond_pf.WhenLocals(); + + // Make any when-locals part of our captures, if not already present, + // to enable sharing between the condition and the body/timeout code. + for ( auto& wl : when_new_locals ) + { + bool is_present = false; + + for ( auto& c : *cl ) + if ( c.Id() == wl ) + { + is_present = true; + break; + } + + if ( ! is_present ) + { + IDPtr wl_ptr = {NewRef{}, const_cast(wl)}; + cl->emplace_back(FuncType::Capture{wl_ptr, false}); + } + + // In addition, don't treat them as external locals that + // existed at the onset. + when_expr_locals_set.erase(wl); + } + + for ( auto& w : when_expr_locals_set ) + { + // We need IDPtr versions of the locals so we can manipulate + // them during script optimization. + auto non_const_w = const_cast(w); + when_expr_locals.push_back({NewRef{}, non_const_w}); + } + } + void WhenInfo::Build(StmtPtr ws) { - lambda_ft->SetCaptures(*cl); - // Our general strategy is to construct a single lambda (so that // the values of captures are shared across all of its elements) // that's used for all three of the "when" components: condition, @@ -2086,10 +2123,13 @@ void WhenInfo::Build(StmtPtr ws) // First, the constants we'll need. BuildInvokeElems(); + if ( lambda ) + // No need to build the lambda. + return; + auto true_const = make_intrusive(val_mgr->True()); // Access to the parameter that selects which action we're doing. - auto param_id = lookup_ID(lambda_param_id.c_str(), current_module.c_str()); ASSERT(param_id); auto param = make_intrusive(param_id); @@ -2109,11 +2149,14 @@ void WhenInfo::Build(StmtPtr ws) auto shebang = make_intrusive(do_test, do_bodies, dummy_return); - auto ingredients = std::make_unique(current_scope(), shebang, + auto ingredients = std::make_shared(current_scope(), shebang, current_module); auto outer_ids = gather_outer_ids(pop_scope(), ingredients->Body()); - lambda = make_intrusive(std::move(ingredients), std::move(outer_ids), ws); + lambda = make_intrusive(std::move(ingredients), std::move(outer_ids), "", ws); + lambda->SetPrivateCaptures(when_new_locals); + + analyze_when_lambda(lambda.get()); } void WhenInfo::Instantiate(Frame* f) @@ -2205,8 +2248,7 @@ ValPtr WhenStmt::Exec(Frame* f, StmtFlowType& flow) std::vector local_aggrs; for ( auto& l : wi->WhenExprLocals() ) { - IDPtr l_ptr = {NewRef{}, const_cast(l)}; - auto v = f->GetElementByID(l_ptr); + auto v = f->GetElementByID(l); if ( v && v->Modifiable() ) local_aggrs.emplace_back(std::move(v)); } @@ -2226,6 +2268,23 @@ void WhenStmt::StmtDescribe(ODesc* d) const { Stmt::StmtDescribe(d); + auto cl = wi->Captures(); + if ( d->IsReadable() && ! cl->empty() ) + { + d->Add("["); + for ( auto& c : *cl ) + { + if ( &c != &(*cl)[0] ) + d->AddSP(","); + + if ( c.IsDeepCopy() ) + d->Add("copy "); + + d->Add(c.Id()->Name()); + } + d->Add("]"); + } + if ( d->IsReadable() ) d->Add("("); @@ -2267,32 +2326,13 @@ TraversalCode WhenStmt::Traverse(TraversalCallback* cb) const TraversalCode tc = cb->PreStmt(this); HANDLE_TC_STMT_PRE(tc); - auto wl = wi->Lambda(); + tc = wi->Lambda()->Traverse(cb); + HANDLE_TC_STMT_PRE(tc); - if ( wl ) + auto e = wi->TimeoutExpr(); + if ( e ) { - tc = wl->Traverse(cb); - HANDLE_TC_STMT_PRE(tc); - } - - else - { - tc = wi->OrigCond()->Traverse(cb); - HANDLE_TC_STMT_PRE(tc); - - tc = wi->OrigBody()->Traverse(cb); - HANDLE_TC_STMT_PRE(tc); - - if ( wi->OrigTimeoutStmt() ) - { - tc = wi->OrigTimeoutStmt()->Traverse(cb); - HANDLE_TC_STMT_PRE(tc); - } - } - - if ( wi->OrigTimeout() ) - { - tc = wi->OrigTimeout()->Traverse(cb); + tc = e->Traverse(cb); HANDLE_TC_STMT_PRE(tc); } diff --git a/src/Stmt.h b/src/Stmt.h index dc1a33a45b..614d39d5bc 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -450,6 +450,7 @@ public: ReturnStmt(ExprPtr e, bool ignored); // Optimization-related: + bool IsReduced(Reducer* c) const override; StmtPtr DoReduce(Reducer* c) override; bool NoFlowAfter(bool ignore_break) const override { return true; } @@ -575,6 +576,9 @@ public: // Takes ownership of the CaptureList. WhenInfo(ExprPtr cond, FuncType::CaptureList* cl, bool is_return); + // Used for duplication to support inlining. + WhenInfo(const WhenInfo* orig); + // Constructor used by script optimization to create a stub. WhenInfo(bool is_return); @@ -615,6 +619,7 @@ public: StmtPtr TimeoutStmt(); ExprPtr TimeoutExpr() const { return timeout; } + void SetTimeoutExpr(ExprPtr e) { timeout = std::move(e); } double TimeoutVal(Frame* f); FuncType::CaptureList* Captures() { return cl; } @@ -624,10 +629,22 @@ public: // The locals and globals used in the conditional expression // (other than newly introduced locals), necessary for registering // the associated triggers for when their values change. - const IDSet& WhenExprLocals() const { return when_expr_locals; } - const IDSet& WhenExprGlobals() const { return when_expr_globals; } + const auto& WhenExprLocals() const { return when_expr_locals; } + const auto& WhenExprGlobals() const { return when_expr_globals; } + + // The locals introduced in the conditional expression. + const auto& WhenNewLocals() const { return when_new_locals; } + + // Used for script optimization when in-lining needs to revise + // identifiers. + bool HasUnreducedIDs(Reducer* c) const; + void UpdateIDs(Reducer* c); private: + // Profile the original AST elements to extract things like + // globals and locals used. + void BuildProfile(); + // Build those elements we'll need for invoking our lambda. void BuildInvokeElems(); @@ -639,8 +656,10 @@ private: bool is_return = false; - // The name of parameter passed to the lambda. + // The name of parameter passed to the lambda, and the corresponding + // identifier. std::string lambda_param_id; + IDPtr param_id; // The expression for constructing the lambda, and its type. LambdaExprPtr lambda; @@ -661,7 +680,7 @@ private: ConstExprPtr two_const; ConstExprPtr three_const; - IDSet when_expr_locals; + std::vector when_expr_locals; IDSet when_expr_globals; // Locals introduced via "local" in the "when" clause itself. @@ -684,7 +703,7 @@ public: StmtPtr TimeoutBody() const { return wi->TimeoutStmt(); } bool IsReturn() const { return wi->IsReturn(); } - const WhenInfo* Info() const { return wi; } + WhenInfo* Info() const { return wi; } void StmtDescribe(ODesc* d) const override; @@ -692,9 +711,9 @@ public: // Optimization-related: StmtPtr Duplicate() override; - void Inline(Inliner* inl) override; bool IsReduced(Reducer* c) const override; + StmtPtr DoReduce(Reducer* c) override; private: WhenInfo* wi; diff --git a/src/StmtEnums.h b/src/StmtEnums.h index 75f500f91c..f28025108a 100644 --- a/src/StmtEnums.h +++ b/src/StmtEnums.h @@ -31,7 +31,6 @@ enum StmtTag STMT_CHECK_ANY_LEN, // internal reduced statement STMT_CPP, // compiled C++ STMT_ZAM, // a ZAM function body - STMT_ZAM_RESUMPTION, // resumes ZAM execution for "when" statements STMT_NULL, STMT_ASSERT, #define NUM_STMTS (int(STMT_ASSERT) + 1) diff --git a/src/Trigger.h b/src/Trigger.h index 71362ed511..2de6a4686b 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -81,6 +81,8 @@ public: // if the Val is null or it's disabled. The cache is managed using // void*'s so that the value can be associated with either a CallExpr // (for interpreted execution) or a C++ function (for compiled-to-C++). + // + // Lookup() returned value must be Ref()'d if you want to hang onto it. bool Cache(const void* obj, Val* val); Val* Lookup(const void* obj); diff --git a/src/Type.cc b/src/Type.cc index c07a7b5fec..5f041f58dd 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -699,6 +699,15 @@ TypePtr SetType::ShallowClone() SetType::~SetType() = default; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +FuncType::Capture::Capture(detail::IDPtr _id, bool _deep_copy) + : id(std::move(_id)), deep_copy(_deep_copy) + { + is_managed = id ? ZVal::IsManagedType(id->GetType()) : false; + } +#pragma GCC diagnostic pop + FuncType::FuncType(RecordTypePtr arg_args, TypePtr arg_yield, FunctionFlavor arg_flavor) : Type(TYPE_FUNC), args(std::move(arg_args)), arg_types(make_intrusive()), yield(std::move(arg_yield)) @@ -788,9 +797,12 @@ bool FuncType::CheckArgs(const std::vector& args, bool is_init, bool do if ( my_args.size() != args.size() ) { if ( do_warn ) + { Warn(util::fmt("Wrong number of arguments for function. Expected %zu, got %zu.", args.size(), my_args.size())); - const_cast(this)->reported_error = true; + const_cast(this)->reported_error = true; + } + return false; } diff --git a/src/Type.h b/src/Type.h index 020ab6e358..056645158b 100644 --- a/src/Type.h +++ b/src/Type.h @@ -511,10 +511,36 @@ public: /** * A single lambda "capture" (outer variable used in a lambda's body). */ - struct Capture + class Capture { - detail::IDPtr id; - bool deep_copy; + public: + Capture(detail::IDPtr _id, bool _deep_copy); + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + Capture(const Capture&) = default; + Capture(Capture&&) = default; + Capture& operator=(const Capture&) = default; + Capture& operator=(Capture&&) = default; + ~Capture() = default; + + auto& Id() const { return id; } + bool IsDeepCopy() const { return deep_copy; } + bool IsManaged() const { return is_managed; } + + // For script optimization: + void SetID(detail::IDPtr new_id) { id = std::move(new_id); } +#pragma GCC diagnostic pop + + [[deprecated( + "Remove in v7.1. Use non-default constructor and associated accessors.")]] detail:: + IDPtr id; + [[deprecated( + "Remove in v7.1. Use non-default constructor and associated accessors.")]] bool + deep_copy; + [[deprecated( + "Remove in v7.1. Use non-default constructor and associated accessors.")]] bool + is_managed; }; using CaptureList = std::vector; diff --git a/src/Val.cc b/src/Val.cc index 0288cc1738..6c9e214bc2 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2787,6 +2787,11 @@ void TableVal::InitDefaultFunc(detail::Frame* f) def_val = def_attr->GetExpr()->Eval(f); } +void TableVal::InitDefaultVal(ValPtr _def_val) + { + def_val = std::move(_def_val); + } + void TableVal::InitTimer(double delay) { timer = new TableValTimer(this, run_state::network_time + delay); diff --git a/src/Val.h b/src/Val.h index 75dd006df3..6909f4807c 100644 --- a/src/Val.h +++ b/src/Val.h @@ -968,9 +968,13 @@ public: // If the &default attribute is not a function, or the function has // already been initialized, this does nothing. Otherwise, evaluates - // the function in the frame allowing it to capture its closure. + // the function in the frame, allowing it to capture its closure. void InitDefaultFunc(detail::Frame* f); + // An alternative that assigns the default value directly. Used + // by ZAM compilation. + void InitDefaultVal(ValPtr def_val); + void ClearTimer(detail::Timer* t) { if ( timer == t ) diff --git a/src/Var.cc b/src/Var.cc index 70540cef48..f125540cac 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -768,11 +768,10 @@ TraversalCode OuterIDBindingFinder::PreStmt(const Stmt* stmt) if ( stmt->Tag() != STMT_WHEN ) return TC_CONTINUE; - // The semantics of identifiers for the "when" statement are those - // of the lambda it's transformed into. - auto ws = static_cast(stmt); - ws->Info()->Lambda()->Traverse(this); + + for ( auto& cl : ws->Info()->WhenExprLocals() ) + outer_id_references.insert(const_cast(cl.get())); return TC_ABORTSTMT; } @@ -789,18 +788,19 @@ TraversalCode OuterIDBindingFinder::PreExpr(const Expr* expr) if ( expr->Tag() != EXPR_NAME ) return TC_CONTINUE; - auto* e = static_cast(expr); + auto e = static_cast(expr); + auto id = e->Id(); - if ( e->Id()->IsGlobal() ) + if ( id->IsGlobal() ) return TC_CONTINUE; for ( const auto& scope : scopes ) - if ( scope->Find(e->Id()->Name()) ) + if ( scope->Find(id->Name()) ) // Shadowing is not allowed, so if it's found at inner scope, it's // not something we have to worry about also being at outer scope. return TC_CONTINUE; - outer_id_references.insert(e->Id()); + outer_id_references.insert(id); return TC_CONTINUE; } @@ -845,9 +845,7 @@ void end_func(StmtPtr body, const char* module_name, bool free_of_conditionals) id->SetConst(); } - id->GetVal()->AsFunc()->AddBody(ingredients->Body(), ingredients->Inits(), - ingredients->FrameSize(), ingredients->Priority(), - ingredients->Groups()); + id->GetVal()->AsFunc()->AddBody(*ingredients); script_coverage_mgr.AddFunction(id, ingredients->Body()); diff --git a/src/Var.h b/src/Var.h index 51a6543202..29cd2ce604 100644 --- a/src/Var.h +++ b/src/Var.h @@ -38,7 +38,7 @@ extern void add_global(const IDPtr& id, TypePtr t, InitClass c, ExprPtr init, extern StmtPtr add_local(IDPtr id, TypePtr t, InitClass c, ExprPtr init, std::unique_ptr> attr, DeclType dt); -extern ExprPtr add_and_assign_local(IDPtr id, ExprPtr init, ValPtr val = nullptr); +extern ExprPtr add_and_assign_local(IDPtr id, ExprPtr init, ValPtr val); extern void add_type(ID* id, TypePtr t, std::unique_ptr> attr); diff --git a/src/parse.y b/src/parse.y index d16fc5042c..b9a80e50a9 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1566,7 +1566,7 @@ lambda_body: // Gather the ingredients for a Func from the // current scope. - auto ingredients = std::make_unique( + auto ingredients = std::make_shared( current_scope(), IntrusivePtr{AdoptRef{}, $3}, current_module.c_str()); auto outer_ids = gather_outer_ids(pop_scope(), ingredients->Body()); @@ -1640,9 +1640,7 @@ capture: delete [] $2; - $$ = new FuncType::Capture; - $$->id = id; - $$->deep_copy = $1; + $$ = new FuncType::Capture(id, $1); } ; diff --git a/src/script_opt/CPP/DeclFunc.cc b/src/script_opt/CPP/DeclFunc.cc index 12bedf82b2..b53ea918ad 100644 --- a/src/script_opt/CPP/DeclFunc.cc +++ b/src/script_opt/CPP/DeclFunc.cc @@ -29,8 +29,8 @@ void CPPCompile::DeclareLambda(const LambdaExpr* l, const ProfileFunc* pf) ASSERT(is_CPP_compilable(pf)); auto lname = Canonicalize(l->Name().c_str()) + "_lb"; - auto body = l->Ingredients().Body(); - auto l_id = l->Ingredients().GetID(); + auto body = l->Ingredients()->Body(); + auto l_id = l->Ingredients()->GetID(); auto& ids = l->OuterIDs(); for ( auto id : ids ) diff --git a/src/script_opt/CPP/Driver.cc b/src/script_opt/CPP/Driver.cc index de6396a19a..a93b0f603b 100644 --- a/src/script_opt/CPP/Driver.cc +++ b/src/script_opt/CPP/Driver.cc @@ -77,6 +77,12 @@ void CPPCompile::Compile(bool report_uncompilable) continue; } + if ( is_when_lambda(f) ) + { + func.SetSkip(true); + continue; + } + const char* reason; if ( IsCompilable(func, &reason) ) { @@ -150,7 +156,7 @@ void CPPCompile::Compile(bool report_uncompilable) for ( const auto& l : pfs.Lambdas() ) { const auto& n = l->Name(); - const auto body = l->Ingredients().Body().get(); + const auto body = l->Ingredients()->Body().get(); if ( lambda_ASTs.count(n) > 0 ) // Reuse previous body. body_names[body] = body_names[lambda_ASTs[n]]; @@ -176,7 +182,7 @@ void CPPCompile::Compile(bool report_uncompilable) continue; CompileLambda(l, pfs.ExprProf(l).get()); - lambda_ASTs[n] = l->Ingredients().Body().get(); + lambda_ASTs[n] = l->Ingredients()->Body().get(); } NL(); diff --git a/src/script_opt/CPP/Exprs.cc b/src/script_opt/CPP/Exprs.cc index c9cde5ab0c..90e7b0bd8d 100644 --- a/src/script_opt/CPP/Exprs.cc +++ b/src/script_opt/CPP/Exprs.cc @@ -1301,7 +1301,7 @@ string CPPCompile::GenLambdaClone(const LambdaExpr* l, bool all_deep) if ( captures && ! IsNativeType(id_t) ) { for ( const auto& c : *captures ) - if ( id == c.id && (c.deep_copy || all_deep) ) + if ( id == c.Id() && (c.IsDeepCopy() || all_deep) ) arg = string("cast_intrusive<") + TypeName(id_t) + ">(" + arg + "->Clone())"; } diff --git a/src/script_opt/CPP/GenFunc.cc b/src/script_opt/CPP/GenFunc.cc index d6076453cc..9971ada0cf 100644 --- a/src/script_opt/CPP/GenFunc.cc +++ b/src/script_opt/CPP/GenFunc.cc @@ -23,8 +23,8 @@ void CPPCompile::CompileFunc(const FuncInfo& func) void CPPCompile::CompileLambda(const LambdaExpr* l, const ProfileFunc* pf) { auto lname = Canonicalize(l->Name().c_str()) + "_lb"; - auto body = l->Ingredients().Body(); - auto l_id = l->Ingredients().GetID(); + auto body = l->Ingredients()->Body(); + auto l_id = l->Ingredients()->GetID(); auto& ids = l->OuterIDs(); DefineBody(l_id->GetType(), pf, lname, body, &ids, FUNC_FLAVOR_FUNCTION); diff --git a/src/script_opt/CPP/RuntimeOps.cc b/src/script_opt/CPP/RuntimeOps.cc index b74ac496e7..28fdea66d9 100644 --- a/src/script_opt/CPP/RuntimeOps.cc +++ b/src/script_opt/CPP/RuntimeOps.cc @@ -105,7 +105,7 @@ ValPtr when_invoke__CPP(Func* f, std::vector args, Frame* frame, void* c auto res = f->Invoke(&args, frame); if ( ! res ) - throw DelayedCallException(); + throw CPPDelayedCallException(); return res; } diff --git a/src/script_opt/CPP/RuntimeOps.h b/src/script_opt/CPP/RuntimeOps.h index 8ed8622941..9573e0b3fd 100644 --- a/src/script_opt/CPP/RuntimeOps.h +++ b/src/script_opt/CPP/RuntimeOps.h @@ -80,7 +80,7 @@ inline ValPtr invoke__CPP(Func* f, std::vector args, Frame* frame) extern ValPtr when_invoke__CPP(Func* f, std::vector args, Frame* frame, void* caller_addr); // Thrown when a call inside a "when" delays. -class DelayedCallException : public InterpreterException +class CPPDelayedCallException : public InterpreterException { }; diff --git a/src/script_opt/CPP/Stmts.cc b/src/script_opt/CPP/Stmts.cc index fdee490711..b9666573a3 100644 --- a/src/script_opt/CPP/Stmts.cc +++ b/src/script_opt/CPP/Stmts.cc @@ -420,9 +420,9 @@ void CPPCompile::GenWhenStmt(const WhenStmt* w) NL(); Emit("std::vector CPP__local_aggrs;"); - for ( auto l : wi->WhenExprLocals() ) + for ( auto& l : wi->WhenExprLocals() ) if ( IsAggr(l->GetType()) ) - Emit("CPP__local_aggrs.emplace_back(%s);", IDNameStr(l)); + Emit("CPP__local_aggrs.emplace_back(%s);", IDNameStr(l.get())); Emit("CPP__wi->Instantiate(%s);", GenExpr(wi->Lambda(), GEN_NATIVE)); @@ -446,7 +446,7 @@ void CPPCompile::GenWhenStmt(const WhenStmt* w) { Emit("ValPtr retval = {NewRef{}, curr_t->Lookup(curr_assoc)};"); Emit("if ( ! retval )"); - Emit("\tthrow DelayedCallException();"); + Emit("\tthrow CPPDelayedCallException();"); Emit("return %s;", GenericValPtrToGT("retval", ret_type, GEN_NATIVE)); } diff --git a/src/script_opt/Expr.cc b/src/script_opt/Expr.cc index b2e9880056..dbf40a1991 100644 --- a/src/script_opt/Expr.cc +++ b/src/script_opt/Expr.cc @@ -1851,7 +1851,12 @@ ExprPtr AssignExpr::ReduceToSingleton(Reducer* c, StmtPtr& red_stmt) if ( val ) return make_intrusive(val); - return op1->AsRefExprPtr()->GetOp1(); + auto lhs = op1->AsRefExprPtr()->GetOp1(); + StmtPtr lhs_stmt; + auto new_op1 = lhs->ReduceToSingleton(c, lhs_stmt); + red_stmt = MergeStmts(red_stmt, lhs_stmt); + + return new_op1; } ExprPtr IndexSliceAssignExpr::Duplicate() @@ -2337,7 +2342,7 @@ ExprPtr CallExpr::Duplicate() auto func_type = func->GetType(); auto in_hook = func_type->AsFuncType()->Flavor() == FUNC_FLAVOR_HOOK; - return SetSucc(new CallExpr(func_d, args_d, in_hook)); + return SetSucc(new CallExpr(func_d, args_d, in_hook, in_when)); } ExprPtr CallExpr::Inline(Inliner* inl) @@ -2368,7 +2373,14 @@ bool CallExpr::HasReducedOps(Reducer* c) const if ( ! func->IsSingleton(c) ) return NonReduced(this); - return args->HasReducedOps(c); + // We don't use args->HasReducedOps() here because for ListExpr's + // the method has some special-casing that isn't germane for calls. + + for ( const auto& expr : args->Exprs() ) + if ( ! expr->IsSingleton(c) ) + return false; + + return true; } ExprPtr CallExpr::Reduce(Reducer* c, StmtPtr& red_stmt) @@ -2386,9 +2398,7 @@ ExprPtr CallExpr::Reduce(Reducer* c, StmtPtr& red_stmt) if ( ! func->IsSingleton(c) ) func = func->ReduceToSingleton(c, red_stmt); - StmtPtr red2_stmt; - // We assume that ListExpr won't transform itself fundamentally. - (void)args->Reduce(c, red2_stmt); + StmtPtr red2_stmt = args->ReduceToSingletons(c); // ### could check here for (1) pure function, and (2) all // arguments constants, and call it to fold right now. @@ -2415,23 +2425,60 @@ StmtPtr CallExpr::ReduceToSingletons(Reducer* c) ExprPtr LambdaExpr::Duplicate() { - auto ingr = std::make_unique(*ingredients); - ingr->SetBody(ingr->Body()->Duplicate()); - return SetSucc(new LambdaExpr(std::move(ingr), outer_ids)); + return SetSucc(new LambdaExpr(this)); } -ExprPtr LambdaExpr::Inline(Inliner* inl) +bool LambdaExpr::IsReduced(Reducer* c) const { - // Don't inline these, we currently don't get the closure right. - return ThisPtr(); + if ( ! captures ) + return true; + + for ( auto& cp : *captures ) + { + auto& cid = cp.Id(); + + if ( private_captures.count(cid.get()) == 0 && ! c->ID_IsReduced(cid) ) + return NonReduced(this); + } + + return true; + } + +bool LambdaExpr::HasReducedOps(Reducer* c) const + { + return IsReduced(c); } ExprPtr LambdaExpr::Reduce(Reducer* c, StmtPtr& red_stmt) { if ( c->Optimizing() ) return ThisPtr(); - else - return AssignToTemporary(c, red_stmt); + + UpdateCaptures(c); + + return AssignToTemporary(c, red_stmt); + } + +StmtPtr LambdaExpr::ReduceToSingletons(Reducer* c) + { + UpdateCaptures(c); + return nullptr; + } + +void LambdaExpr::UpdateCaptures(Reducer* c) + { + if ( captures ) + { + for ( auto& cp : *captures ) + { + auto& cid = cp.Id(); + + if ( private_captures.count(cid.get()) == 0 ) + cp.SetID(c->UpdateID(cid)); + } + + c->UpdateIDs(&outer_ids); + } } ExprPtr EventExpr::Duplicate() @@ -2558,11 +2605,14 @@ StmtPtr ListExpr::ReduceToSingletons(Reducer* c) loop_over_list(exprs, i) { - if ( exprs[i]->IsSingleton(c) ) + auto& e_i = exprs[i]; + + if ( e_i->IsSingleton(c) ) continue; StmtPtr e_stmt; - auto old = exprs.replace(i, exprs[i]->Reduce(c, e_stmt).release()); + auto new_e_i = e_i->ReduceToSingleton(c, e_stmt); + auto old = exprs.replace(i, new_e_i.release()); Unref(old); if ( e_stmt ) @@ -2696,6 +2746,13 @@ void InlineExpr::ExprDescribe(ODesc* d) const { d->Add("inline("); args->Describe(d); + d->Add(")("); + for ( auto& p : params ) + { + if ( &p != ¶ms[0] ) + d->AddSP(","); + d->Add(p->Name()); + } d->Add("){"); body->Describe(d); d->Add("}"); diff --git a/src/script_opt/GenIDDefs.cc b/src/script_opt/GenIDDefs.cc index d12eb50afd..222272a431 100644 --- a/src/script_opt/GenIDDefs.cc +++ b/src/script_opt/GenIDDefs.cc @@ -196,12 +196,6 @@ TraversalCode GenIDDefs::PreStmt(const Stmt* s) return TC_ABORTSTMT; } - case STMT_WHEN: - { - // ### punt on these for now, need to reflect on bindings. - return TC_ABORTSTMT; - } - default: return TC_CONTINUE; } diff --git a/src/script_opt/Inline.cc b/src/script_opt/Inline.cc index 311b771823..ebd7d5f0b9 100644 --- a/src/script_opt/Inline.cc +++ b/src/script_opt/Inline.cc @@ -115,16 +115,21 @@ void Inliner::Analyze() const auto& func = func_ptr.get(); const auto& body = f.Body(); + if ( ! should_analyze(func_ptr, body) ) + continue; + // Candidates are non-event, non-hook, non-recursive, - // non-compiled functions ... that don't use lambdas or when's, - // since we don't currently compute the closures/frame - // sizes for them correctly, and more fundamentally since - // we don't compile them and hence inlining them will - // make the parent non-compilable. - if ( should_analyze(func_ptr, body) && func->Flavor() == FUNC_FLAVOR_FUNCTION && - non_recursive_funcs.count(func) > 0 && f.Profile()->NumLambdas() == 0 && - f.Profile()->NumWhenStmts() == 0 && body->Tag() != STMT_CPP ) - inline_ables.insert(func); + // non-compiled functions ... + if ( func->Flavor() != FUNC_FLAVOR_FUNCTION ) + continue; + + if ( non_recursive_funcs.count(func) == 0 ) + continue; + + if ( body->Tag() == STMT_CPP ) + continue; + + inline_ables.insert(func); } for ( auto& f : funcs ) @@ -165,6 +170,11 @@ ExprPtr Inliner::CheckForInlining(CallExprPtr c) // We don't inline indirect calls. return c; + if ( c->IsInWhen() ) + // Don't inline these, as doing so requires propagating + // the in-when attribute to the inlined function body. + return c; + auto n = f->AsNameExpr(); auto func = n->Id(); diff --git a/src/script_opt/ProfileFunc.cc b/src/script_opt/ProfileFunc.cc index 1da75df4f0..015c395994 100644 --- a/src/script_opt/ProfileFunc.cc +++ b/src/script_opt/ProfileFunc.cc @@ -28,7 +28,23 @@ ProfileFunc::ProfileFunc(const Func* func, const StmtPtr& body, bool _abs_rec_fi profiled_func = func; profiled_body = body.get(); abs_rec_fields = _abs_rec_fields; - Profile(func->GetType().get(), body); + + auto ft = func->GetType()->AsFuncType(); + auto& fcaps = ft->GetCaptures(); + + if ( fcaps ) + { + int offset = 0; + + for ( auto& c : *fcaps ) + { + auto cid = c.Id().get(); + captures.insert(cid); + captures_offsets[cid] = offset++; + } + } + + Profile(ft, body); } ProfileFunc::ProfileFunc(const Stmt* s, bool _abs_rec_fields) @@ -48,10 +64,15 @@ ProfileFunc::ProfileFunc(const Expr* e, bool _abs_rec_fields) { auto func = e->AsLambdaExpr(); - for ( auto oid : func->OuterIDs() ) - captures.insert(oid); + int offset = 0; - Profile(func->GetType()->AsFuncType(), func->Ingredients().Body()); + for ( auto oid : func->OuterIDs() ) + { + captures.insert(oid); + captures_offsets[oid] = offset++; + } + + Profile(func->GetType()->AsFuncType(), func->Ingredients()->Body()); } else @@ -91,9 +112,9 @@ TraversalCode ProfileFunc::PreStmt(const Stmt* s) auto w = s->AsWhenStmt(); auto wi = w->Info(); - auto wl = wi ? wi->Lambda() : nullptr; - if ( wl ) - lambdas.push_back(wl.get()); + + for ( auto wl : wi->WhenNewLocals() ) + when_locals.insert(wl); } break; @@ -171,6 +192,11 @@ TraversalCode ProfileFunc::PreExpr(const Expr* e) auto n = e->AsNameExpr(); auto id = n->Id(); + // Turns out that NameExpr's can be constructed using a + // different Type* than that of the identifier itself, + // so be sure we track the latter too. + TrackType(id->GetType()); + if ( id->IsGlobal() ) { globals.insert(id); @@ -179,30 +205,24 @@ TraversalCode ProfileFunc::PreExpr(const Expr* e) const auto& t = id->GetType(); if ( t->Tag() == TYPE_FUNC && t->AsFuncType()->Flavor() == FUNC_FLAVOR_EVENT ) events.insert(id->Name()); + + break; } - else - { - // This is a tad ugly. Unfortunately due to the - // weird way that Zeek function *declarations* work, - // there's no reliable way to get the list of - // parameters for a function *definition*, since - // they can have different names than what's present - // in the declaration. So we identify them directly, - // by knowing that they come at the beginning of the - // frame ... and being careful to avoid misconfusing - // a lambda capture with a low frame offset as a - // parameter. - if ( captures.count(id) == 0 && id->Offset() < num_params ) - params.insert(id); + // This is a tad ugly. Unfortunately due to the + // weird way that Zeek function *declarations* work, + // there's no reliable way to get the list of + // parameters for a function *definition*, since + // they can have different names than what's present + // in the declaration. So we identify them directly, + // by knowing that they come at the beginning of the + // frame ... and being careful to avoid misconfusing + // a lambda capture with a low frame offset as a + // parameter. + if ( captures.count(id) == 0 && id->Offset() < num_params ) + params.insert(id); - locals.insert(id); - } - - // Turns out that NameExpr's can be constructed using a - // different Type* than that of the identifier itself, - // so be sure we track the latter too. - TrackType(id->GetType()); + locals.insert(id); break; } @@ -350,7 +370,12 @@ TraversalCode ProfileFunc::PreExpr(const Expr* e) params.insert(i); } - // Avoid recursing into the body. + // In general, we don't want to recurse into the body. + // However, we still want to *profile* it so we can + // identify calls within it. + ProfileFunc body_pf(l->Ingredients()->Body().get(), false); + script_calls.insert(body_pf.ScriptCalls().begin(), body_pf.ScriptCalls().end()); + return TC_ABORTSTMT; } diff --git a/src/script_opt/ProfileFunc.h b/src/script_opt/ProfileFunc.h index 07d56ef39e..861a0c3a3e 100644 --- a/src/script_opt/ProfileFunc.h +++ b/src/script_opt/ProfileFunc.h @@ -100,6 +100,8 @@ public: const IDSet& Globals() const { return globals; } const IDSet& AllGlobals() const { return all_globals; } const IDSet& Locals() const { return locals; } + const IDSet& Captures() const { return captures; } + const auto& CapturesOffsets() const { return captures_offsets; } const IDSet& WhenLocals() const { return when_locals; } const IDSet& Params() const { return params; } const std::unordered_map& Assignees() const { return assignees; } @@ -208,6 +210,9 @@ protected: // If we're profiling a lambda function, this holds the captures. IDSet captures; + // This maps capture identifiers to their offsets. + std::map captures_offsets; + // Constants seen in the function. std::vector constants; @@ -224,7 +229,8 @@ protected: // The same, but in a deterministic order, with duplicates removed. std::vector ordered_types; - // Script functions that this script calls. + // Script functions that this script calls. Includes calls made + // by lambdas and when bodies, as the goal is to identify recursion. std::unordered_set script_calls; // Same for BiF's, though for them we record the corresponding global diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index 8a44ffcd84..dc0440b90c 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -36,15 +36,35 @@ static ZAMCompiler* ZAM = nullptr; static bool generating_CPP = false; static std::string CPP_dir; // where to generate C++ code +static std::unordered_set lambdas; +static std::unordered_set when_lambdas; static ScriptFuncPtr global_stmts; void analyze_func(ScriptFuncPtr f) { // Even if we're analyzing only a subset of the scripts, we still // track all functions here because the inliner will need the full list. + ASSERT(f->GetScope()); funcs.emplace_back(f, f->GetScope(), f->CurrentBody(), f->CurrentPriority()); } +void analyze_lambda(LambdaExpr* l) + { + auto& pf = l->PrimaryFunc(); + analyze_func(pf); + lambdas.insert(pf.get()); + } + +void analyze_when_lambda(LambdaExpr* l) + { + when_lambdas.insert(l->PrimaryFunc().get()); + } + +bool is_when_lambda(const ScriptFunc* f) + { + return when_lambdas.count(f) > 0; + } + const FuncInfo* analyze_global_stmts(Stmt* stmts) { // We ignore analysis_options.only_{files,funcs} - if they're in use, later @@ -208,7 +228,8 @@ static void optimize_func(ScriptFunc* f, std::shared_ptr pf, ScopeP rc->SetReadyToOptimize(); - auto ud = std::make_shared(body, rc); + auto ft = cast_intrusive(f->GetType()); + auto ud = std::make_shared(body, rc, ft); ud->Analyze(); if ( analysis_options.dump_uds ) @@ -216,6 +237,9 @@ static void optimize_func(ScriptFunc* f, std::shared_ptr pf, ScopeP new_body = ud->RemoveUnused(); + if ( analysis_options.dump_xform ) + printf("Post removal of unused: %s\n", obj_desc(new_body.get()).c_str()); + if ( new_body != body ) { f->ReplaceBody(body, new_body); @@ -442,10 +466,10 @@ static void analyze_scripts_for_ZAM(std::unique_ptr& pfs) } // Re-profile the functions, now without worrying about compatibility - // with compilation to C++. Note that the first profiling pass earlier - // may have marked some of the functions as to-skip, so first clear - // those markings. Once we have full compile-to-C++ and ZAM support - // for all Zeek language features, we can remove the re-profiling here. + // with compilation to C++. + + // The first profiling pass earlier may have marked some of the + // functions as to-skip, so clear those markings. for ( auto& f : funcs ) f.SetSkip(false); @@ -496,6 +520,7 @@ static void analyze_scripts_for_ZAM(std::unique_ptr& pfs) for ( auto& f : funcs ) { auto func = f.Func(); + bool is_lambda = lambdas.count(func) > 0; if ( ! analysis_options.only_funcs.empty() || ! analysis_options.only_files.empty() ) { @@ -503,14 +528,17 @@ static void analyze_scripts_for_ZAM(std::unique_ptr& pfs) continue; } - else if ( ! analysis_options.compile_all && inl && inl->WasInlined(func) && + else if ( ! analysis_options.compile_all && ! is_lambda && inl && inl->WasInlined(func) && func_used_indirectly.count(func) == 0 ) + { // No need to compile as it won't be called directly. continue; + } auto new_body = f.Body(); optimize_func(func, f.ProfilePtr(), f.Scope(), new_body); f.SetBody(new_body); + did_one = true; } @@ -593,6 +621,9 @@ void analyze_scripts(bool no_unused_warnings) // At this point we're done with C++ considerations, so instead // are compiling to ZAM. analyze_scripts_for_ZAM(pfs); + + if ( reporter->Errors() > 0 ) + reporter->FatalError("Optimized script execution aborted due to errors"); } void profile_script_execution() diff --git a/src/script_opt/ScriptOpt.h b/src/script_opt/ScriptOpt.h index f262149a37..8b83d4920c 100644 --- a/src/script_opt/ScriptOpt.h +++ b/src/script_opt/ScriptOpt.h @@ -138,7 +138,6 @@ public: std::shared_ptr ProfilePtr() const { return pf; } void SetBody(StmtPtr new_body) { body = std::move(new_body); } - // void SetProfile(std::shared_ptr _pf); void SetProfile(std::shared_ptr _pf) { pf = std::move(_pf); } // The following provide a way of marking FuncInfo's as @@ -168,6 +167,16 @@ extern std::unordered_set non_recursive_funcs; // Analyze a given function for optimization. extern void analyze_func(ScriptFuncPtr f); +// Same, for lambdas. +extern void analyze_lambda(LambdaExpr* f); + +// Same, for lambdas used in "when" statements. For these, analyze_lambda() +// has already been called. +extern void analyze_when_lambda(LambdaExpr* f); + +// Whether a given script function is a "when" lambda. +extern bool is_when_lambda(const ScriptFunc* f); + // Analyze the given top-level statement(s) for optimization. Returns // a pointer to a FuncInfo for an argument-less quasi-function that can // be Invoked, or its body executed directly, to execute the statements. diff --git a/src/script_opt/Stmt.cc b/src/script_opt/Stmt.cc index 5ffdda0f28..65570c8e31 100644 --- a/src/script_opt/Stmt.cc +++ b/src/script_opt/Stmt.cc @@ -646,21 +646,26 @@ StmtPtr ReturnStmt::Duplicate() ReturnStmt::ReturnStmt(ExprPtr arg_e, bool ignored) : ExprStmt(STMT_RETURN, std::move(arg_e)) { } +bool ReturnStmt::IsReduced(Reducer* c) const + { + if ( ! e || e->IsSingleton(c) ) + return true; + + return NonReduced(e.get()); + } + StmtPtr ReturnStmt::DoReduce(Reducer* c) { if ( ! e ) return ThisPtr(); if ( c->Optimizing() ) - { e = c->OptExpr(e); - return ThisPtr(); - } - if ( ! e->IsSingleton(c) ) + else if ( ! e->IsSingleton(c) ) { StmtPtr red_e_stmt; - e = e->Reduce(c, red_e_stmt); + e = e->ReduceToSingleton(c, red_e_stmt); if ( red_e_stmt ) { @@ -934,34 +939,85 @@ StmtPtr AssertStmt::DoReduce(Reducer* c) return make_intrusive(); } -StmtPtr WhenStmt::Duplicate() +bool WhenInfo::HasUnreducedIDs(Reducer* c) const { - FuncType::CaptureList* cl_dup = nullptr; - - if ( wi->Captures() ) + for ( auto& cp : *cl ) { - cl_dup = new FuncType::CaptureList; - *cl_dup = *wi->Captures(); + auto cid = cp.Id(); + + if ( when_new_locals.count(cid.get()) == 0 && ! c->ID_IsReduced(cp.Id()) ) + return true; } - auto new_wi = new WhenInfo(Cond(), cl_dup, IsReturn()); - new_wi->AddBody(Body()); - new_wi->AddTimeout(TimeoutExpr(), TimeoutBody()); + for ( auto& l : when_expr_locals ) + if ( ! c->ID_IsReduced(l) ) + return true; - return SetSucc(new WhenStmt(wi)); + return false; } -void WhenStmt::Inline(Inliner* inl) +void WhenInfo::UpdateIDs(Reducer* c) { - // Don't inline, since we currently don't correctly capture - // the frames of closures. + for ( auto& cp : *cl ) + { + auto& cid = cp.Id(); + if ( when_new_locals.count(cid.get()) == 0 ) + cp.SetID(c->UpdateID(cid)); + } + + for ( auto& l : when_expr_locals ) + l = c->UpdateID(l); + } + +StmtPtr WhenStmt::Duplicate() + { + return SetSucc(new WhenStmt(new WhenInfo(wi))); } bool WhenStmt::IsReduced(Reducer* c) const { - // We consider these always reduced because they're not - // candidates for any further optimization. - return true; + if ( wi->HasUnreducedIDs(c) ) + return false; + + if ( ! wi->Lambda()->IsReduced(c) ) + return false; + + if ( ! wi->TimeoutExpr() ) + return true; + + return wi->TimeoutExpr()->IsReduced(c); + } + +StmtPtr WhenStmt::DoReduce(Reducer* c) + { + if ( ! c->Optimizing() ) + { + wi->UpdateIDs(c); + (void)wi->Lambda()->ReduceToSingletons(c); + } + + auto e = wi->TimeoutExpr(); + + if ( ! e ) + return ThisPtr(); + + if ( c->Optimizing() ) + wi->SetTimeoutExpr(c->OptExpr(e)); + + else if ( ! e->IsSingleton(c) ) + { + StmtPtr red_e_stmt; + auto new_e = e->ReduceToSingleton(c, red_e_stmt); + wi->SetTimeoutExpr(new_e); + + if ( red_e_stmt ) + { + auto s = make_intrusive(red_e_stmt, ThisPtr()); + return TransformMe(s, c); + } + } + + return ThisPtr(); } CatchReturnStmt::CatchReturnStmt(StmtPtr _block, NameExprPtr _ret_var) : Stmt(STMT_CATCH_RETURN) diff --git a/src/script_opt/UseDefs.cc b/src/script_opt/UseDefs.cc index 8e54ab18b7..2f4d328d5c 100644 --- a/src/script_opt/UseDefs.cc +++ b/src/script_opt/UseDefs.cc @@ -17,10 +17,11 @@ void UseDefSet::Dump() const printf(" %s", u->Name()); } -UseDefs::UseDefs(StmtPtr _body, std::shared_ptr _rc) +UseDefs::UseDefs(StmtPtr _body, std::shared_ptr _rc, FuncTypePtr _ft) { body = std::move(_body); rc = std::move(_rc); + ft = std::move(_ft); } void UseDefs::Analyze() @@ -164,6 +165,13 @@ bool UseDefs::CheckIfUnused(const Stmt* s, const ID* id, bool report) if ( id->IsGlobal() ) return false; + if ( auto& captures = ft->GetCaptures() ) + { + for ( auto& c : *captures ) + if ( c.Id() == id ) + return false; + } + auto uds = FindSuccUsage(s); if ( ! uds || ! uds->HasID(id) ) { @@ -283,9 +291,7 @@ UDs UseDefs::PropagateUDs(const Stmt* s, UDs succ_UDs, const Stmt* succ_stmt, bo auto true_UDs = PropagateUDs(i->TrueBranch(), succ_UDs, succ_stmt, second_pass); auto false_UDs = PropagateUDs(i->FalseBranch(), succ_UDs, succ_stmt, second_pass); - auto uds = CreateUDs(s, UD_Union(cond_UDs, true_UDs, false_UDs)); - - return uds; + return CreateUDs(s, UD_Union(cond_UDs, true_UDs, false_UDs)); } case STMT_INIT: @@ -295,13 +301,17 @@ UDs UseDefs::PropagateUDs(const Stmt* s, UDs succ_UDs, const Stmt* succ_stmt, bo return UseUDs(s, succ_UDs); case STMT_WHEN: - // ### Once we support compiling functions with "when" - // statements in them, we'll need to revisit this. - // For now, we don't worry about it (because the current - // "when" body semantics of deep-copy frames has different - // implications than potentially switching those shallow-copy - // frames). - return UseUDs(s, succ_UDs); + { + auto w = s->AsWhenStmt(); + auto wi = w->Info(); + auto uds = UD_Union(succ_UDs, ExprUDs(wi->Lambda().get())); + + auto timeout = wi->TimeoutExpr(); + if ( timeout ) + uds = UD_Union(uds, ExprUDs(timeout.get())); + + return CreateUDs(s, uds); + } case STMT_SWITCH: { @@ -450,6 +460,7 @@ UDs UseDefs::ExprUDs(const Expr* e) switch ( e->Tag() ) { case EXPR_NAME: + case EXPR_LAMBDA: AddInExprUDs(uds, e); break; @@ -482,19 +493,23 @@ UDs UseDefs::ExprUDs(const Expr* e) break; } - case EXPR_CONST: - break; - - case EXPR_LAMBDA: + case EXPR_TABLE_CONSTRUCTOR: { - auto l = static_cast(e); - auto ids = l->OuterIDs(); + auto t = static_cast(e); + AddInExprUDs(uds, t->GetOp1().get()); + + auto& t_attrs = t->GetAttrs(); + auto def_attr = t_attrs ? t_attrs->Find(ATTR_DEFAULT) : nullptr; + auto& def_expr = def_attr ? def_attr->GetExpr() : nullptr; + if ( def_expr && def_expr->Tag() == EXPR_LAMBDA ) + uds = ExprUDs(def_expr.get()); - for ( const auto& id : ids ) - AddID(uds, id); break; } + case EXPR_CONST: + break; + case EXPR_CALL: { auto c = e->AsCallExpr(); @@ -577,6 +592,14 @@ void UseDefs::AddInExprUDs(UDs uds, const Expr* e) AddInExprUDs(uds, e->AsFieldExpr()->Op()); break; + case EXPR_LAMBDA: + { + auto outer_ids = e->AsLambdaExpr()->OuterIDs(); + for ( auto& i : outer_ids ) + AddID(uds, i); + break; + } + case EXPR_CONST: // Nothing to do. break; diff --git a/src/script_opt/UseDefs.h b/src/script_opt/UseDefs.h index 03bdcc4735..4b53776005 100644 --- a/src/script_opt/UseDefs.h +++ b/src/script_opt/UseDefs.h @@ -51,7 +51,7 @@ class Reducer; class UseDefs { public: - UseDefs(StmtPtr body, std::shared_ptr rc); + UseDefs(StmtPtr body, std::shared_ptr rc, FuncTypePtr ft); // Does a full pass over the function body's AST. We can wind // up doing this multiple times because when we use use-defs to @@ -173,6 +173,7 @@ private: StmtPtr body; std::shared_ptr rc; + FuncTypePtr ft; }; } // zeek::detail diff --git a/src/script_opt/ZAM/AM-Opt.cc b/src/script_opt/ZAM/AM-Opt.cc index 7735167fc4..2da500681c 100644 --- a/src/script_opt/ZAM/AM-Opt.cc +++ b/src/script_opt/ZAM/AM-Opt.cc @@ -258,9 +258,9 @@ bool ZAMCompiler::PruneUnused() KillInst(i); } - if ( inst->IsGlobalLoad() ) + if ( inst->IsNonLocalLoad() ) { - // Any straight-line load of the same global + // Any straight-line load of the same global/capture // is redundant. for ( unsigned int j = i + 1; j < insts1.size(); ++j ) { @@ -277,14 +277,14 @@ bool ZAMCompiler::PruneUnused() // Inbound branch ends block. break; - if ( i1->aux && i1->aux->can_change_globals ) + if ( i1->aux && i1->aux->can_change_non_locals ) break; - if ( ! i1->IsGlobalLoad() ) + if ( ! i1->IsNonLocalLoad() ) continue; - if ( i1->v2 == inst->v2 ) - { // Same global + if ( i1->v2 == inst->v2 && i1->IsGlobalLoad() == inst->IsGlobalLoad() ) + { // Same global/capture did_prune = true; KillInst(i1); } @@ -299,9 +299,10 @@ bool ZAMCompiler::PruneUnused() // Variable is used, keep assignment. continue; - if ( frame_denizens[slot]->IsGlobal() ) + auto& id = frame_denizens[slot]; + if ( id->IsGlobal() || IsCapture(id) ) { - // Extend the global's range to the end of the + // Extend the global/capture's range to the end of the // function. denizen_ending[slot] = insts1.back(); continue; @@ -466,18 +467,30 @@ void ZAMCompiler::ComputeFrameLifetimes() break; } + case OP_LAMBDA_VV: + { + auto aux = inst->aux; + int n = aux->n; + auto& slots = aux->slots; + for ( int i = 0; i < n; ++i ) + ExtendLifetime(slots[i], EndOfLoop(inst, 1)); + break; + } + default: // Look for slots in auxiliary information. auto aux = inst->aux; if ( ! aux || ! aux->slots ) break; - for ( auto j = 0; j < aux->n; ++j ) + int n = aux->n; + auto& slots = aux->slots; + for ( auto j = 0; j < n; ++j ) { - if ( aux->slots[j] < 0 ) + if ( slots[j] < 0 ) continue; - ExtendLifetime(aux->slots[j], EndOfLoop(inst, 1)); + ExtendLifetime(slots[j], EndOfLoop(inst, 1)); } break; } @@ -759,7 +772,6 @@ void ZAMCompiler::ReMapVar(const ID* id, int slot, zeek_uint_t inst) void ZAMCompiler::CheckSlotAssignment(int slot, const ZInstI* inst) { ASSERT(slot >= 0 && static_cast(slot) < frame_denizens.size()); - // We construct temporaries such that their values are never used // earlier than their definitions in loop bodies. For other // denizens, however, they can be, so in those cases we expand the @@ -915,47 +927,6 @@ const ZInstI* ZAMCompiler::EndOfLoop(const ZInstI* inst, int depth) const return insts1[i]; } -bool ZAMCompiler::VarIsAssigned(int slot) const - { - for ( auto& inst : insts1 ) - if ( inst->live && VarIsAssigned(slot, inst) ) - return true; - - return false; - } - -bool ZAMCompiler::VarIsAssigned(int slot, const ZInstI* i) const - { - // Special-case for table iterators, which assign to a bunch - // of variables but they're not immediately visible in the - // instruction layout. - if ( i->op == OP_NEXT_TABLE_ITER_VAL_VAR_VVV || i->op == OP_NEXT_TABLE_ITER_VV ) - { - auto& iter_vars = i->aux->loop_vars; - for ( auto v : iter_vars ) - if ( v == slot ) - return true; - - if ( i->op != OP_NEXT_TABLE_ITER_VAL_VAR_VVV ) - return false; - - // Otherwise fall through, since that flavor of iterate - // *does* also assign to slot 1. - } - - if ( i->op == OP_NEXT_VECTOR_ITER_VAL_VAR_VVVV && i->v2 == slot ) - return true; - - if ( i->op_type == OP_VV_FRAME ) - // We don't want to consider these as assigning to the - // variable, since the point of this method is to figure - // out which variables don't need storing to the frame - // because their internal value is never modified. - return false; - - return i->AssignsToSlot1() && i->v1 == slot; - } - bool ZAMCompiler::VarIsUsed(int slot) const { for ( auto& inst : insts1 ) diff --git a/src/script_opt/ZAM/Compile.h b/src/script_opt/ZAM/Compile.h index 1287dcdc6a..b990a1e9a5 100644 --- a/src/script_opt/ZAM/Compile.h +++ b/src/script_opt/ZAM/Compile.h @@ -92,6 +92,7 @@ private: void Init(); void InitGlobals(); void InitArgs(); + void InitCaptures(); void InitLocals(); void TrackMemoryManagement(); @@ -137,6 +138,7 @@ private: const ZAMStmt CompileCatchReturn(const CatchReturnStmt* cr); const ZAMStmt CompileStmts(const StmtList* sl); const ZAMStmt CompileInit(const InitStmt* is); + const ZAMStmt CompileWhen(const WhenStmt* ws); const ZAMStmt CompileNext() { return GenGoTo(nexts.back()); } const ZAMStmt CompileBreak() { return GenGoTo(breaks.back()); } @@ -219,10 +221,15 @@ private: const ZAMStmt CompileInExpr(const NameExpr* n1, const ListExpr* l, const NameExpr* n2, const ConstExpr* c); - const ZAMStmt CompileIndex(const NameExpr* n1, const NameExpr* n2, const ListExpr* l); - const ZAMStmt CompileIndex(const NameExpr* n1, const ConstExpr* c, const ListExpr* l); + const ZAMStmt CompileIndex(const NameExpr* n1, const NameExpr* n2, const ListExpr* l, + bool in_when); + const ZAMStmt CompileIndex(const NameExpr* n1, const ConstExpr* c, const ListExpr* l, + bool in_when); const ZAMStmt CompileIndex(const NameExpr* n1, int n2_slot, const TypePtr& n2_type, - const ListExpr* l); + const ListExpr* l, bool in_when); + + const ZAMStmt BuildLambda(const NameExpr* n, LambdaExpr* le); + const ZAMStmt BuildLambda(int n_slot, LambdaExpr* le); // Second argument is which instruction slot holds the branch target. const ZAMStmt GenCond(const Expr* e, int& branch_v); @@ -350,8 +357,15 @@ private: bool IsUnused(const IDPtr& id, const Stmt* where) const; + bool IsCapture(const IDPtr& id) const { return IsCapture(id.get()); } + bool IsCapture(const ID* id) const; + + int CaptureOffset(const IDPtr& id) const { return IsCapture(id.get()); } + int CaptureOffset(const ID* id) const; + void LoadParam(const ID* id); const ZAMStmt LoadGlobal(const ID* id); + const ZAMStmt LoadCapture(const ID* id); int AddToFrame(const ID*); @@ -445,14 +459,6 @@ private: const ZInstI* BeginningOfLoop(const ZInstI* inst, int depth) const; const ZInstI* EndOfLoop(const ZInstI* inst, int depth) const; - // True if any statement other than a frame sync assigns to the - // given slot. - bool VarIsAssigned(int slot) const; - - // True if the given statement assigns to the given slot, and - // it's not a frame sync. - bool VarIsAssigned(int slot, const ZInstI* i) const; - // True if any statement other than a frame sync uses the given slot. bool VarIsUsed(int slot) const; @@ -599,8 +605,10 @@ private: // Used for communication between Frame1Slot and a subsequent // AddInst. If >= 0, then upon adding the next instruction, - // it should be followed by Store-Global for the given slot. + // it should be followed by Store-Global or Store-Capture for + // the given slot. int pending_global_store = -1; + int pending_capture_store = -1; }; // Invokes after compiling all of the function bodies. diff --git a/src/script_opt/ZAM/Driver.cc b/src/script_opt/ZAM/Driver.cc index 18426909da..31cf5c9a26 100644 --- a/src/script_opt/ZAM/Driver.cc +++ b/src/script_opt/ZAM/Driver.cc @@ -33,20 +33,9 @@ void ZAMCompiler::Init() { InitGlobals(); InitArgs(); + InitCaptures(); InitLocals(); -#if 0 - // Complain about unused aggregates ... but not if we're inlining, - // as that can lead to optimizations where they wind up being unused - // but the original logic for using them was sound. - if ( ! analysis_options.inliner ) - for ( auto a : pf->Inits() ) - { - if ( pf->Locals().find(a) == pf->Locals().end() ) - reporter->Warning("%s unused", a->Name()); - } -#endif - TrackMemoryManagement(); non_recursive = non_recursive_funcs.count(func) > 0; @@ -92,12 +81,25 @@ void ZAMCompiler::InitArgs() pop_scope(); } +void ZAMCompiler::InitCaptures() + { + for ( auto c : pf->Captures() ) + (void)AddToFrame(c); + } + void ZAMCompiler::InitLocals() { // Assign slots for locals (which includes temporaries). for ( auto l : pf->Locals() ) { + if ( IsCapture(l) ) + continue; + + if ( pf->WhenLocals().count(l) > 0 ) + continue; + auto non_const_l = const_cast(l); + // Don't add locals that were already added because they're // parameters. // @@ -203,11 +205,6 @@ StmtPtr ZAMCompiler::CompileBody() // Could erase insts1 here to recover memory, but it's handy // for debugging. -#if 0 - if ( non_recursive ) - func->UseStaticFrame(); -#endif - auto zb = make_intrusive(func->Name(), this); zb->SetInsts(insts2); diff --git a/src/script_opt/ZAM/Expr.cc b/src/script_opt/ZAM/Expr.cc index cbf29900fa..c7d962bbfe 100644 --- a/src/script_opt/ZAM/Expr.cc +++ b/src/script_opt/ZAM/Expr.cc @@ -4,6 +4,7 @@ #include "zeek/Desc.h" #include "zeek/Reporter.h" +#include "zeek/script_opt/ProfileFunc.h" #include "zeek/script_opt/ZAM/Compile.h" namespace zeek::detail @@ -176,12 +177,6 @@ const ZAMStmt ZAMCompiler::CompileAssignExpr(const AssignExpr* e) auto r2 = rhs->GetOp2(); auto r3 = rhs->GetOp3(); - if ( rhs->Tag() == EXPR_LAMBDA ) - { - // reporter->Error("lambda expressions not supported for compiling"); - return ErrorStmt(); - } - if ( rhs->Tag() == EXPR_NAME ) return AssignVV(lhs, rhs->AsNameExpr()); @@ -213,6 +208,9 @@ const ZAMStmt ZAMCompiler::CompileAssignExpr(const AssignExpr* e) if ( rhs->Tag() == EXPR_ANY_INDEX ) return AnyIndexVVi(lhs, r1->AsNameExpr(), rhs->AsAnyIndexExpr()->Index()); + if ( rhs->Tag() == EXPR_LAMBDA ) + return BuildLambda(lhs, rhs->AsLambdaExpr()); + if ( rhs->Tag() == EXPR_COND && r1->GetType()->Tag() == TYPE_VECTOR ) return Bool_Vec_CondVVVV(lhs, r1->AsNameExpr(), r2->AsNameExpr(), r3->AsNameExpr()); @@ -285,7 +283,18 @@ const ZAMStmt ZAMCompiler::CompileAssignToIndex(const NameExpr* lhs, const Index : IndexVecIntSelectVVV(lhs, n, index); } - return const_aggr ? IndexVCL(lhs, con, indexes_expr) : IndexVVL(lhs, n, indexes_expr); + if ( rhs->IsInsideWhen() ) + { + if ( const_aggr ) + return WhenIndexVCL(lhs, con, indexes_expr); + else + return WhenIndexVVL(lhs, n, indexes_expr); + } + + if ( const_aggr ) + return IndexVCL(lhs, con, indexes_expr); + else + return IndexVVL(lhs, n, indexes_expr); } const ZAMStmt ZAMCompiler::CompileFieldLHSAssignExpr(const FieldLHSAssignExpr* e) @@ -616,26 +625,28 @@ const ZAMStmt ZAMCompiler::CompileInExpr(const NameExpr* n1, const ListExpr* l, return AddInst(z); } -const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, const NameExpr* n2, const ListExpr* l) +const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, const NameExpr* n2, const ListExpr* l, + bool in_when) { - return CompileIndex(n1, FrameSlot(n2), n2->GetType(), l); + return CompileIndex(n1, FrameSlot(n2), n2->GetType(), l, in_when); } -const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n, const ConstExpr* c, const ListExpr* l) +const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n, const ConstExpr* c, const ListExpr* l, + bool in_when) { auto tmp = TempForConst(c); - return CompileIndex(n, tmp, c->GetType(), l); + return CompileIndex(n, tmp, c->GetType(), l, in_when); } const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, int n2_slot, const TypePtr& n2t, - const ListExpr* l) + const ListExpr* l, bool in_when) { ZInstI z; int n = l->Exprs().length(); auto n2tag = n2t->Tag(); - if ( n == 1 ) + if ( n == 1 && ! in_when ) { auto ind = l->Exprs()[0]; auto var_ind = ind->Tag() == EXPR_NAME; @@ -677,12 +688,28 @@ const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, int n2_slot, const T if ( n3 ) { int n3_slot = FrameSlot(n3); - auto zop = is_any ? OP_INDEX_ANY_VEC_VVV : OP_INDEX_VEC_VVV; + + ZOp zop; + if ( in_when ) + zop = OP_WHEN_INDEX_VEC_VVV; + else if ( is_any ) + zop = OP_INDEX_ANY_VEC_VVV; + else + zop = OP_INDEX_VEC_VVV; + z = ZInstI(zop, Frame1Slot(n1, zop), n2_slot, n3_slot); } else { - auto zop = is_any ? OP_INDEX_ANY_VECC_VVV : OP_INDEX_VECC_VVV; + ZOp zop; + + if ( in_when ) + zop = OP_WHEN_INDEX_VECC_VVV; + else if ( is_any ) + zop = OP_INDEX_ANY_VECC_VVV; + else + zop = OP_INDEX_VECC_VVV; + z = ZInstI(zop, Frame1Slot(n1, zop), n2_slot, c); z.op_type = OP_VVV_I3; } @@ -720,13 +747,13 @@ const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, int n2_slot, const T switch ( n2tag ) { case TYPE_VECTOR: - op = OP_INDEX_VEC_SLICE_VV; + op = in_when ? OP_WHEN_INDEX_VEC_SLICE_VV : OP_INDEX_VEC_SLICE_VV; z = ZInstI(op, Frame1Slot(n1, op), n2_slot); z.SetType(n2t); break; case TYPE_TABLE: - op = OP_TABLE_INDEX_VV; + op = in_when ? OP_WHEN_TABLE_INDEX_VV : OP_TABLE_INDEX_VV; z = ZInstI(op, Frame1Slot(n1, op), n2_slot); z.SetType(n1->GetType()); break; @@ -747,6 +774,38 @@ const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, int n2_slot, const T return AddInst(z); } +const ZAMStmt ZAMCompiler::BuildLambda(const NameExpr* n, LambdaExpr* le) + { + return BuildLambda(Frame1Slot(n, OP1_WRITE), le); + } + +const ZAMStmt ZAMCompiler::BuildLambda(int n_slot, LambdaExpr* le) + { + auto& captures = le->GetCaptures(); + int ncaptures = captures ? captures->size() : 0; + + auto aux = new ZInstAux(ncaptures); + aux->primary_func = le->PrimaryFunc(); + aux->lambda_name = le->Name(); + aux->id_val = le->Ingredients()->GetID(); + + for ( int i = 0; i < ncaptures; ++i ) + { + auto& id_i = (*captures)[i].Id(); + + if ( pf->WhenLocals().count(id_i.get()) > 0 ) + aux->Add(i, nullptr); + else + aux->Add(i, FrameSlot(id_i), id_i->GetType()); + } + + auto z = ZInstI(OP_LAMBDA_VV, n_slot, le->PrimaryFunc()->FrameSize()); + z.op_type = OP_VV_I2; + z.aux = aux; + + return AddInst(z); + } + const ZAMStmt ZAMCompiler::AssignVecElems(const Expr* e) { auto index_assign = e->AsIndexAssignExpr(); @@ -887,16 +946,17 @@ const ZAMStmt ZAMCompiler::AssignToCall(const ExprStmt* e) const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n) { auto func = c->Func()->AsNameExpr(); - auto func_id = func->Id(); + auto func_id = func->IdPtr(); auto& args = c->Args()->Exprs(); int nargs = args.length(); int call_case = nargs; bool indirect = ! func_id->IsGlobal() || ! func_id->GetVal(); + bool in_when = c->IsInWhen(); - if ( indirect ) - call_case = -1; // force default of CallN + if ( indirect || in_when ) + call_case = -1; // force default of some flavor of CallN auto nt = n ? n->GetType()->Tag() : TYPE_VOID; auto n_slot = n ? Frame1Slot(n, OP1_WRITE) : -1; @@ -973,8 +1033,17 @@ const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n) break; default: - if ( indirect ) + if ( in_when ) + { + if ( indirect ) + op = OP_WHENINDCALLN_VV; + else + op = OP_WHENCALLN_V; + } + + else if ( indirect ) op = n ? OP_INDCALLN_VV : OP_INDCALLN_V; + else op = n ? OP_CALLN_V : OP_CALLN_X; break; @@ -982,7 +1051,9 @@ const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n) if ( n ) { - op = AssignmentFlavor(op, nt); + if ( ! in_when ) + op = AssignmentFlavor(op, nt); + auto n_slot = Frame1Slot(n, OP1_WRITE); if ( indirect ) @@ -1023,10 +1094,13 @@ const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n) if ( ! z.aux ) z.aux = new ZInstAux(0); - z.aux->can_change_globals = true; + z.aux->can_change_non_locals = true; z.call_expr = c; + if ( in_when ) + z.SetType(n->GetType()); + if ( ! indirect || func_id->IsGlobal() ) { z.aux->id_val = func_id; @@ -1035,19 +1109,6 @@ const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n) z.func = func_id->GetVal()->AsFunc(); } - if ( n ) - { - auto id = n->Id(); - if ( id->IsGlobal() ) - { - AddInst(z); - auto global_slot = global_id_to_info[id]; - z = ZInstI(OP_STORE_GLOBAL_V, global_slot); - z.op_type = OP_V_I1; - z.t = globalsI[global_slot].id->GetType(); - } - } - return AddInst(z); } @@ -1062,6 +1123,31 @@ const ZAMStmt ZAMCompiler::ConstructTable(const NameExpr* n, const Expr* e) z.t = tt; z.attrs = e->AsTableConstructorExpr()->GetAttrs(); + auto zstmt = AddInst(z); + + auto def_attr = z.attrs ? z.attrs->Find(ATTR_DEFAULT) : nullptr; + if ( ! def_attr || def_attr->GetExpr()->Tag() != EXPR_LAMBDA ) + return zstmt; + + auto def_lambda = def_attr->GetExpr()->AsLambdaExpr(); + auto dl_t = def_lambda->GetType()->AsFuncType(); + auto& captures = dl_t->GetCaptures(); + + if ( ! captures ) + return zstmt; + + // What a pain. The table's default value is a lambda that has + // captures. The semantics of this are that the captures are + // evaluated at table-construction time. We need to build the + // lambda and assign it as the table's default. + + auto slot = NewSlot(true); // since func_val's are managed + (void)BuildLambda(slot, def_lambda); + + z = GenInst(OP_SET_TABLE_DEFAULT_LAMBDA_VV, n, slot); + z.op_type = OP_VV; + z.t = def_lambda->GetType(); + return AddInst(z); } diff --git a/src/script_opt/ZAM/Low-Level.cc b/src/script_opt/ZAM/Low-Level.cc index 40fd4c30df..0db09a0147 100644 --- a/src/script_opt/ZAM/Low-Level.cc +++ b/src/script_opt/ZAM/Low-Level.cc @@ -149,6 +149,9 @@ const ZAMStmt ZAMCompiler::AddInst(const ZInstI& inst, bool suppress_non_local) if ( suppress_non_local ) return ZAMStmt(top_main_inst); + // Ensure we haven't confused ourselves about any pending stores. + ASSERT(pending_global_store == -1 || pending_capture_store == -1); + if ( pending_global_store >= 0 ) { auto gs = pending_global_store; @@ -161,6 +164,27 @@ const ZAMStmt ZAMCompiler::AddInst(const ZInstI& inst, bool suppress_non_local) return AddInst(store_inst); } + if ( pending_capture_store >= 0 ) + { + auto cs = pending_capture_store; + pending_capture_store = -1; + + auto& cv = *func->GetType()->AsFuncType()->GetCaptures(); + auto& c_id = cv[cs].Id(); + + ZOp op; + + if ( ZVal::IsManagedType(c_id->GetType()) ) + op = OP_STORE_MANAGED_CAPTURE_VV; + else + op = OP_STORE_CAPTURE_VV; + + auto store_inst = ZInstI(op, RawSlot(c_id.get()), cs); + store_inst.op_type = OP_VV_I2; + + return AddInst(store_inst); + } + return ZAMStmt(top_main_inst); } diff --git a/src/script_opt/ZAM/Ops.in b/src/script_opt/ZAM/Ops.in index 89bfccdc35..8c0264ce1b 100644 --- a/src/script_opt/ZAM/Ops.in +++ b/src/script_opt/ZAM/Ops.in @@ -12,7 +12,7 @@ # The Gen-ZAM utility processes this file to generate numerous C++ inclusion # files that are then compiled into Zeek. These files span the range of (1) # hooks that enable run-time generation of ZAM code to execute ASTs (which -# have first been transformed to "reduced" form, (2) specifications of the +# have first been transformed to "reduced" form), (2) specifications of the # properties of the different instructions, (3) code to evaluate (execute) # each instruction, and (4) macros (C++ #define's) to aid in writing that # code. See Gen-ZAM.h for a list of the different inclusion files. @@ -947,11 +947,19 @@ eval EvalIndexVecIntSelect(z.c, frame[z.v2]) op Index type VVL -custom-method return CompileIndex(n1, n2, l); +custom-method return CompileIndex(n1, n2, l, false); op Index type VCL -custom-method return CompileIndex(n, c, l); +custom-method return CompileIndex(n, c, l, false); + +op WhenIndex +type VVL +custom-method return CompileIndex(n1, n2, l, true); + +op WhenIndex +type VCL +custom-method return CompileIndex(n, c, l, true); internal-op Index-Vec type VVV @@ -988,22 +996,51 @@ internal-op Index-Any-VecC type VVV eval EvalIndexAnyVec(z.v3) -internal-op Index-Vec-Slice -type VV -eval auto vec = frame[z.v2].vector_val; +macro WhenIndexResCheck() + auto& res = frame[z.v1].vector_val; + if ( res && IndexExprWhen::evaluating > 0 ) + IndexExprWhen::results.push_back({NewRef{}, res}); + +internal-op When-Index-Vec +type VVV +eval EvalIndexAnyVec(frame[z.v3].uint_val) + WhenIndexResCheck() + +internal-op When-Index-VecC +type VVV +eval EvalIndexAnyVec(z.v3) + WhenIndexResCheck() + +macro EvalVecSlice() + auto vec = frame[z.v2].vector_val; auto lv = z.aux->ToListVal(frame); auto v = index_slice(vec, lv.get()); Unref(frame[z.v1].vector_val); frame[z.v1].vector_val = v.release(); +internal-op Index-Vec-Slice +type VV +eval EvalVecSlice() + +internal-op When-Index-Vec-Slice +type VV +eval EvalVecSlice() + WhenIndexResCheck() + internal-op Table-Index type VV eval EvalTableIndex(z.aux->ToListVal(frame)) AssignV1(BuildVal(v, z.t)) +internal-op When-Table-Index +type VV +eval EvalTableIndex(z.aux->ToListVal(frame)) + if ( IndexExprWhen::evaluating > 0 ) + IndexExprWhen::results.emplace_back(v); + AssignV1(BuildVal(v, z.t)) + macro EvalTableIndex(index) - auto v2 = index; - auto v = frame[z.v2].table_val->FindOrDefault(v2); + auto v = frame[z.v2].table_val->FindOrDefault(index); if ( ! v ) { ZAM_run_time_error(z.loc, "no such index"); @@ -1092,6 +1129,15 @@ eval ConstructTableOrSetPre() } ConstructTableOrSetPost() +# When tables are constructed, if their &default is a lambda with captures +# then we need to explicitly set up the default. +internal-op Set-Table-Default-Lambda +type VV +op1-read +eval auto& tbl = frame[z.v1].table_val; + auto lambda = frame[z.v2].ToVal(z.t); + tbl->InitDefaultVal(lambda); + direct-unary-op Set-Constructor ConstructSet internal-op Construct-Set @@ -1527,6 +1573,49 @@ assign-val v indirect-call num-call-args n +# A call made in a "when" context. These always have assignment targets. +# To keep things simple, we just use one generic flavor (for N arguments, +# doing a less-streamlined-but-simpler Val-based assignment). +macro WhenCall(func) + if ( ! func ) + throw ZAMDelayedCallException(); + auto& lhs = frame[z.v1]; + auto trigger = f->GetTrigger(); + Val* v = trigger ? trigger->Lookup(z.call_expr) : nullptr; + ValPtr vp; + if ( v ) + vp = {NewRef{}, v}; + else + { + auto aux = z.aux; + auto current_assoc = f->GetTriggerAssoc(); + auto n = aux->n; + std::vector args; + for ( auto i = 0; i < n; ++i ) + args.push_back(aux->ToVal(frame, i)); + f->SetCall(z.call_expr); + vp = func->Invoke(&args, f); + f->SetTriggerAssoc(current_assoc); + if ( ! vp ) + throw ZAMDelayedCallException(); + } + if ( z.is_managed ) + ZVal::DeleteManagedType(lhs); + lhs = ZVal(vp, z.t); + +internal-op WhenCallN +type V +side-effects +eval WhenCall(z.func) + +internal-op WhenIndCallN +type VV +side-effects +eval auto sel = z.v2; + auto func = (sel < 0) ? z.aux->id_val->GetVal()->AsFunc() : frame[sel].AsFunc(); + WhenCall(func) + + ########## Statements ########## macro EvalScheduleArgs(time, is_delta, build_args) @@ -1956,6 +2045,35 @@ eval auto tt = cast_intrusive(z.t); Unref(frame[z.v1].table_val); frame[z.v1].table_val = t; +op When +type V +op1-read +eval BuildWhen(-1.0) + +op When-Timeout +type VV +op1-read +eval BuildWhen(frame[z.v2].double_val) + +op When-Timeout +type VC +op1-read +eval BuildWhen(z.c.double_val) + +macro BuildWhen(timeout) + auto& aux = z.aux; + auto wi = aux->wi; + FuncPtr func{NewRef{}, frame[z.v1].func_val}; + auto lambda = make_intrusive(func); + wi->Instantiate(lambda); + std::vector local_aggrs; + for ( int i = 0; i < aux->n; ++i ) + { + auto v = aux->ToVal(frame, i); + if ( v ) + local_aggrs.push_back(v); + } + new trigger::Trigger(wi, timeout, wi->WhenExprGlobals(), local_aggrs, f, z.loc); ######################################## # Internal @@ -1994,7 +2112,7 @@ assign-val v eval auto v = globals[z.v2].id->GetVal(); if ( ! v ) { - ZAM_run_time_error(z.loc, "value used but not set", z.aux->id_val); + ZAM_run_time_error(z.loc, "value used but not set", z.aux->id_val.get()); break; } @@ -2007,12 +2125,41 @@ eval auto& v = frame[z.v1].type_val; auto t = globals[z.v2].id->GetType(); v = new TypeVal(t, true); +internal-op Load-Capture +type VV +eval frame[z.v1] = f->GetFunction()->GetCapturesVec()[z.v2]; + +internal-op Load-Managed-Capture +type VV +eval auto& lhs = frame[z.v1]; + auto& rhs = f->GetFunction()->GetCapturesVec()[z.v2]; + zeek::Ref(rhs.ManagedVal()); + ZVal::DeleteManagedType(lhs); + lhs = rhs; + internal-op Store-Global op1-internal type V eval auto& g = globals[z.v1]; g.id->SetVal(frame[g.slot].ToVal(z.t)); +# Both of these have the LHS as v2 not v1, to keep with existing +# conventions of OP_VV_I2 op type (as opposed to OP_VV_I1_V2, which doesn't +# currently exist, and would be a pain to add). +internal-op Store-Capture +op1-read +type VV +eval f->GetFunction()->GetCapturesVec()[z.v2] = frame[z.v1]; + +internal-op Store-Managed-Capture +op1-read +type VV +eval auto& lhs = f->GetFunction()->GetCapturesVec()[z.v2]; + auto& rhs = frame[z.v1]; + zeek::Ref(rhs.ManagedVal()); + ZVal::DeleteManagedType(lhs); + lhs = rhs; + internal-op Copy-To type VC @@ -2029,6 +2176,37 @@ eval flow = FLOW_BREAK; pc = end_pc; continue; +# Slot 2 gives frame size. +internal-op Lambda +type VV +eval auto& aux = z.aux; + auto& primary_func = aux->primary_func; + auto& body = primary_func->GetBodies()[0].stmts; + ASSERT(body->Tag() == STMT_ZAM); + auto lamb = make_intrusive(aux->id_val); + lamb->AddBody(body, z.v2); + lamb->SetName(aux->lambda_name.c_str()); + if ( aux->n > 0 ) + { + auto captures = std::make_unique>(); + for ( auto i = 0; i < aux->n; ++i ) + { + auto slot = aux->slots[i]; + if ( slot >= 0 ) + { + auto& cp = frame[aux->slots[i]]; + if ( aux->is_managed[i] ) + zeek::Ref(cp.ManagedVal()); + captures->push_back(cp); + } + else + // Used for when-locals. + captures->push_back(ZVal()); + } + lamb->CreateCaptures(std::move(captures)); + } + ZVal::DeleteManagedType(frame[z.v1]); + frame[z.v1].func_val = lamb.release(); ######################################## # Built-in Functions diff --git a/src/script_opt/ZAM/README.md b/src/script_opt/ZAM/README.md index 9d2ac86c72..a1313a6dd3 100644 --- a/src/script_opt/ZAM/README.md +++ b/src/script_opt/ZAM/README.md @@ -41,35 +41,29 @@ the _Logging Framework_, for example, and thus you won't see much improvement. * Those two factors add up to gains very often on the order of only 10-15%, rather than something a lot more dramatic. -* In addition, there are some -[types of scripts that currently can't be compiled](#Scripts-that-cannot-be-compiled), -and thus will remain interpreted. If your processing bottlenecks in such -scripts, you won't see much in the way of gains. -
## Known Issues Here we list various issues with using script optimization, including both -deficiencies (problems to eventually fix) and incompatibilities (differences -in behavior from the default of script interpretation, not necessarily -fixable). For each, the corresponding list is roughly ordered from -you're-most-likely-to-care-about-it to you're-less-likely-to-care, though -of course this varies for different users. +deficiencies (things that don't work as well as you might like) +and incompatibilities (differences in behavior from the default +of script interpretation).
-### Deficiencies to eventually fix: +### Deficiencies: -* Error messages in compiled scripts have diminished identifying +* Run-time error messages in compiled scripts have diminished identifying information. * The optimizer assumes you have ensured initialization of your variables. If your script uses a variable that hasn't been set, the compiled code may crash or behave aberrantly. You can use the `-u` command-line flag to find such potential usage issues. -* Certain complex "when" expressions may fail to reevaluate when elements -of the expression are modified by compiled scripts. +* When printing scripts (such as in some error messages), the names of +variables often reflect internal temporaries rather than the original +variables.
@@ -77,30 +71,11 @@ of the expression are modified by compiled scripts. * ZAM ignores `assert` statements. -* When printing scripts (such as in some error messages), the names of -variables often reflect internal temporaries rather than the original -variables. - * The `same_object()` BiF will always deem two non-container values as different.
-### Scripts that cannot be compiled: - -The ZAM optimizer does not compile scripts that include "when" statements or -lambda expressions. These will take substantial work to support. It also -will not inline such scripts, nor will it inline scripts that are either -directly or indirectly recursive. - -You can get a list of non-compilable scripts using -`-O ZAM -O report-uncompilable`. For recursive scripts, use -`-O report-recursive` (no `-O ZAM` required, since it doesn't apply to the -alternative optimization, `-O gen-C++`). - -
- - ## Script Optimization Options Users will generally simply use `-O ZAM` to invoke the script optimizer. @@ -120,7 +95,7 @@ issues: |`optimize-AST` | Optimize the (transform) AST; implies `xform`.| |`profile-ZAM` | Generate to _stdout_ a ZAM execution profile. (Requires configuring with `--enable-debug`.)| |`report-recursive` | Report on recursive functions and exit.| -|`report-uncompilable` | Report on uncompilable functions and exit.| +|`report-uncompilable` | Report on uncompilable functions and exit. For ZAM, all functions should be compilable.| |`xform` | Transform scripts to "reduced" form.|
diff --git a/src/script_opt/ZAM/Stmt.cc b/src/script_opt/ZAM/Stmt.cc index 4d8e385c14..a798347bd3 100644 --- a/src/script_opt/ZAM/Stmt.cc +++ b/src/script_opt/ZAM/Stmt.cc @@ -60,6 +60,9 @@ const ZAMStmt ZAMCompiler::CompileStmt(const Stmt* s) case STMT_INIT: return CompileInit(static_cast(s)); + case STMT_WHEN: + return CompileWhen(static_cast(s)); + case STMT_NULL: return EmptyStmt(); @@ -1093,6 +1096,60 @@ const ZAMStmt ZAMCompiler::CompileInit(const InitStmt* is) return last; } +const ZAMStmt ZAMCompiler::CompileWhen(const WhenStmt* ws) + { + auto wi = ws->Info(); + auto timeout = wi->TimeoutExpr(); + + auto lambda = NewSlot(true); + (void)BuildLambda(lambda, wi->Lambda().get()); + + std::vector local_aggr_slots; + for ( auto& l : wi->WhenExprLocals() ) + if ( IsAggr(l->GetType()->Tag()) ) + local_aggr_slots.push_back(l); + + int n = local_aggr_slots.size(); + auto aux = new ZInstAux(n); + aux->wi = wi; + + for ( auto i = 0; i < n; ++i ) + { + auto la = local_aggr_slots[i]; + aux->Add(i, FrameSlot(la), la->GetType()); + } + + ZInstI z; + + if ( timeout ) + { + if ( timeout->Tag() == EXPR_NAME ) + { + auto ns = FrameSlot(timeout->AsNameExpr()); + z = ZInstI(OP_WHEN_TIMEOUT_VV, lambda, ns); + } + else + { + ASSERT(timeout->Tag() == EXPR_CONST); + z = ZInstI(OP_WHEN_TIMEOUT_VC, lambda, timeout->AsConstExpr()); + } + } + + else + z = ZInstI(OP_WHEN_V, lambda); + + z.aux = aux; + + if ( ws->IsReturn() ) + { + (void)AddInst(z); + z = ZInstI(OP_RETURN_C); + z.c = ZVal(); + } + + return AddInst(z); + } + const ZAMStmt ZAMCompiler::InitRecord(IDPtr id, RecordType* rt) { auto z = ZInstI(OP_INIT_RECORD_V, FrameSlot(id)); diff --git a/src/script_opt/ZAM/Support.cc b/src/script_opt/ZAM/Support.cc index f813f86758..b1056c7608 100644 --- a/src/script_opt/ZAM/Support.cc +++ b/src/script_opt/ZAM/Support.cc @@ -20,20 +20,6 @@ bool ZAM_error = false; bool is_ZAM_compilable(const ProfileFunc* pf, const char** reason) { - if ( pf->NumLambdas() > 0 ) - { - if ( reason ) - *reason = "use of lambda"; - return false; - } - - if ( pf->NumWhenStmts() > 0 ) - { - if ( reason ) - *reason = "use of \"when\""; - return false; - } - auto b = pf->ProfiledBody(); auto is_hook = pf->ProfiledFunc()->Flavor() == FUNC_FLAVOR_HOOK; if ( b && ! script_is_valid(b, is_hook) ) diff --git a/src/script_opt/ZAM/Vars.cc b/src/script_opt/ZAM/Vars.cc index 4bd1653c80..61460d3cbd 100644 --- a/src/script_opt/ZAM/Vars.cc +++ b/src/script_opt/ZAM/Vars.cc @@ -24,6 +24,17 @@ bool ZAMCompiler::IsUnused(const IDPtr& id, const Stmt* where) const return ! usage || ! usage->HasID(id.get()); } +bool ZAMCompiler::IsCapture(const ID* id) const + { + const auto& c = pf->CapturesOffsets(); + return c.find(id) != c.end(); + } + +int ZAMCompiler::CaptureOffset(const ID* id) const + { + return pf->CapturesOffsets().find(id)->second; + } + void ZAMCompiler::LoadParam(const ID* id) { if ( id->IsType() ) @@ -64,7 +75,24 @@ const ZAMStmt ZAMCompiler::LoadGlobal(const ID* id) // We use the id_val for reporting used-but-not-set errors. z.aux = new ZInstAux(0); - z.aux->id_val = id; + z.aux->id_val = {NewRef{}, const_cast(id)}; + + return AddInst(z, true); + } + +const ZAMStmt ZAMCompiler::LoadCapture(const ID* id) + { + ZOp op; + + if ( ZVal::IsManagedType(id->GetType()) ) + op = OP_LOAD_MANAGED_CAPTURE_VV; + else + op = OP_LOAD_CAPTURE_VV; + + auto slot = RawSlot(id); + + ZInstI z(op, slot, CaptureOffset(id)); + z.op_type = OP_VV_I2; return AddInst(z, true); } @@ -83,6 +111,9 @@ int ZAMCompiler::FrameSlot(const ID* id) if ( id->IsGlobal() ) (void)LoadGlobal(id); + else if ( IsCapture(id) ) + (void)LoadCapture(id); + return slot; } @@ -103,6 +134,13 @@ int ZAMCompiler::Frame1Slot(const ID* id, ZAMOp1Flavor fl) if ( id->IsGlobal() ) pending_global_store = global_id_to_info[id]; + else if ( IsCapture(id) ) + pending_capture_store = CaptureOffset(id); + + // Make sure we don't think we're storing to both a global and + // a capture. + ASSERT(pending_global_store == -1 || pending_capture_store == -1); + return slot; } diff --git a/src/script_opt/ZAM/ZBody.cc b/src/script_opt/ZAM/ZBody.cc index fcb831b351..63ebe9a7e4 100644 --- a/src/script_opt/ZAM/ZBody.cc +++ b/src/script_opt/ZAM/ZBody.cc @@ -31,6 +31,11 @@ namespace zeek::detail using std::vector; +// Thrown when a call inside a "when" delays. +class ZAMDelayedCallException : public InterpreterException + { + }; + static bool did_init = false; // Count of how often each type of ZOP executed, and how much CPU it @@ -188,10 +193,10 @@ ZBody::~ZBody() void ZBody::SetInsts(vector& _insts) { - ninst = _insts.size(); - auto insts_copy = new ZInst[ninst]; + end_pc = _insts.size(); + auto insts_copy = new ZInst[end_pc]; - for ( auto i = 0U; i < ninst; ++i ) + for ( auto i = 0U; i < end_pc; ++i ) insts_copy[i] = *_insts[i]; insts = insts_copy; @@ -201,10 +206,10 @@ void ZBody::SetInsts(vector& _insts) void ZBody::SetInsts(vector& instsI) { - ninst = instsI.size(); - auto insts_copy = new ZInst[ninst]; + end_pc = instsI.size(); + auto insts_copy = new ZInst[end_pc]; - for ( auto i = 0U; i < ninst; ++i ) + for ( auto i = 0U; i < end_pc; ++i ) { auto& iI = *instsI[i]; insts_copy[i] = iI; @@ -223,7 +228,7 @@ void ZBody::InitProfile() { inst_count = new vector; inst_CPU = new vector; - for ( auto i = 0U; i < ninst; ++i ) + for ( auto i = 0U; i < end_pc; ++i ) { inst_count->push_back(0); inst_CPU->push_back(0.0); @@ -240,7 +245,7 @@ ValPtr ZBody::Exec(Frame* f, StmtFlowType& flow) double t = analysis_options.profile_ZAM ? util::curr_CPU_time() : 0.0; #endif - auto val = DoExec(f, 0, flow); + auto val = DoExec(f, flow); #ifdef DEBUG if ( analysis_options.profile_ZAM ) @@ -250,10 +255,9 @@ ValPtr ZBody::Exec(Frame* f, StmtFlowType& flow) return val; } -ValPtr ZBody::DoExec(Frame* f, int start_pc, StmtFlowType& flow) +ValPtr ZBody::DoExec(Frame* f, StmtFlowType& flow) { - int pc = start_pc; - const int end_pc = ninst; + int pc = 0; // Return value, or nil if none. const ZVal* ret_u = nullptr; @@ -288,6 +292,9 @@ ValPtr ZBody::DoExec(Frame* f, int start_pc, StmtFlowType& flow) flow = FLOW_RETURN; // can be over-written by a Hook-Break + // Clear any leftover error state. + ZAM_error = false; + while ( pc < end_pc && ! ZAM_error ) { auto& z = insts[pc]; @@ -364,9 +371,6 @@ ValPtr ZBody::DoExec(Frame* f, int start_pc, StmtFlowType& flow) delete[] frame; } - // Clear any error state. - ZAM_error = false; - return result; } @@ -444,7 +448,7 @@ void ZBody::Dump() const printf("Final code:\n"); - for ( unsigned i = 0; i < ninst; ++i ) + for ( unsigned i = 0; i < end_pc; ++i ) { auto& inst = insts[i]; printf("%d: ", i); @@ -467,25 +471,6 @@ TraversalCode ZBody::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -ValPtr ZAMResumption::Exec(Frame* f, StmtFlowType& flow) - { - return am->DoExec(f, xfer_pc, flow); - } - -void ZAMResumption::StmtDescribe(ODesc* d) const - { - d->Add(""); - } - -TraversalCode ZAMResumption::Traverse(TraversalCallback* cb) const - { - TraversalCode tc = cb->PreStmt(this); - HANDLE_TC_STMT_PRE(tc); - - tc = cb->PostStmt(this); - HANDLE_TC_STMT_POST(tc); - } - // Unary vector operation of v1 v2. static void vec_exec(ZOp op, TypePtr t, VectorVal*& v1, const VectorVal* v2, const ZInst& z) { diff --git a/src/script_opt/ZAM/ZBody.h b/src/script_opt/ZAM/ZBody.h index 512fa56b15..6a41357738 100644 --- a/src/script_opt/ZAM/ZBody.h +++ b/src/script_opt/ZAM/ZBody.h @@ -52,12 +52,10 @@ public: void ProfileExecution() const; protected: - friend class ZAMResumption; - // Initializes profiling information, if needed. void InitProfile(); - ValPtr DoExec(Frame* f, int start_pc, StmtFlowType& flow); + ValPtr DoExec(Frame* f, StmtFlowType& flow); // Run-time checking for "any" type being consistent with // expected typed. Returns true if the type match is okay. @@ -73,7 +71,7 @@ private: const char* func_name = nullptr; const ZInst* insts = nullptr; - unsigned int ninst = 0; + unsigned int end_pc = 0; FrameReMap frame_denizens; int frame_size; @@ -117,29 +115,6 @@ private: CaseMaps str_cases; }; -// This is a statement that resumes execution into a code block in a -// ZBody. Used for deferred execution for "when" statements. - -class ZAMResumption : public Stmt - { -public: - ZAMResumption(ZBody* _am, int _xfer_pc) : Stmt(STMT_ZAM_RESUMPTION), am(_am), xfer_pc(_xfer_pc) - { - } - - ValPtr Exec(Frame* f, StmtFlowType& flow) override; - - StmtPtr Duplicate() override { return {NewRef{}, this}; } - - void StmtDescribe(ODesc* d) const override; - -protected: - TraversalCode Traverse(TraversalCallback* cb) const override; - - ZBody* am; - int xfer_pc = 0; - }; - // Prints the execution profile. extern void report_ZOP_profile(); diff --git a/src/script_opt/ZAM/ZInst.cc b/src/script_opt/ZAM/ZInst.cc index de47d0afb8..04605df140 100644 --- a/src/script_opt/ZAM/ZInst.cc +++ b/src/script_opt/ZAM/ZInst.cc @@ -383,16 +383,20 @@ bool ZInstI::IsDirectAssignment() const switch ( op ) { - case OP_ASSIGN_VV_N: case OP_ASSIGN_VV_A: + case OP_ASSIGN_VV_D: + case OP_ASSIGN_VV_F: + case OP_ASSIGN_VV_I: + case OP_ASSIGN_VV_L: + case OP_ASSIGN_VV_N: case OP_ASSIGN_VV_O: case OP_ASSIGN_VV_P: case OP_ASSIGN_VV_R: case OP_ASSIGN_VV_S: - case OP_ASSIGN_VV_F: case OP_ASSIGN_VV_T: + case OP_ASSIGN_VV_U: case OP_ASSIGN_VV_V: - case OP_ASSIGN_VV_L: + case OP_ASSIGN_VV_a: case OP_ASSIGN_VV_f: case OP_ASSIGN_VV_t: case OP_ASSIGN_VV: @@ -403,6 +407,21 @@ bool ZInstI::IsDirectAssignment() const } } +bool ZInstI::HasCaptures() const + { + switch ( op ) + { + case OP_LAMBDA_VV: + case OP_WHEN_V: + case OP_WHEN_TIMEOUT_VV: + case OP_WHEN_TIMEOUT_VC: + return true; + + default: + return false; + } + } + bool ZInstI::HasSideEffects() const { return op_side_effects[op]; @@ -647,6 +666,11 @@ bool ZInstI::IsGlobalLoad() const return global_ops.count(op) > 0; } +bool ZInstI::IsCaptureLoad() const + { + return op == OP_LOAD_CAPTURE_VV || op == OP_LOAD_MANAGED_CAPTURE_VV; + } + void ZInstI::InitConst(const ConstExpr* ce) { auto v = ce->ValuePtr(); diff --git a/src/script_opt/ZAM/ZInst.h b/src/script_opt/ZAM/ZInst.h index 8102fbbb29..8e9158e044 100644 --- a/src/script_opt/ZAM/ZInst.h +++ b/src/script_opt/ZAM/ZInst.h @@ -5,6 +5,7 @@ #pragma once #include "zeek/Desc.h" +#include "zeek/Func.h" #include "zeek/script_opt/ZAM/BuiltInSupport.h" #include "zeek/script_opt/ZAM/Support.h" #include "zeek/script_opt/ZAM/ZOp.h" @@ -220,6 +221,9 @@ public: // True if this instruction is of the form "v1 = v2". bool IsDirectAssignment() const; + // True if this instruction includes captures in its aux slots. + bool HasCaptures() const; + // True if this instruction has side effects when executed, so // should not be pruned even if it has a dead assignment. bool HasSideEffects() const; @@ -247,9 +251,17 @@ public: // the ZAM frame. bool IsGlobalLoad() const; + // True if the instruction corresponds to loading a capture into + // the ZAM frame. + bool IsCaptureLoad() const; + + // True if the instruction does not correspond to a load from the + // ZAM frame. + bool IsNonLocalLoad() const { return IsGlobalLoad() || IsCaptureLoad(); } + // True if the instruction corresponds to some sort of load, - // either from the interpreter frame or of a global. - bool IsLoad() const { return op_type == OP_VV_FRAME || IsGlobalLoad(); } + // either from the interpreter frame or of a global/capture. + bool IsLoad() const { return op_type == OP_VV_FRAME || IsNonLocalLoad(); } // True if the instruction corresponds to storing a global. bool IsGlobalStore() const { return op == OP_STORE_GLOBAL_V; } @@ -317,6 +329,7 @@ public: slots = ints = new int[n]; constants = new ValPtr[n]; types = new TypePtr[n]; + is_managed = new bool[n]; } } @@ -325,6 +338,7 @@ public: delete[] ints; delete[] constants; delete[] types; + delete[] is_managed; delete[] cat_args; } @@ -384,6 +398,7 @@ public: ints[i] = slot; constants[i] = nullptr; types[i] = t; + is_managed[i] = t ? ZVal::IsManagedType(t) : false; } // Same but for constants. @@ -392,6 +407,7 @@ public: ints[i] = -1; constants[i] = c; types[i] = nullptr; + is_managed[i] = false; } // Member variables. We could add accessors for manipulating @@ -404,24 +420,37 @@ public: // if not, it's nil). // // We track associated types, too, enabling us to use - // ZVal::ToVal to convert frame slots or constants to ValPtr's. + // ZVal::ToVal to convert frame slots or constants to ValPtr's; + // and, as a performance optimization, whether those types + // indicate the slot needs to be managed. int n; // size of arrays int* slots = nullptr; // either nil or points to ints int* ints = nullptr; ValPtr* constants = nullptr; TypePtr* types = nullptr; + bool* is_managed = nullptr; + + // Ingredients associated with lambdas ... + ScriptFuncPtr primary_func; + + // ... and its name. + std::string lambda_name; + + // For "when" statements. Needs to be non-const so we can + // Instantiate() it as needed. + WhenInfo* wi; // A parallel array for the cat() built-in replacement. std::unique_ptr* cat_args = nullptr; // Used for accessing function names. - const ID* id_val = nullptr; + IDPtr id_val = nullptr; - // Whether the instruction can lead to globals changing. - // Currently only needed by the optimizer, but convenient - // to store here. - bool can_change_globals = false; + // Whether the instruction can lead to globals/captures changing. + // Currently only needed by the optimizer, but convenient to + // store here. + bool can_change_non_locals = false; // The following is only used for OP_CONSTRUCT_KNOWN_RECORD_V, // to map elements in slots/constants/types to record field offsets. diff --git a/testing/btest/Baseline.zam/bifs.from_json-11/.stderr b/testing/btest/Baseline.zam/bifs.from_json-11/.stderr index 83cccb763e..a94a645929 100644 --- a/testing/btest/Baseline.zam/bifs.from_json-11/.stderr +++ b/testing/btest/Baseline.zam/bifs.from_json-11/.stderr @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/from_json.zeek, line 8: required field Foo$hello is missing in JSON (from_json({"t":null}, ::#0)) -error in <...>/from_json.zeek, line 9: required field Foo$hello is null in JSON (from_json({"hello": null, "t": true}, ::#2)) +error in <...>/from_json.zeek, line 8: required field Foo$hello is missing in JSON (from_json({"t":null}, ::#0, from_json_default_key_mapper)) +error in <...>/from_json.zeek, line 9: required field Foo$hello is null in JSON (from_json({"hello": null, "t": true}, ::#2, from_json_default_key_mapper)) diff --git a/testing/btest/Baseline.zam/bifs.from_json-13/.stderr b/testing/btest/Baseline.zam/bifs.from_json-13/.stderr new file mode 100644 index 0000000000..ec05b0d535 --- /dev/null +++ b/testing/btest/Baseline.zam/bifs.from_json-13/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/from_json.zeek, line 7: required field Foo$id_field is missing in JSON (from_json({"id-field": "Hello!"}, ::#0, from_json_default_key_mapper)) diff --git a/testing/btest/Baseline.zam/bifs.from_json-13/.stdout b/testing/btest/Baseline.zam/bifs.from_json-13/.stdout new file mode 100644 index 0000000000..b75189ee1e --- /dev/null +++ b/testing/btest/Baseline.zam/bifs.from_json-13/.stdout @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=, valid=F] +[v=[id_field=Hello!], valid=T] diff --git a/testing/btest/Baseline.zam/bifs.from_json-14/.stderr b/testing/btest/Baseline.zam/bifs.from_json-14/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline.zam/bifs.from_json-14/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline.zam/bifs.from_json-14/.stdout b/testing/btest/Baseline.zam/bifs.from_json-14/.stdout new file mode 100644 index 0000000000..a7f883dc82 --- /dev/null +++ b/testing/btest/Baseline.zam/bifs.from_json-14/.stdout @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +T +[ietf_mud_mud=[mud_version=1, mud_url=https://lighting.example.com/lightbulb2000, last_update=2019-01-28T11:20:51+01:00, cache_validity=48, is_supported=T, systeminfo=The BMS Example Light Bulb, from_device_policy=[access_lists=[access_list=[[name=mud-76100-v6fr]]]], to_device_policy=[access_lists=[access_list=[[name=mud-76100-v6to]]]]], ietf_access_control_list_acls=[acl=[[name=mud-76100-v6to, _type=ipv6-acl-type, aces=[ace=[[name=cl0-todev, matches=[ipv6=[ietf_acldns_dst_dnsname=, ietf_mud_direction_initiated=, destination_port=], tcp=[ietf_acldns_dst_dnsname=, ietf_mud_direction_initiated=from-device, destination_port=]]]]]], [name=mud-76100-v6fr, _type=ipv6-acl-type, aces=[ace=[[name=cl0-frdev, matches=[ipv6=[ietf_acldns_dst_dnsname=test.example.com, ietf_mud_direction_initiated=, destination_port=], tcp=[ietf_acldns_dst_dnsname=, ietf_mud_direction_initiated=from-device, destination_port=[operator=eq, _port=443]]]]]]]]]] diff --git a/testing/btest/Baseline.zam/bifs.from_json-2/.stderr b/testing/btest/Baseline.zam/bifs.from_json-2/.stderr index 5fe8977244..372f599e7f 100644 --- a/testing/btest/Baseline.zam/bifs.from_json-2/.stderr +++ b/testing/btest/Baseline.zam/bifs.from_json-2/.stderr @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/from_json.zeek, line 4: from_json() requires a type argument (from_json([], 10)) +error in <...>/from_json.zeek, line 4: from_json() requires a type argument (from_json([], 10, from_json_default_key_mapper)) diff --git a/testing/btest/Baseline.zam/bifs.from_json-3/.stderr b/testing/btest/Baseline.zam/bifs.from_json-3/.stderr index e8e76fd280..233c4318e1 100644 --- a/testing/btest/Baseline.zam/bifs.from_json-3/.stderr +++ b/testing/btest/Baseline.zam/bifs.from_json-3/.stderr @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/from_json.zeek, line 4: JSON parse error: Missing a closing quotation mark in string. Offset: 5 (from_json({"hel, ::#0)) +error in <...>/from_json.zeek, line 4: JSON parse error: Missing a closing quotation mark in string. Offset: 5 (from_json({"hel, ::#0, from_json_default_key_mapper)) diff --git a/testing/btest/Baseline.zam/bifs.from_json-4/.stderr b/testing/btest/Baseline.zam/bifs.from_json-4/.stderr index ed567bc817..42e6dad6f0 100644 --- a/testing/btest/Baseline.zam/bifs.from_json-4/.stderr +++ b/testing/btest/Baseline.zam/bifs.from_json-4/.stderr @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/from_json.zeek, line 9: cannot convert JSON type 'array' to Zeek type 'bool' (from_json([], ::#0)) -error in <...>/from_json.zeek, line 10: cannot convert JSON type 'string' to Zeek type 'bool' (from_json({"a": "hello"}, ::#2)) +error in <...>/from_json.zeek, line 9: cannot convert JSON type 'array' to Zeek type 'bool' (from_json([], ::#0, from_json_default_key_mapper)) +error in <...>/from_json.zeek, line 10: cannot convert JSON type 'string' to Zeek type 'bool' (from_json({"a": "hello"}, ::#2, from_json_default_key_mapper)) diff --git a/testing/btest/Baseline.zam/bifs.from_json-5/.stderr b/testing/btest/Baseline.zam/bifs.from_json-5/.stderr index a8d80a29c7..ed0056356c 100644 --- a/testing/btest/Baseline.zam/bifs.from_json-5/.stderr +++ b/testing/btest/Baseline.zam/bifs.from_json-5/.stderr @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/from_json.zeek, line 4: tables are not supported (from_json([], ::#0)) +error in <...>/from_json.zeek, line 4: tables are not supported (from_json([], ::#0, from_json_default_key_mapper)) diff --git a/testing/btest/Baseline.zam/bifs.from_json-6/.stderr b/testing/btest/Baseline.zam/bifs.from_json-6/.stderr index 2ac321f4e0..7a7c048f3c 100644 --- a/testing/btest/Baseline.zam/bifs.from_json-6/.stderr +++ b/testing/btest/Baseline.zam/bifs.from_json-6/.stderr @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/from_json.zeek, line 5: wrong port format, must be <...>/(tcp|udp|icmp|unknown)/ (from_json("80", ::#0)) +error in <...>/from_json.zeek, line 5: wrong port format, must be <...>/(tcp|udp|icmp|unknown)/ (from_json("80", ::#0, from_json_default_key_mapper)) diff --git a/testing/btest/Baseline.zam/bifs.from_json-7/.stderr b/testing/btest/Baseline.zam/bifs.from_json-7/.stderr index fd5ec83642..315336902b 100644 --- a/testing/btest/Baseline.zam/bifs.from_json-7/.stderr +++ b/testing/btest/Baseline.zam/bifs.from_json-7/.stderr @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/from_json.zeek, line 5: index type doesn't match (from_json([[1, false], [2]], ::#0)) -error in <...>/from_json.zeek, line 6: cannot convert JSON type 'number' to Zeek type 'bool' (from_json([[1, false], [2, 1]], ::#2)) +error in <...>/from_json.zeek, line 5: index type doesn't match (from_json([[1, false], [2]], ::#0, from_json_default_key_mapper)) +error in <...>/from_json.zeek, line 6: cannot convert JSON type 'number' to Zeek type 'bool' (from_json([[1, false], [2, 1]], ::#2, from_json_default_key_mapper)) diff --git a/testing/btest/Baseline.zam/bifs.from_json-8/.stderr b/testing/btest/Baseline.zam/bifs.from_json-8/.stderr index ba565788a5..6c89085ca4 100644 --- a/testing/btest/Baseline.zam/bifs.from_json-8/.stderr +++ b/testing/btest/Baseline.zam/bifs.from_json-8/.stderr @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. error: error compiling pattern /^?(.|\n)*(([[:print:]]{-}[[:alnum:]]foo))/ -error in <...>/from_json.zeek, line 5: error compiling pattern (from_json("/([[:print:]]{-}[[:alnum:]]foo)/", ::#0)) +error in <...>/from_json.zeek, line 5: error compiling pattern (from_json("/([[:print:]]{-}[[:alnum:]]foo)/", ::#0, from_json_default_key_mapper)) diff --git a/testing/btest/Baseline.zam/bifs.from_json-9/.stderr b/testing/btest/Baseline.zam/bifs.from_json-9/.stderr index 14894c2146..7ff4fdd21e 100644 --- a/testing/btest/Baseline.zam/bifs.from_json-9/.stderr +++ b/testing/btest/Baseline.zam/bifs.from_json-9/.stderr @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/from_json.zeek, line 7: 'Yellow' is not a valid enum for 'Color'. (from_json("Yellow", ::#0)) +error in <...>/from_json.zeek, line 7: 'Yellow' is not a valid enum for 'Color'. (from_json("Yellow", ::#0, from_json_default_key_mapper)) diff --git a/testing/btest/Baseline.zam/broker.store.create-failure/zeek.err b/testing/btest/Baseline.zam/broker.store.create-failure/zeek.err index 901a6bb7b2..05fd841e54 100644 --- a/testing/btest/Baseline.zam/broker.store.create-failure/zeek.err +++ b/testing/btest/Baseline.zam/broker.store.create-failure/zeek.err @@ -2,8 +2,8 @@ error in <...>/store.zeek, line 780: Failed to attach master store backend_failure: (Broker::__create_master(../fail, Broker::SQLITE, Broker::options.2)) error in <...>/store.zeek, line 780: Could not create Broker master store '../fail' (Broker::__create_master(../fail, Broker::SQLITE, Broker::options.2)) error in <...>/create-failure.zeek, line 49: invalid Broker store handle (Broker::keys(s) and broker::store::{}) -error in <...>/store.zeek, line 794: invalid Broker store handle (Broker::__close(Broker::h) and broker::store::{}) -error in <...>/store.zeek, line 794: invalid Broker store handle (Broker::__close(Broker::h) and broker::store::{}) +error in <...>/store.zeek, line 794: invalid Broker store handle (Broker::__close(Broker::h.12) and broker::store::{}) +error in <...>/store.zeek, line 794: invalid Broker store handle (Broker::__close(Broker::h.21) and broker::store::{}) error in <...>/create-failure.zeek, line 49: invalid Broker store handle (Broker::keys(s) and broker::store::{}) error in <...>/create-failure.zeek, line 49: invalid Broker store handle (Broker::keys(s) and broker::store::{}) error in <...>/create-failure.zeek, line 49: invalid Broker store handle (Broker::keys(s) and broker::store::{}) diff --git a/testing/btest/Baseline.zam/core.when-interpreter-exceptions/zeek.output b/testing/btest/Baseline.zam/core.when-interpreter-exceptions/zeek.output new file mode 100644 index 0000000000..d1fba762a4 --- /dev/null +++ b/testing/btest/Baseline.zam/core.when-interpreter-exceptions/zeek.output @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +### NOTE: This file has been sorted with diff-sort. +[f(F)] +[f(T)] +[zeek_init()] +f() done, no exception, T +g() done, no exception, T +received termination signal +runtime error in <...>/when-interpreter-exceptions.zeek, line 103: field value missing: $notset +runtime error in <...>/when-interpreter-exceptions.zeek, line 47: field value missing: $notset +runtime error in <...>/when-interpreter-exceptions.zeek, line 72: field value missing: $notset +runtime error in <...>/when-interpreter-exceptions.zeek, line 91: field value missing: $notset +timeout +timeout g(), F +timeout g(), T diff --git a/testing/btest/Baseline.zam/language.when-capture-errors/out b/testing/btest/Baseline.zam/language.when-capture-errors/out new file mode 100644 index 0000000000..35ae709e9b --- /dev/null +++ b/testing/btest/Baseline.zam/language.when-capture-errors/out @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/when-capture-errors.zeek, lines 19-22: orig1 is used inside "when" statement but not captured (when (0 < g) { print orig1}) +error in <...>/when-capture-errors.zeek, lines 25-28: orig3 is used inside "when" statement but not captured (when (0 < g || orig3) { print g}) +error in <...>/when-capture-errors.zeek, lines 34-38: orig1 is used inside "when" statement but not captured (when (0 < g) { print g} timeout 1.0 sec { print orig1}) +error in <...>/when-capture-errors.zeek, lines 66-70: orig2 is used inside "when" statement but not captured (when [orig1](0 < g) { print orig1} timeout 1.0 sec { print orig2}) +error in <...>/when-capture-errors.zeek, lines 76-80: orig3 is captured but not used inside "when" statement (when [orig1, orig2, orig3](0 < g) { print orig1} timeout 1.0 sec { print orig2}) +error in <...>/when-capture-errors.zeek, line 83: no such local identifier: l1 +error in <...>/when-capture-errors.zeek, line 89: no such local identifier: l2 diff --git a/testing/btest/Baseline/language.uninitialized-local/out b/testing/btest/Baseline/language.uninitialized-local/out index 71f956d196..44e2ccadc9 100644 --- a/testing/btest/Baseline/language.uninitialized-local/out +++ b/testing/btest/Baseline/language.uninitialized-local/out @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in <...>/uninitialized-local.zeek, line 16: value used but not set (my_string) +expression error in <...>/uninitialized-local.zeek, line 19: value used but not set (my_string) diff --git a/testing/btest/Baseline/language.when-capture-errors/out b/testing/btest/Baseline/language.when-capture-errors/out index d73d761602..35ae709e9b 100644 --- a/testing/btest/Baseline/language.when-capture-errors/out +++ b/testing/btest/Baseline/language.when-capture-errors/out @@ -2,7 +2,7 @@ error in <...>/when-capture-errors.zeek, lines 19-22: orig1 is used inside "when" statement but not captured (when (0 < g) { print orig1}) error in <...>/when-capture-errors.zeek, lines 25-28: orig3 is used inside "when" statement but not captured (when (0 < g || orig3) { print g}) error in <...>/when-capture-errors.zeek, lines 34-38: orig1 is used inside "when" statement but not captured (when (0 < g) { print g} timeout 1.0 sec { print orig1}) -error in <...>/when-capture-errors.zeek, lines 66-70: orig2 is used inside "when" statement but not captured (when (0 < g) { print orig1} timeout 1.0 sec { print orig2}) -error in <...>/when-capture-errors.zeek, lines 76-80: orig3 is captured but not used inside "when" statement (when (0 < g) { print orig1} timeout 1.0 sec { print orig2}) +error in <...>/when-capture-errors.zeek, lines 66-70: orig2 is used inside "when" statement but not captured (when [orig1](0 < g) { print orig1} timeout 1.0 sec { print orig2}) +error in <...>/when-capture-errors.zeek, lines 76-80: orig3 is captured but not used inside "when" statement (when [orig1, orig2, orig3](0 < g) { print orig1} timeout 1.0 sec { print orig2}) error in <...>/when-capture-errors.zeek, line 83: no such local identifier: l1 error in <...>/when-capture-errors.zeek, line 89: no such local identifier: l2 diff --git a/testing/btest/Baseline/language.when-unitialized-rhs/out b/testing/btest/Baseline/language.when-unitialized-rhs/out index fef12b931c..fc12245c91 100644 --- a/testing/btest/Baseline/language.when-unitialized-rhs/out +++ b/testing/btest/Baseline/language.when-unitialized-rhs/out @@ -1,6 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in <...>/when-unitialized-rhs.zeek, line 12: value used but not set (crashMe) -expression error in <...>/when-unitialized-rhs.zeek, line 17: value used but not set (x) +expression error in <...>/when-unitialized-rhs.zeek, line 13: value used but not set (crashMe) +expression error in <...>/when-unitialized-rhs.zeek, line 18: value used but not set (x) 1 2 3 diff --git a/testing/btest/language/uninitialized-local.ZAM.zeek b/testing/btest/language/uninitialized-local.ZAM.zeek new file mode 100644 index 0000000000..d41f3431df --- /dev/null +++ b/testing/btest/language/uninitialized-local.ZAM.zeek @@ -0,0 +1,23 @@ +# A version of uninitialized-local.zeek suitable for ZAM's behavior. +# @TEST-REQUIRES: test "${ZEEK_ZAM}" == "1" +# @TEST-EXEC-FAIL: zeek -b %INPUT >out 2>&1 + +event testit() &priority=10 + { + local my_count: count = 10; + } + +event testit() + { + # my_string's value occupies same Frame offset as my_count's from above + # handler, but execution of this handler body should still "initialize" + # it to a null value instead of referring to a left-over value of my_count. + local my_string: string; + local my_vector: vector of string; + my_vector[0] = my_string; + } + +event zeek_init() + { + event testit(); + } diff --git a/testing/btest/language/uninitialized-local.zeek b/testing/btest/language/uninitialized-local.zeek index 6d8e26be72..c6f2972553 100644 --- a/testing/btest/language/uninitialized-local.zeek +++ b/testing/btest/language/uninitialized-local.zeek @@ -1,3 +1,6 @@ +# Not suitable for ZAM since it spots the error at compile time and exits +# with an error status code. +# @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1" # @TEST-EXEC: zeek -b %INPUT >out 2>&1 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out diff --git a/testing/btest/language/when-unitialized-rhs.zeek b/testing/btest/language/when-unitialized-rhs.zeek index 2e30ae35bb..88e71ce111 100644 --- a/testing/btest/language/when-unitialized-rhs.zeek +++ b/testing/btest/language/when-unitialized-rhs.zeek @@ -1,5 +1,6 @@ -# C++-compiled scripts don't check for missing functions - not clear it's +# Compiled scripts don't fully check for missing functions - not clear it's # worth fixing that. +# @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1" # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" # @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT >out 2>&1 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 8a79b0e32b..d57e1cb4df 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -d59caff708b41db11fa0cbfe0b1f95b46c3e700e +828845c99306c6d5d6811fa42987de5b16f530b9 diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index 72b4b0a144..c5dd200880 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -7162c907aa25e155ea841710ef30b65afb578c3f +b121bfe4d869f1f5e334505b970cd456558ef6a1