Merge remote-tracking branch 'origin/topic/vern/zam-feature-complete'

* origin/topic/vern/zam-feature-complete: (23 commits)
  documentation updates
  updates to ZAM-specific BTest baseline
  Address PR review feedback on zam-feature-complete
  Updates to BTest baselines due to previous BTest tweaks
  tweaks to BTests for ZAM feature completeness; new ZAM-only btest
  removal of unused functionality and some follow-on simplifications
  feature completeness for ZAM
  -O gen-C++ tweaks to be compatible with ZAM changes
  ZAM support for "when" statements
  ZAM changes intermixed with lambda and "when" support
  WhenStmt/WhenInfo restructuring in support of ZAM "when" statements
  ZAM support for lambdas
  ZAM internals have a notion of "captures" as global-like variables
  AST profiling enhnacements in support of script optimization for lambdas/whens
  refinements to LambdaExpr's to provide flexibility, support for ZVal captures
  support in ScriptFunc class for ZVal-oriented vector of captures
  simplifications to the Frame class now that it no longer has to support old-style captures
  use Ingredients directly for constructing functions
  the "Capture" struct is now a class
  more debugging information when dumping script optimization data structures
  ...

CHANGES

@@ -1,3 +1,53 @@
+6.1.0-dev.152 | 2023-06-30 10:19:19 +0200
+
+  * documentation updates (Vern Paxson, Corelight)
+
+  * updates to ZAM-specific BTest baseline (Vern Paxson, Corelight)
+
+  * Address PR review feedback on zam-feature-complete (Vern Paxson, Corelight)
+
+  * Updates to BTest baselines due to previous BTest tweaks (Vern Paxson, Corelight)
+
+  * tweaks to BTests for ZAM feature completeness; new ZAM-only btest (Vern Paxson, Corelight)
+
+  * removal of unused functionality and some follow-on simplifications (Vern Paxson, Corelight)
+
+  * feature completeness for ZAM (Vern Paxson, Corelight)
+
+  * -O gen-C++ tweaks to be compatible with ZAM changes (Vern Paxson, Corelight)
+
+  * ZAM support for "when" statements (Vern Paxson, Corelight)
+
+  * ZAM changes intermixed with lambda and "when" support (Vern Paxson, Corelight)
+
+  * WhenStmt/WhenInfo restructuring in support of ZAM "when" statements (Vern Paxson, Corelight)
+
+  * ZAM support for lambdas (Vern Paxson, Corelight)
+
+  * ZAM internals have a notion of "captures" as global-like variables (Vern Paxson, Corelight)
+
+  * AST profiling enhnacements in support of script optimization for lambdas/whens (Vern Paxson, Corelight)
+
+  * refinements to LambdaExpr's to provide flexibility, support for ZVal captures (Vern Paxson, Corelight)
+
+  * support in ScriptFunc class for ZVal-oriented vector of captures (Vern Paxson, Corelight)
+
+  * simplifications to the Frame class now that it no longer has to support old-style captures (Vern Paxson, Corelight)
+
+  * use Ingredients directly for constructing functions (Vern Paxson, Corelight)
+
+  * the "Capture" struct is now a class (Vern Paxson, Corelight)
+
+  * more debugging information when dumping script optimization data structures (Vern Paxson, Corelight)
+
+  * bug fixes for script optimization intermediate forms (Vern Paxson, Corelight)
+
+  * clarifying comments, interface tightening (Vern Paxson, Corelight)
+
+  * added some class accessors/set-ers (Vern Paxson, Corelight)
+
+  * Update doc submodule [nomail] [skip ci] (zeek-bot)
+
 6.1.0-dev.127 | 2023-06-29 11:22:58 -0700
 
   * Move CMake template files to separate directory (Tim Wojtulewicz, Corelight)

VERSION

@@ -1 +1 @@
-6.1.0-dev.127
+6.1.0-dev.152

auxil/gen-zam

@@ -1 +1 @@
-Subproject commit 16841c95849d4d82f239bfb0c46bc217af368da2
+Subproject commit f62a0215f063bc768d7e85b4a49faeb07ca3cb14

src/Expr.cc

@@ -25,6 +25,7 @@
 #include "zeek/digest.h"
 #include "zeek/module_util.h"
 #include "zeek/script_opt/ExprOptInfo.h"
+#include "zeek/script_opt/ScriptOpt.h"
 
 namespace zeek::detail
 	{
@@ -4644,14 +4645,15 @@ void CallExpr::ExprDescribe(ODesc* d) const
 		args->Describe(d);
 	}
 
-LambdaExpr::LambdaExpr(std::unique_ptr<FunctionIngredients> arg_ing, IDPList arg_outer_ids,
+LambdaExpr::LambdaExpr(FunctionIngredientsPtr arg_ing, IDPList arg_outer_ids, std::string name,
                        StmtPtr when_parent)
 	: Expr(EXPR_LAMBDA)
 	{
 	ingredients = std::move(arg_ing);
 	outer_ids = std::move(arg_outer_ids);
 
-	SetType(ingredients->GetID()->GetType());
+	auto ingr_t = ingredients->GetID()->GetType<FuncType>();
+	SetType(ingr_t);
 
 	if ( ! CheckCaptures(when_parent) )
 		{
@@ -4659,47 +4661,61 @@ LambdaExpr::LambdaExpr(std::unique_ptr<FunctionIngredients> arg_ing, IDPList arg
 		return;
 		}
 
-	// Install a dummy version of the function globally for use only
+	// Install a primary version of the function globally.  This is used
-	// when broker provides a closure.
+	// by both broker (for transmitting closures) and script optimization
-	auto dummy_func = make_intrusive<ScriptFunc>(ingredients->GetID());
+	// (replacing its AST body with a compiled one).
-	dummy_func->AddBody(ingredients->Body(), ingredients->Inits(), ingredients->FrameSize(),
+	primary_func = make_intrusive<ScriptFunc>(ingredients->GetID());
-	                    ingredients->Priority());
+	primary_func->SetOuterIDs(outer_ids);
 
-	dummy_func->SetOuterIDs(outer_ids);
+	// When we build the body, it will get updated with initialization
+	// statements.  Update the ingredients to reflect the new body,
+	// and no more need for initializers.
+	primary_func->AddBody(*ingredients);
+	primary_func->SetScope(ingredients->Scope());
+	ingredients->ClearInits();
 
-	// Get the body's "string" representation.
+	if ( name.empty() )
-	ODesc d;
+		BuildName();
-	dummy_func->Describe(&d);
+	else
-
+		my_name = name;
-	for ( ;; )
-		{
-		hash128_t h;
-		KeyedHash::Hash128(d.Bytes(), d.Len(), &h);
-
-		my_name = "lambda_<" + std::to_string(h[0]) + ">";
-		auto fullname = make_full_var_name(current_module.data(), my_name.data());
-		const auto& id = global_scope()->Find(fullname);
-
-		if ( id )
-			// Just try again to make a unique lambda name.
-			// If two peer processes need to agree on the same
-			// lambda name, this assumes they're loading the same
-			// scripts and thus have the same hash collisions.
-			d.Add(" ");
-		else
-			break;
-		}
 
 	// Install that in the global_scope
 	lambda_id = install_ID(my_name.c_str(), current_module.c_str(), true, false);
 
 	// Update lamb's name
-	dummy_func->SetName(my_name.c_str());
+	primary_func->SetName(my_name.c_str());
 
-	auto v = make_intrusive<FuncVal>(std::move(dummy_func));
+	auto v = make_intrusive<FuncVal>(primary_func);
 	lambda_id->SetVal(std::move(v));
-	lambda_id->SetType(ingredients->GetID()->GetType());
+	lambda_id->SetType(ingr_t);
 	lambda_id->SetConst();
+
+	captures = ingr_t->GetCaptures();
+
+	analyze_lambda(this);
+	}
+
+LambdaExpr::LambdaExpr(LambdaExpr* orig) : Expr(EXPR_LAMBDA)
+	{
+	primary_func = orig->primary_func;
+	ingredients = orig->ingredients;
+	lambda_id = orig->lambda_id;
+	my_name = orig->my_name;
+	private_captures = orig->private_captures;
+
+	// We need to have our own copies of the outer IDs and captures so
+	// we can rename them when inlined.
+	for ( auto i : orig->outer_ids )
+		outer_ids.append(i);
+
+	if ( orig->captures )
+		{
+		captures = std::vector<FuncType::Capture>{};
+		for ( auto& c : *orig->captures )
+			captures->push_back(c);
+		}
+
+	SetType(orig->GetType());
 	}
 
 bool LambdaExpr::CheckCaptures(StmtPtr when_parent)
@@ -4726,7 +4742,7 @@ bool LambdaExpr::CheckCaptures(StmtPtr when_parent)
 
 	for ( const auto& c : *captures )
 		{
-		auto cid = c.id.get();
+		auto cid = c.Id().get();
 
 		if ( ! cid )
 			// This happens for undefined/inappropriate
@@ -4768,7 +4784,7 @@ bool LambdaExpr::CheckCaptures(StmtPtr when_parent)
 
 	for ( const auto& c : *captures )
 		{
-		auto cid = c.id.get();
+		auto cid = c.Id().get();
 		if ( cid && capture_is_matched.count(cid) == 0 )
 			{
 			auto msg = util::fmt("%s is captured but not used inside %s", cid->Name(), desc);
@@ -4784,6 +4800,32 @@ bool LambdaExpr::CheckCaptures(StmtPtr when_parent)
 	return true;
 	}
 
+void LambdaExpr::BuildName()
+	{
+	// Get the body's "string" representation.
+	ODesc d;
+	primary_func->Describe(&d);
+
+	for ( ;; )
+		{
+		hash128_t h;
+		KeyedHash::Hash128(d.Bytes(), d.Len(), &h);
+
+		my_name = "lambda_<" + std::to_string(h[0]) + ">";
+		auto fullname = make_full_var_name(current_module.data(), my_name.data());
+		const auto& id = global_scope()->Find(fullname);
+
+		if ( id )
+			// Just try again to make a unique lambda name.
+			// If two peer processes need to agree on the same
+			// lambda name, this assumes they're loading the same
+			// scripts and thus have the same hash collisions.
+			d.Add(" ");
+		else
+			break;
+		}
+	}
+
 ScopePtr LambdaExpr::GetScope() const
 	{
 	return ingredients->Scope();
@@ -4792,12 +4834,23 @@ ScopePtr LambdaExpr::GetScope() const
 ValPtr LambdaExpr::Eval(Frame* f) const
 	{
 	auto lamb = make_intrusive<ScriptFunc>(ingredients->GetID());
-	lamb->AddBody(ingredients->Body(), ingredients->Inits(), ingredients->FrameSize(),
-	              ingredients->Priority());
 
+	StmtPtr body = primary_func->GetBodies()[0].stmts;
+
+	if ( run_state::is_parsing )
+		// We're evaluating this lambda at parse time, which happens
+		// for initializations.  If we're doing script optimization
+		// then the current version of the body might be left in an
+		// inconsistent state (e.g., if it's replaced with ZAM code)
+		// causing problems if we execute this lambda subsequently.
+		// To avoid that problem, we duplicate the AST so it's
+		// distinct.
+		body = body->Duplicate();
+
+	lamb->AddBody(*ingredients, body);
 	lamb->CreateCaptures(f);
 
-	// Set name to corresponding dummy func.
+	// Set name to corresponding master func.
 	// Allows for lookups by the receiver.
 	lamb->SetName(my_name.c_str());
 
@@ -4807,6 +4860,22 @@ ValPtr LambdaExpr::Eval(Frame* f) const
 void LambdaExpr::ExprDescribe(ODesc* d) const
 	{
 	d->Add(expr_name(Tag()));
+
+	if ( captures && d->IsReadable() )
+		{
+		d->Add("[");
+
+		for ( auto& c : *captures )
+			{
+			if ( &c != &(*captures)[0] )
+				d->AddSP(", ");
+
+			d->Add(c.Id()->Name());
+			}
+
+		d->Add("]");
+		}
+
 	ingredients->Body()->Describe(d);
 	}
 

src/Expr.h

@@ -28,8 +28,11 @@ namespace detail
 class Frame;
 class Scope;
 class FunctionIngredients;
+class WhenInfo;
 using IDPtr = IntrusivePtr<ID>;
 using ScopePtr = IntrusivePtr<Scope>;
+using ScriptFuncPtr = IntrusivePtr<ScriptFunc>;
+using FunctionIngredientsPtr = std::shared_ptr<FunctionIngredients>;
 
 enum ExprTag : int
 	{
@@ -1452,12 +1455,17 @@ protected:
 class LambdaExpr final : public Expr
 	{
 public:
-	LambdaExpr(std::unique_ptr<FunctionIngredients> ingredients, IDPList outer_ids,
+	LambdaExpr(FunctionIngredientsPtr ingredients, IDPList outer_ids, std::string name = "",
 	           StmtPtr when_parent = nullptr);
 
 	const std::string& Name() const { return my_name; }
+
 	const IDPList& OuterIDs() const { return outer_ids; }
-	const FunctionIngredients& Ingredients() const { return *ingredients; }
+
+	// Lambda's potentially have their own private copy of captures,
+	// to enable updates to the set during script optimization.
+	using CaptureList = std::vector<FuncType::Capture>;
+	const std::optional<CaptureList>& GetCaptures() const { return captures; }
 
 	ValPtr Eval(Frame* f) const override;
 	TraversalCode Traverse(TraversalCallback* cb) const override;
@@ -1466,19 +1474,43 @@ public:
 
 	// Optimization-related:
 	ExprPtr Duplicate() override;
-	ExprPtr Inline(Inliner* inl) override;
 
+	const ScriptFuncPtr& PrimaryFunc() const { return primary_func; }
+
+	const FunctionIngredientsPtr& Ingredients() const { return ingredients; }
+
+	bool IsReduced(Reducer* c) const override;
+	bool HasReducedOps(Reducer* c) const override;
 	ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
+	StmtPtr ReduceToSingletons(Reducer* c) override;
 
 protected:
+	// Constructor used for script optimization.
+	LambdaExpr(LambdaExpr* orig);
+
 	void ExprDescribe(ODesc* d) const override;
 
 private:
-	bool CheckCaptures(StmtPtr when_parent);
+	friend class WhenInfo;
 
-	std::unique_ptr<FunctionIngredients> ingredients;
+	// "Private" captures are captures that correspond to "when"
+	// condition locals.  These aren't true captures in that they
+	// don't come from the outer frame when the lambda is constructed,
+	// but they otherwise behave like captures in that they persist
+	// across function invocations.
+	void SetPrivateCaptures(const IDSet& pcaps) { private_captures = pcaps; }
+
+	bool CheckCaptures(StmtPtr when_parent);
+	void BuildName();
+
+	void UpdateCaptures(Reducer* c);
+
+	FunctionIngredientsPtr ingredients;
+	ScriptFuncPtr primary_func;
 	IDPtr lambda_id;
 	IDPList outer_ids;
+	std::optional<CaptureList> captures;
+	IDSet private_captures;
 
 	std::string my_name;
 	};

src/Frame.cc

@@ -139,11 +139,8 @@ static bool val_is_func(const ValPtr& v, ScriptFunc* func)
 	return v->AsFunc() == func;
 	}
 
-broker::expected<broker::data> Frame::SerializeCopyFrame()
+broker::expected<broker::data> Frame::Serialize()
 	{
-	broker::vector rval;
-	rval.emplace_back(std::string("CopyFrame"));
-
 	broker::vector body;
 
 	for ( int i = 0; i < size; ++i )
@@ -158,28 +155,18 @@ broker::expected<broker::data> Frame::SerializeCopyFrame()
 		body.emplace_back(std::move(val_tuple));
 		}
 
+	broker::vector rval;
 	rval.emplace_back(std::move(body));
 
 	return {std::move(rval)};
 	}
 
-std::pair<bool, FramePtr> Frame::Unserialize(const broker::vector& data,
+std::pair<bool, FramePtr> Frame::Unserialize(const broker::vector& data)
-                                             const std::optional<FuncType::CaptureList>& captures)
 	{
 	if ( data.size() == 0 )
 		return std::make_pair(true, nullptr);
 
 	auto where = data.begin();
-
-	auto has_name = broker::get_if<std::string>(*where);
-	if ( ! has_name )
-		return std::make_pair(false, nullptr);
-
-	std::advance(where, 1);
-
-	if ( captures )
-		ASSERT(*has_name == "CopyFrame");
-
 	auto has_body = broker::get_if<broker::vector>(*where);
 	if ( ! has_body )
 		return std::make_pair(false, nullptr);

src/Frame.h

@@ -53,6 +53,13 @@ public:
 	 */
 	Frame(int size, const ScriptFunc* func, const zeek::Args* fn_args);
 
+	/**
+	 * Returns the size of the frame.
+	 *
+	 * @return the number of elements in the frame.
+	 */
+	int FrameSize() const { return size; }
+
 	/**
 	 * @param n the index to get.
 	 * @return the value at index *n* of the underlying array.
@@ -165,16 +172,11 @@ public:
 	Frame* CloneForTrigger() const;
 
 	/**
-	 * Serializes the frame in support of copy semantics for lambdas:
+	 * Serializes the frame (only done for lambda/when captures) as a
-	 *
+	 * sequence of two-element vectors, the first element reflecting
-	 * [ "CopyFrame", serialized_values ]
+	 * the frame value, the second its type.
-	 *
-	 * where serialized_values are two-element vectors. A serialized_value
-	 * has the result of calling broker::data_to_val on the value in the
-	 * first index, and an integer representing that value's type in the
-	 * second index.
 	 */
-	broker::expected<broker::data> SerializeCopyFrame();
+	broker::expected<broker::data> Serialize();
 
 	/**
 	 * Instantiates a Frame from a serialized one.
@@ -182,13 +184,8 @@ public:
 	 * @return a pair in which the first item is the status of the serialization;
 	 * and the second is the unserialized frame with reference count +1, or
 	 * null if the serialization wasn't successful.
-	 *
-	 * The *captures* argument, if non-nil, specifies that the frame
-	 * reflects captures with copy-semantics rather than deprecated
-	 * reference semantics.
 	 */
-	static std::pair<bool, FramePtr>
+	static std::pair<bool, FramePtr> Unserialize(const broker::vector& data);
-	Unserialize(const broker::vector& data, const std::optional<FuncType::CaptureList>& captures);
 
 	// If the frame is run in the context of a trigger condition evaluation,
 	// the trigger needs to be registered.
@@ -239,7 +236,7 @@ private:
 	 */
 	int current_offset;
 
-	/** Frame used for captures (if any) with copy semantics. */
+	/** Frame used for lambda/when captures. */
 	Frame* captures;
 
 	/** Maps IDs to offsets into the "captures" frame.  If the ID

src/Func.cc

@@ -123,6 +123,14 @@ std::string render_call_stack()
 	return rval;
 	}
 
+void Func::AddBody(const detail::FunctionIngredients& ingr, detail::StmtPtr new_body)
+	{
+	if ( ! new_body )
+		new_body = ingr.Body();
+
+	AddBody(new_body, ingr.Inits(), ingr.FrameSize(), ingr.Priority(), ingr.Groups());
+	}
+
 void Func::AddBody(detail::StmtPtr new_body, const std::vector<detail::IDPtr>& new_inits,
                    size_t new_frame_size, int priority)
 	{
@@ -130,6 +138,13 @@ void Func::AddBody(detail::StmtPtr new_body, const std::vector<detail::IDPtr>& n
 	AddBody(new_body, new_inits, new_frame_size, priority, groups);
 	}
 
+void Func::AddBody(detail::StmtPtr new_body, size_t new_frame_size)
+	{
+	std::vector<detail::IDPtr> no_inits;
+	std::set<EventGroupPtr> no_groups;
+	AddBody(new_body, no_inits, new_frame_size, 0, no_groups);
+	}
+
 void Func::AddBody(detail::StmtPtr /* new_body */,
                    const std::vector<detail::IDPtr>& /* new_inits */, size_t /* new_frame_size */,
                    int /* priority */, const std::set<EventGroupPtr>& /* groups */)
@@ -316,6 +331,15 @@ ScriptFunc::ScriptFunc(std::string _name, FuncTypePtr ft, std::vector<StmtPtr> b
 
 ScriptFunc::~ScriptFunc()
 	{
+	if ( captures_vec )
+		{
+		auto& cvec = *captures_vec;
+		auto& captures = *type->GetCaptures();
+		for ( auto i = 0u; i < captures.size(); ++i )
+			if ( captures[i].IsManaged() )
+				ZVal::DeleteManagedType(cvec[i]);
+		}
+
 	delete captures_frame;
 	delete captures_offset_mapping;
 	}
@@ -498,31 +522,77 @@ void ScriptFunc::CreateCaptures(Frame* f)
 	if ( ! captures )
 		return;
 
-	// Create a private Frame to hold the values of captured variables,
+	// Create *either* a private Frame to hold the values of captured
-	// and a mapping from those variables to their offsets in the Frame.
+	// variables, and a mapping from those variables to their offsets
-	delete captures_frame;
+	// in the Frame; *or* a ZVal frame if this script has a ZAM-compiled
-	delete captures_offset_mapping;
+	// body.
-	captures_frame = new Frame(captures->size(), this, nullptr);
+	ASSERT(bodies.size() == 1);
-	captures_offset_mapping = new OffsetMap;
+
+	if ( bodies[0].stmts->Tag() == STMT_ZAM )
+		captures_vec = std::make_unique<std::vector<ZVal>>();
+	else
+		{
+		delete captures_frame;
+		delete captures_offset_mapping;
+		captures_frame = new Frame(captures->size(), this, nullptr);
+		captures_offset_mapping = new OffsetMap;
+		}
 
 	int offset = 0;
 	for ( const auto& c : *captures )
 		{
-		auto v = f->GetElementByID(c.id);
+		auto v = f->GetElementByID(c.Id());
 
 		if ( v )
 			{
-			if ( c.deep_copy || ! v->Modifiable() )
+			if ( c.IsDeepCopy() || ! v->Modifiable() )
 				v = v->Clone();
 
-			captures_frame->SetElement(offset, std::move(v));
+			if ( captures_vec )
+				// Don't use v->GetType() here, as that might
+				// be "any" and we need to convert.
+				captures_vec->push_back(ZVal(v, c.Id()->GetType()));
+			else
+				captures_frame->SetElement(offset, std::move(v));
 			}
 
-		(*captures_offset_mapping)[c.id->Name()] = offset;
+		else if ( captures_vec )
+			captures_vec->push_back(ZVal());
+
+		if ( ! captures_vec )
+			captures_offset_mapping->insert_or_assign(c.Id()->Name(), offset);
+
 		++offset;
 		}
 	}
 
+void ScriptFunc::CreateCaptures(std::unique_ptr<std::vector<ZVal>> cvec)
+	{
+	const auto& captures = *type->GetCaptures();
+
+	ASSERT(cvec->size() == captures.size());
+	ASSERT(bodies.size() == 1 && bodies[0].stmts->Tag() == STMT_ZAM);
+
+	captures_vec = std::move(cvec);
+
+	auto n = captures.size();
+	for ( auto i = 0U; i < n; ++i )
+		{
+		auto& c_i = captures[i];
+		auto& cv_i = (*captures_vec)[i];
+
+		if ( c_i.IsDeepCopy() )
+			{
+			auto& t = c_i.Id()->GetType();
+			auto new_cv_i = cv_i.ToVal(t)->Clone();
+			if ( c_i.IsManaged() )
+				ZVal::DeleteManagedType(cv_i);
+
+			cv_i = ZVal(new_cv_i, t);
+			}
+		}
+	}
+
 void ScriptFunc::SetCaptures(Frame* f)
 	{
 	const auto& captures = type->GetCaptures();
@@ -536,7 +606,7 @@ void ScriptFunc::SetCaptures(Frame* f)
 	int offset = 0;
 	for ( const auto& c : *captures )
 		{
-		(*captures_offset_mapping)[c.id->Name()] = offset;
+		captures_offset_mapping->insert_or_assign(c.Id()->Name(), offset);
 		++offset;
 		}
 	}
@@ -595,11 +665,32 @@ void ScriptFunc::ReplaceBody(const StmtPtr& old_body, StmtPtr new_body)
 
 bool ScriptFunc::DeserializeCaptures(const broker::vector& data)
 	{
-	auto result = Frame::Unserialize(data, GetType()->GetCaptures());
+	auto result = Frame::Unserialize(data);
 
 	ASSERT(result.first);
 
-	SetCaptures(result.second.release());
+	auto& f = result.second;
+
+	if ( bodies[0].stmts->Tag() == STMT_ZAM )
+		{
+		auto& captures = *type->GetCaptures();
+		int n = f->FrameSize();
+
+		ASSERT(captures.size() == static_cast<size_t>(n));
+
+		auto cvec = std::make_unique<std::vector<ZVal>>();
+
+		for ( int i = 0; i < n; ++i )
+			{
+			auto& f_i = f->GetElement(i);
+			cvec->push_back(ZVal(f_i, captures[i].Id()->GetType()));
+			}
+
+		CreateCaptures(std::move(cvec));
+		}
+
+	else
+		SetCaptures(f.release());
 
 	return true;
 	}
@@ -608,6 +699,9 @@ FuncPtr ScriptFunc::DoClone()
 	{
 	// ScriptFunc could hold a closure. In this case a clone of it must
 	// store a copy of this closure.
+	//
+	// We don't use make_intrusive<> directly because we're accessing
+	// a protected constructor.
 	auto other = IntrusivePtr{AdoptRef{}, new ScriptFunc()};
 
 	CopyStateInto(other.get());
@@ -622,13 +716,43 @@ FuncPtr ScriptFunc::DoClone()
 		*other->captures_offset_mapping = *captures_offset_mapping;
 		}
 
+	if ( captures_vec )
+		{
+		auto cv_i = captures_vec->begin();
+		other->captures_vec = std::make_unique<std::vector<ZVal>>();
+		for ( auto& c : *type->GetCaptures() )
+			{
+			// Need to clone cv_i.
+			auto& t_i = c.Id()->GetType();
+			auto cv_i_val = cv_i->ToVal(t_i)->Clone();
+			other->captures_vec->push_back(ZVal(cv_i_val, t_i));
+			++cv_i;
+			}
+		}
+
 	return other;
 	}
 
 broker::expected<broker::data> ScriptFunc::SerializeCaptures() const
 	{
+	if ( captures_vec )
+		{
+		auto& cv = *captures_vec;
+		auto& captures = *type->GetCaptures();
+		int n = captures_vec->size();
+		auto temp_frame = make_intrusive<Frame>(n, this, nullptr);
+
+		for ( int i = 0; i < n; ++i )
+			{
+			auto c_i = cv[i].ToVal(captures[i].Id()->GetType());
+			temp_frame->SetElement(i, c_i);
+			}
+
+		return temp_frame->Serialize();
+		}
+
 	if ( captures_frame )
-		return captures_frame->SerializeCopyFrame();
+		return captures_frame->Serialize();
 
 	// No captures, return an empty vector.
 	return broker::vector{};

src/Func.h

@@ -44,6 +44,7 @@ using IDPtr = IntrusivePtr<ID>;
 using StmtPtr = IntrusivePtr<Stmt>;
 
 class ScriptFunc;
+class FunctionIngredients;
 
 	} // namespace detail
 
@@ -114,14 +115,18 @@ public:
 		return Invoke(&zargs);
 		}
 
-	// Add a new event handler to an existing function (event).
+	// Various ways to add a new event handler to an existing function
+	// (event).  The usual version to use is the first with its default
+	// parameter.  All of the others are for use by script optimization,
+	// as is a non-default second parameter to the first method, which
+	// overrides the function body in "ingr".
+	void AddBody(const detail::FunctionIngredients& ingr, detail::StmtPtr new_body = nullptr);
 	virtual void AddBody(detail::StmtPtr new_body, const std::vector<detail::IDPtr>& new_inits,
 	                     size_t new_frame_size, int priority,
 	                     const std::set<EventGroupPtr>& groups);
-
+	void AddBody(detail::StmtPtr new_body, const std::vector<detail::IDPtr>& new_inits,
-	// Add a new event handler to an existing function (event).
+	             size_t new_frame_size, int priority = 0);
-	virtual void AddBody(detail::StmtPtr new_body, const std::vector<detail::IDPtr>& new_inits,
+	void AddBody(detail::StmtPtr new_body, size_t new_frame_size);
-	                     size_t new_frame_size, int priority = 0);
 
 	virtual void SetScope(detail::ScopePtr newscope);
 	virtual detail::ScopePtr GetScope() const { return scope; }
@@ -190,6 +195,17 @@ public:
 	 */
 	void CreateCaptures(Frame* f);
 
+	/**
+	 * Uses the given set of ZVal's for captures.  Note that this is
+	 * different from the method above, which uses its argument to
+	 * compute the captures, rather than here where they are pre-computed.
+	 *
+	 * Makes deep copies if required.
+	 *
+	 * @param cvec  a vector of ZVal's corresponding to the captures.
+	 */
+	void CreateCaptures(std::unique_ptr<std::vector<ZVal>> cvec);
+
 	/**
 	 * Returns the frame associated with this function for tracking
 	 * captures, or nil if there isn't one.
@@ -198,6 +214,18 @@ public:
 	 */
 	Frame* GetCapturesFrame() const { return captures_frame; }
 
+	/**
+	 * Returns the set of ZVal's used for captures.  It's okay to modify
+	 * these as long as memory-management is done for managed entries.
+	 *
+	 * @return internal vector of ZVal's kept for persisting captures
+	 */
+	auto& GetCapturesVec() const
+		{
+		ASSERT(captures_vec);
+		return *captures_vec;
+		}
+
 	// Same definition as in Frame.h.
 	using OffsetMap = std::unordered_map<std::string, int>;
 
@@ -273,6 +301,7 @@ protected:
 	/**
 	 * Uses the given frame for captures, and generates the
 	 * mapping from captured variables to offsets in the frame.
+	 * Virtual so it can be modified for script optimization uses.
 	 *
 	 * @param f  the frame holding the values of capture variables
 	 */
@@ -291,6 +320,9 @@ private:
 
 	OffsetMap* captures_offset_mapping = nullptr;
 
+	// Captures when using ZVal block instead of a Frame.
+	std::unique_ptr<std::vector<ZVal>> captures_vec;
+
 	// The most recently added/updated body ...
 	StmtPtr current_body;
 
@@ -332,8 +364,7 @@ struct CallInfo
 	const zeek::Args& args;
 	};
 
-// Struct that collects all the specifics defining a Func. Used for ScriptFuncs
+// Class that collects all the specifics defining a Func.
-// with closures.
 class FunctionIngredients
 	{
 public:
@@ -344,14 +375,19 @@ public:
 	const IDPtr& GetID() const { return id; }
 
 	const StmtPtr& Body() const { return body; }
-	void SetBody(StmtPtr _body) { body = std::move(_body); }
 
 	const auto& Inits() const { return inits; }
+	void ClearInits() { inits.clear(); }
+
 	size_t FrameSize() const { return frame_size; }
 	int Priority() const { return priority; }
 	const ScopePtr& Scope() const { return scope; }
 	const auto& Groups() const { return groups; }
 
+	// Used by script optimization to update lambda ingredients
+	// after compilation.
+	void SetFrameSize(size_t _frame_size) { frame_size = _frame_size; }
+
 private:
 	IDPtr id;
 	StmtPtr body;
@@ -362,6 +398,8 @@ private:
 	std::set<EventGroupPtr> groups;
 	};
 
+using FunctionIngredientsPtr = std::shared_ptr<FunctionIngredients>;
+
 extern std::vector<CallInfo> call_stack;
 
 /**

src/Stmt.cc

@@ -52,7 +52,6 @@ const char* stmt_name(StmtTag t)
 		"check-any-length",
 		"compiled-C++",
 		"ZAM",
-		"ZAM-resumption",
 		"null",
 		"assert",
 	};
@@ -2005,35 +2004,7 @@ WhenInfo::WhenInfo(ExprPtr arg_cond, FuncType::CaptureList* arg_cl, bool arg_is_
 	if ( ! cl )
 		cl = new zeek::FuncType::CaptureList;
 
-	ProfileFunc cond_pf(cond.get());
+	BuildProfile();
-
-	when_expr_locals = cond_pf.Locals();
-	when_expr_globals = cond_pf.AllGlobals();
-	when_new_locals = cond_pf.WhenLocals();
-
-	// Make any when-locals part of our captures, if not already present,
-	// to enable sharing between the condition and the body/timeout code.
-	for ( auto& wl : when_new_locals )
-		{
-		bool is_present = false;
-
-		for ( auto& c : *cl )
-			if ( c.id == wl )
-				{
-				is_present = true;
-				break;
-				}
-
-		if ( ! is_present )
-			{
-			IDPtr wl_ptr = {NewRef{}, const_cast<ID*>(wl)};
-			cl->emplace_back(FuncType::Capture{wl_ptr, false});
-			}
-
-		// In addition, don't treat them as external locals that
-		// existed at the onset.
-		when_expr_locals.erase(wl);
-		}
 
 	// Create the internal lambda we'll use to manage the captures.
 	static int num_params = 0; // to ensure each is distinct
@@ -2049,12 +2020,39 @@ WhenInfo::WhenInfo(ExprPtr arg_cond, FuncType::CaptureList* arg_cl, bool arg_is_
 	if ( ! is_return )
 		lambda_ft->SetExpressionlessReturnOkay(true);
 
+	lambda_ft->SetCaptures(*cl);
+
 	auto id = current_scope()->GenerateTemporary("when-internal");
 	id->SetType(lambda_ft);
 	push_scope(std::move(id), nullptr);
 
-	auto arg_id = install_ID(lambda_param_id.c_str(), current_module.c_str(), false, false);
+	param_id = install_ID(lambda_param_id.c_str(), current_module.c_str(), false, false);
-	arg_id->SetType(count_t);
+	param_id->SetType(count_t);
+	}
+
+WhenInfo::WhenInfo(const WhenInfo* orig)
+	{
+	if ( orig->cl )
+		{
+		cl = new FuncType::CaptureList;
+		*cl = *orig->cl;
+		}
+
+	cond = orig->OrigCond()->Duplicate();
+
+	// We don't duplicate these, as they'll be compiled separately.
+	s = orig->OrigBody();
+	timeout_s = orig->OrigBody();
+
+	timeout = orig->OrigTimeout();
+	if ( timeout )
+		timeout = timeout->Duplicate();
+
+	lambda = cast_intrusive<LambdaExpr>(orig->Lambda()->Duplicate());
+
+	is_return = orig->IsReturn();
+
+	BuildProfile();
 	}
 
 WhenInfo::WhenInfo(bool arg_is_return) : is_return(arg_is_return)
@@ -2063,10 +2061,49 @@ WhenInfo::WhenInfo(bool arg_is_return) : is_return(arg_is_return)
 	BuildInvokeElems();
 	}
 
+void WhenInfo::BuildProfile()
+	{
+	ProfileFunc cond_pf(cond.get());
+
+	auto when_expr_locals_set = cond_pf.Locals();
+	when_expr_globals = cond_pf.AllGlobals();
+	when_new_locals = cond_pf.WhenLocals();
+
+	// Make any when-locals part of our captures, if not already present,
+	// to enable sharing between the condition and the body/timeout code.
+	for ( auto& wl : when_new_locals )
+		{
+		bool is_present = false;
+
+		for ( auto& c : *cl )
+			if ( c.Id() == wl )
+				{
+				is_present = true;
+				break;
+				}
+
+		if ( ! is_present )
+			{
+			IDPtr wl_ptr = {NewRef{}, const_cast<ID*>(wl)};
+			cl->emplace_back(FuncType::Capture{wl_ptr, false});
+			}
+
+		// In addition, don't treat them as external locals that
+		// existed at the onset.
+		when_expr_locals_set.erase(wl);
+		}
+
+	for ( auto& w : when_expr_locals_set )
+		{
+		// We need IDPtr versions of the locals so we can manipulate
+		// them during script optimization.
+		auto non_const_w = const_cast<ID*>(w);
+		when_expr_locals.push_back({NewRef{}, non_const_w});
+		}
+	}
+
 void WhenInfo::Build(StmtPtr ws)
 	{
-	lambda_ft->SetCaptures(*cl);
-
 	// Our general strategy is to construct a single lambda (so that
 	// the values of captures are shared across all of its elements)
 	// that's used for all three of the "when" components: condition,
@@ -2086,10 +2123,13 @@ void WhenInfo::Build(StmtPtr ws)
 	// First, the constants we'll need.
 	BuildInvokeElems();
 
+	if ( lambda )
+		// No need to build the lambda.
+		return;
+
 	auto true_const = make_intrusive<ConstExpr>(val_mgr->True());
 
 	// Access to the parameter that selects which action we're doing.
-	auto param_id = lookup_ID(lambda_param_id.c_str(), current_module.c_str());
 	ASSERT(param_id);
 	auto param = make_intrusive<NameExpr>(param_id);
 
@@ -2109,11 +2149,14 @@ void WhenInfo::Build(StmtPtr ws)
 
 	auto shebang = make_intrusive<StmtList>(do_test, do_bodies, dummy_return);
 
-	auto ingredients = std::make_unique<FunctionIngredients>(current_scope(), shebang,
+	auto ingredients = std::make_shared<FunctionIngredients>(current_scope(), shebang,
 	                                                         current_module);
 	auto outer_ids = gather_outer_ids(pop_scope(), ingredients->Body());
 
-	lambda = make_intrusive<LambdaExpr>(std::move(ingredients), std::move(outer_ids), ws);
+	lambda = make_intrusive<LambdaExpr>(std::move(ingredients), std::move(outer_ids), "", ws);
+	lambda->SetPrivateCaptures(when_new_locals);
+
+	analyze_when_lambda(lambda.get());
 	}
 
 void WhenInfo::Instantiate(Frame* f)
@@ -2205,8 +2248,7 @@ ValPtr WhenStmt::Exec(Frame* f, StmtFlowType& flow)
 	std::vector<ValPtr> local_aggrs;
 	for ( auto& l : wi->WhenExprLocals() )
 		{
-		IDPtr l_ptr = {NewRef{}, const_cast<ID*>(l)};
+		auto v = f->GetElementByID(l);
-		auto v = f->GetElementByID(l_ptr);
 		if ( v && v->Modifiable() )
 			local_aggrs.emplace_back(std::move(v));
 		}
@@ -2226,6 +2268,23 @@ void WhenStmt::StmtDescribe(ODesc* d) const
 	{
 	Stmt::StmtDescribe(d);
 
+	auto cl = wi->Captures();
+	if ( d->IsReadable() && ! cl->empty() )
+		{
+		d->Add("[");
+		for ( auto& c : *cl )
+			{
+			if ( &c != &(*cl)[0] )
+				d->AddSP(",");
+
+			if ( c.IsDeepCopy() )
+				d->Add("copy ");
+
+			d->Add(c.Id()->Name());
+			}
+		d->Add("]");
+		}
+
 	if ( d->IsReadable() )
 		d->Add("(");
 
@@ -2267,32 +2326,13 @@ TraversalCode WhenStmt::Traverse(TraversalCallback* cb) const
 	TraversalCode tc = cb->PreStmt(this);
 	HANDLE_TC_STMT_PRE(tc);
 
-	auto wl = wi->Lambda();
+	tc = wi->Lambda()->Traverse(cb);
+	HANDLE_TC_STMT_PRE(tc);
 
-	if ( wl )
+	auto e = wi->TimeoutExpr();
+	if ( e )
 		{
-		tc = wl->Traverse(cb);
+		tc = e->Traverse(cb);
-		HANDLE_TC_STMT_PRE(tc);
-		}
-
-	else
-		{
-		tc = wi->OrigCond()->Traverse(cb);
-		HANDLE_TC_STMT_PRE(tc);
-
-		tc = wi->OrigBody()->Traverse(cb);
-		HANDLE_TC_STMT_PRE(tc);
-
-		if ( wi->OrigTimeoutStmt() )
-			{
-			tc = wi->OrigTimeoutStmt()->Traverse(cb);
-			HANDLE_TC_STMT_PRE(tc);
-			}
-		}
-
-	if ( wi->OrigTimeout() )
-		{
-		tc = wi->OrigTimeout()->Traverse(cb);
 		HANDLE_TC_STMT_PRE(tc);
 		}
 

src/Stmt.h

@@ -450,6 +450,7 @@ public:
 	ReturnStmt(ExprPtr e, bool ignored);
 
 	// Optimization-related:
+	bool IsReduced(Reducer* c) const override;
 	StmtPtr DoReduce(Reducer* c) override;
 
 	bool NoFlowAfter(bool ignore_break) const override { return true; }
@@ -575,6 +576,9 @@ public:
 	// Takes ownership of the CaptureList.
 	WhenInfo(ExprPtr cond, FuncType::CaptureList* cl, bool is_return);
 
+	// Used for duplication to support inlining.
+	WhenInfo(const WhenInfo* orig);
+
 	// Constructor used by script optimization to create a stub.
 	WhenInfo(bool is_return);
 
@@ -615,6 +619,7 @@ public:
 	StmtPtr TimeoutStmt();
 
 	ExprPtr TimeoutExpr() const { return timeout; }
+	void SetTimeoutExpr(ExprPtr e) { timeout = std::move(e); }
 	double TimeoutVal(Frame* f);
 
 	FuncType::CaptureList* Captures() { return cl; }
@@ -624,10 +629,22 @@ public:
 	// The locals and globals used in the conditional expression
 	// (other than newly introduced locals), necessary for registering
 	// the associated triggers for when their values change.
-	const IDSet& WhenExprLocals() const { return when_expr_locals; }
+	const auto& WhenExprLocals() const { return when_expr_locals; }
-	const IDSet& WhenExprGlobals() const { return when_expr_globals; }
+	const auto& WhenExprGlobals() const { return when_expr_globals; }
+
+	// The locals introduced in the conditional expression.
+	const auto& WhenNewLocals() const { return when_new_locals; }
+
+	// Used for script optimization when in-lining needs to revise
+	// identifiers.
+	bool HasUnreducedIDs(Reducer* c) const;
+	void UpdateIDs(Reducer* c);
 
 private:
+	// Profile the original AST elements to extract things like
+	// globals and locals used.
+	void BuildProfile();
+
 	// Build those elements we'll need for invoking our lambda.
 	void BuildInvokeElems();
 
@@ -639,8 +656,10 @@ private:
 
 	bool is_return = false;
 
-	// The name of parameter passed to the lambda.
+	// The name of parameter passed to the lambda, and the corresponding
+	// identifier.
 	std::string lambda_param_id;
+	IDPtr param_id;
 
 	// The expression for constructing the lambda, and its type.
 	LambdaExprPtr lambda;
@@ -661,7 +680,7 @@ private:
 	ConstExprPtr two_const;
 	ConstExprPtr three_const;
 
-	IDSet when_expr_locals;
+	std::vector<IDPtr> when_expr_locals;
 	IDSet when_expr_globals;
 
 	// Locals introduced via "local" in the "when" clause itself.
@@ -684,7 +703,7 @@ public:
 	StmtPtr TimeoutBody() const { return wi->TimeoutStmt(); }
 	bool IsReturn() const { return wi->IsReturn(); }
 
-	const WhenInfo* Info() const { return wi; }
+	WhenInfo* Info() const { return wi; }
 
 	void StmtDescribe(ODesc* d) const override;
 
@@ -692,9 +711,9 @@ public:
 
 	// Optimization-related:
 	StmtPtr Duplicate() override;
-	void Inline(Inliner* inl) override;
 
 	bool IsReduced(Reducer* c) const override;
+	StmtPtr DoReduce(Reducer* c) override;
 
 private:
 	WhenInfo* wi;

src/StmtEnums.h

@@ -31,7 +31,6 @@ enum StmtTag
 	STMT_CHECK_ANY_LEN, // internal reduced statement
 	STMT_CPP, // compiled C++
 	STMT_ZAM, // a ZAM function body
-	STMT_ZAM_RESUMPTION, // resumes ZAM execution for "when" statements
 	STMT_NULL,
 	STMT_ASSERT,
 #define NUM_STMTS (int(STMT_ASSERT) + 1)

src/Trigger.h

@@ -81,6 +81,8 @@ public:
 	// if the Val is null or it's disabled.  The cache is managed using
 	// void*'s so that the value can be associated with either a CallExpr
 	// (for interpreted execution) or a C++ function (for compiled-to-C++).
+	//
+	// Lookup() returned value must be Ref()'d if you want to hang onto it.
 	bool Cache(const void* obj, Val* val);
 	Val* Lookup(const void* obj);
 

src/Type.cc

@@ -699,6 +699,15 @@ TypePtr SetType::ShallowClone()
 
 SetType::~SetType() = default;
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+FuncType::Capture::Capture(detail::IDPtr _id, bool _deep_copy)
+	: id(std::move(_id)), deep_copy(_deep_copy)
+	{
+	is_managed = id ? ZVal::IsManagedType(id->GetType()) : false;
+	}
+#pragma GCC diagnostic pop
+
 FuncType::FuncType(RecordTypePtr arg_args, TypePtr arg_yield, FunctionFlavor arg_flavor)
 	: Type(TYPE_FUNC), args(std::move(arg_args)), arg_types(make_intrusive<TypeList>()),
 	  yield(std::move(arg_yield))
@@ -788,9 +797,12 @@ bool FuncType::CheckArgs(const std::vector<TypePtr>& args, bool is_init, bool do
 	if ( my_args.size() != args.size() )
 		{
 		if ( do_warn )
+			{
 			Warn(util::fmt("Wrong number of arguments for function. Expected %zu, got %zu.",
 			               args.size(), my_args.size()));
-		const_cast<FuncType*>(this)->reported_error = true;
+			const_cast<FuncType*>(this)->reported_error = true;
+			}
+
 		return false;
 		}
 

src/Type.h

@@ -511,10 +511,36 @@ public:
 	/**
 	 * A single lambda "capture" (outer variable used in a lambda's body).
 	 */
-	struct Capture
+	class Capture
 		{
-		detail::IDPtr id;
+	public:
-		bool deep_copy;
+		Capture(detail::IDPtr _id, bool _deep_copy);
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+		Capture(const Capture&) = default;
+		Capture(Capture&&) = default;
+		Capture& operator=(const Capture&) = default;
+		Capture& operator=(Capture&&) = default;
+		~Capture() = default;
+
+		auto& Id() const { return id; }
+		bool IsDeepCopy() const { return deep_copy; }
+		bool IsManaged() const { return is_managed; }
+
+		// For script optimization:
+		void SetID(detail::IDPtr new_id) { id = std::move(new_id); }
+#pragma GCC diagnostic pop
+
+		[[deprecated(
+			"Remove in v7.1.  Use non-default constructor and associated accessors.")]] detail::
+			IDPtr id;
+		[[deprecated(
+			"Remove in v7.1.  Use non-default constructor and associated accessors.")]] bool
+			deep_copy;
+		[[deprecated(
+			"Remove in v7.1.  Use non-default constructor and associated accessors.")]] bool
+			is_managed;
 		};
 
 	using CaptureList = std::vector<Capture>;

src/Val.cc

@@ -2787,6 +2787,11 @@ void TableVal::InitDefaultFunc(detail::Frame* f)
 	def_val = def_attr->GetExpr()->Eval(f);
 	}
 
+void TableVal::InitDefaultVal(ValPtr _def_val)
+	{
+	def_val = std::move(_def_val);
+	}
+
 void TableVal::InitTimer(double delay)
 	{
 	timer = new TableValTimer(this, run_state::network_time + delay);

src/Val.h

@@ -968,9 +968,13 @@ public:
 
 	// If the &default attribute is not a function, or the function has
 	// already been initialized, this does nothing. Otherwise, evaluates
-	// the function in the frame allowing it to capture its closure.
+	// the function in the frame, allowing it to capture its closure.
 	void InitDefaultFunc(detail::Frame* f);
 
+	// An alternative that assigns the default value directly.  Used
+	// by ZAM compilation.
+	void InitDefaultVal(ValPtr def_val);
+
 	void ClearTimer(detail::Timer* t)
 		{
 		if ( timer == t )

src/Var.cc

@@ -768,11 +768,10 @@ TraversalCode OuterIDBindingFinder::PreStmt(const Stmt* stmt)
 	if ( stmt->Tag() != STMT_WHEN )
 		return TC_CONTINUE;
 
-	// The semantics of identifiers for the "when" statement are those
-	// of the lambda it's transformed into.
-
 	auto ws = static_cast<const WhenStmt*>(stmt);
-	ws->Info()->Lambda()->Traverse(this);
+
+	for ( auto& cl : ws->Info()->WhenExprLocals() )
+		outer_id_references.insert(const_cast<ID*>(cl.get()));
 
 	return TC_ABORTSTMT;
 	}
@@ -789,18 +788,19 @@ TraversalCode OuterIDBindingFinder::PreExpr(const Expr* expr)
 	if ( expr->Tag() != EXPR_NAME )
 		return TC_CONTINUE;
 
-	auto* e = static_cast<const NameExpr*>(expr);
+	auto e = static_cast<const NameExpr*>(expr);
+	auto id = e->Id();
 
-	if ( e->Id()->IsGlobal() )
+	if ( id->IsGlobal() )
 		return TC_CONTINUE;
 
 	for ( const auto& scope : scopes )
-		if ( scope->Find(e->Id()->Name()) )
+		if ( scope->Find(id->Name()) )
 			// Shadowing is not allowed, so if it's found at inner scope, it's
 			// not something we have to worry about also being at outer scope.
 			return TC_CONTINUE;
 
-	outer_id_references.insert(e->Id());
+	outer_id_references.insert(id);
 	return TC_CONTINUE;
 	}
 
@@ -845,9 +845,7 @@ void end_func(StmtPtr body, const char* module_name, bool free_of_conditionals)
 		id->SetConst();
 		}
 
-	id->GetVal()->AsFunc()->AddBody(ingredients->Body(), ingredients->Inits(),
+	id->GetVal()->AsFunc()->AddBody(*ingredients);
-	                                ingredients->FrameSize(), ingredients->Priority(),
-	                                ingredients->Groups());
 
 	script_coverage_mgr.AddFunction(id, ingredients->Body());
 

src/Var.h

@@ -38,7 +38,7 @@ extern void add_global(const IDPtr& id, TypePtr t, InitClass c, ExprPtr init,
 extern StmtPtr add_local(IDPtr id, TypePtr t, InitClass c, ExprPtr init,
                          std::unique_ptr<std::vector<AttrPtr>> attr, DeclType dt);
 
-extern ExprPtr add_and_assign_local(IDPtr id, ExprPtr init, ValPtr val = nullptr);
+extern ExprPtr add_and_assign_local(IDPtr id, ExprPtr init, ValPtr val);
 
 extern void add_type(ID* id, TypePtr t, std::unique_ptr<std::vector<AttrPtr>> attr);
 

src/parse.y

@@ -1566,7 +1566,7 @@ lambda_body:
 
 			// Gather the ingredients for a Func from the
 			// current scope.
-			auto ingredients = std::make_unique<FunctionIngredients>(
+			auto ingredients = std::make_shared<FunctionIngredients>(
 				current_scope(), IntrusivePtr{AdoptRef{}, $3}, current_module.c_str());
 			auto outer_ids = gather_outer_ids(pop_scope(), ingredients->Body());
 
@@ -1640,9 +1640,7 @@ capture:
 
 			delete [] $2;
 
-			$$ = new FuncType::Capture;
+			$$ = new FuncType::Capture(id, $1);
-			$$->id = id;
-			$$->deep_copy = $1;
 			}
 	;
 

src/script_opt/CPP/DeclFunc.cc

@@ -29,8 +29,8 @@ void CPPCompile::DeclareLambda(const LambdaExpr* l, const ProfileFunc* pf)
 	ASSERT(is_CPP_compilable(pf));
 
 	auto lname = Canonicalize(l->Name().c_str()) + "_lb";
-	auto body = l->Ingredients().Body();
+	auto body = l->Ingredients()->Body();
-	auto l_id = l->Ingredients().GetID();
+	auto l_id = l->Ingredients()->GetID();
 	auto& ids = l->OuterIDs();
 
 	for ( auto id : ids )

src/script_opt/CPP/Driver.cc

@@ -77,6 +77,12 @@ void CPPCompile::Compile(bool report_uncompilable)
 			continue;
 			}
 
+		if ( is_when_lambda(f) )
+			{
+			func.SetSkip(true);
+			continue;
+			}
+
 		const char* reason;
 		if ( IsCompilable(func, &reason) )
 			{
@@ -150,7 +156,7 @@ void CPPCompile::Compile(bool report_uncompilable)
 	for ( const auto& l : pfs.Lambdas() )
 		{
 		const auto& n = l->Name();
-		const auto body = l->Ingredients().Body().get();
+		const auto body = l->Ingredients()->Body().get();
 		if ( lambda_ASTs.count(n) > 0 )
 			// Reuse previous body.
 			body_names[body] = body_names[lambda_ASTs[n]];
@@ -176,7 +182,7 @@ void CPPCompile::Compile(bool report_uncompilable)
 			continue;
 
 		CompileLambda(l, pfs.ExprProf(l).get());
-		lambda_ASTs[n] = l->Ingredients().Body().get();
+		lambda_ASTs[n] = l->Ingredients()->Body().get();
 		}
 
 	NL();

src/script_opt/CPP/Exprs.cc

@@ -1301,7 +1301,7 @@ string CPPCompile::GenLambdaClone(const LambdaExpr* l, bool all_deep)
 		if ( captures && ! IsNativeType(id_t) )
 			{
 			for ( const auto& c : *captures )
-				if ( id == c.id && (c.deep_copy || all_deep) )
+				if ( id == c.Id() && (c.IsDeepCopy() || all_deep) )
 					arg = string("cast_intrusive<") + TypeName(id_t) + ">(" + arg + "->Clone())";
 			}
 

src/script_opt/CPP/GenFunc.cc

@@ -23,8 +23,8 @@ void CPPCompile::CompileFunc(const FuncInfo& func)
 void CPPCompile::CompileLambda(const LambdaExpr* l, const ProfileFunc* pf)
 	{
 	auto lname = Canonicalize(l->Name().c_str()) + "_lb";
-	auto body = l->Ingredients().Body();
+	auto body = l->Ingredients()->Body();
-	auto l_id = l->Ingredients().GetID();
+	auto l_id = l->Ingredients()->GetID();
 	auto& ids = l->OuterIDs();
 
 	DefineBody(l_id->GetType<FuncType>(), pf, lname, body, &ids, FUNC_FLAVOR_FUNCTION);

src/script_opt/CPP/RuntimeOps.cc

@@ -105,7 +105,7 @@ ValPtr when_invoke__CPP(Func* f, std::vector<ValPtr> args, Frame* frame, void* c
 
 	auto res = f->Invoke(&args, frame);
 	if ( ! res )
-		throw DelayedCallException();
+		throw CPPDelayedCallException();
 
 	return res;
 	}

src/script_opt/CPP/RuntimeOps.h

@@ -80,7 +80,7 @@ inline ValPtr invoke__CPP(Func* f, std::vector<ValPtr> args, Frame* frame)
 extern ValPtr when_invoke__CPP(Func* f, std::vector<ValPtr> args, Frame* frame, void* caller_addr);
 
 // Thrown when a call inside a "when" delays.
-class DelayedCallException : public InterpreterException
+class CPPDelayedCallException : public InterpreterException
 	{
 	};
 

src/script_opt/CPP/Stmts.cc

@@ -420,9 +420,9 @@ void CPPCompile::GenWhenStmt(const WhenStmt* w)
 	NL();
 
 	Emit("std::vector<ValPtr> CPP__local_aggrs;");
-	for ( auto l : wi->WhenExprLocals() )
+	for ( auto& l : wi->WhenExprLocals() )
 		if ( IsAggr(l->GetType()) )
-			Emit("CPP__local_aggrs.emplace_back(%s);", IDNameStr(l));
+			Emit("CPP__local_aggrs.emplace_back(%s);", IDNameStr(l.get()));
 
 	Emit("CPP__wi->Instantiate(%s);", GenExpr(wi->Lambda(), GEN_NATIVE));
 
@@ -446,7 +446,7 @@ void CPPCompile::GenWhenStmt(const WhenStmt* w)
 		{
 		Emit("ValPtr retval = {NewRef{}, curr_t->Lookup(curr_assoc)};");
 		Emit("if ( ! retval )");
-		Emit("\tthrow DelayedCallException();");
+		Emit("\tthrow CPPDelayedCallException();");
 		Emit("return %s;", GenericValPtrToGT("retval", ret_type, GEN_NATIVE));
 		}
 

src/script_opt/Expr.cc

@@ -1851,7 +1851,12 @@ ExprPtr AssignExpr::ReduceToSingleton(Reducer* c, StmtPtr& red_stmt)
 	if ( val )
 		return make_intrusive<ConstExpr>(val);
 
-	return op1->AsRefExprPtr()->GetOp1();
+	auto lhs = op1->AsRefExprPtr()->GetOp1();
+	StmtPtr lhs_stmt;
+	auto new_op1 = lhs->ReduceToSingleton(c, lhs_stmt);
+	red_stmt = MergeStmts(red_stmt, lhs_stmt);
+
+	return new_op1;
 	}
 
 ExprPtr IndexSliceAssignExpr::Duplicate()
@@ -2337,7 +2342,7 @@ ExprPtr CallExpr::Duplicate()
 	auto func_type = func->GetType();
 	auto in_hook = func_type->AsFuncType()->Flavor() == FUNC_FLAVOR_HOOK;
 
-	return SetSucc(new CallExpr(func_d, args_d, in_hook));
+	return SetSucc(new CallExpr(func_d, args_d, in_hook, in_when));
 	}
 
 ExprPtr CallExpr::Inline(Inliner* inl)
@@ -2368,7 +2373,14 @@ bool CallExpr::HasReducedOps(Reducer* c) const
 	if ( ! func->IsSingleton(c) )
 		return NonReduced(this);
 
-	return args->HasReducedOps(c);
+	// We don't use args->HasReducedOps() here because for ListExpr's
+	// the method has some special-casing that isn't germane for calls.
+
+	for ( const auto& expr : args->Exprs() )
+		if ( ! expr->IsSingleton(c) )
+			return false;
+
+	return true;
 	}
 
 ExprPtr CallExpr::Reduce(Reducer* c, StmtPtr& red_stmt)
@@ -2386,9 +2398,7 @@ ExprPtr CallExpr::Reduce(Reducer* c, StmtPtr& red_stmt)
 	if ( ! func->IsSingleton(c) )
 		func = func->ReduceToSingleton(c, red_stmt);
 
-	StmtPtr red2_stmt;
+	StmtPtr red2_stmt = args->ReduceToSingletons(c);
-	// We assume that ListExpr won't transform itself fundamentally.
-	(void)args->Reduce(c, red2_stmt);
 
 	// ### could check here for (1) pure function, and (2) all
 	// arguments constants, and call it to fold right now.
@@ -2415,23 +2425,60 @@ StmtPtr CallExpr::ReduceToSingletons(Reducer* c)
 
 ExprPtr LambdaExpr::Duplicate()
 	{
-	auto ingr = std::make_unique<FunctionIngredients>(*ingredients);
+	return SetSucc(new LambdaExpr(this));
-	ingr->SetBody(ingr->Body()->Duplicate());
-	return SetSucc(new LambdaExpr(std::move(ingr), outer_ids));
 	}
 
-ExprPtr LambdaExpr::Inline(Inliner* inl)
+bool LambdaExpr::IsReduced(Reducer* c) const
 	{
-	// Don't inline these, we currently don't get the closure right.
+	if ( ! captures )
-	return ThisPtr();
+		return true;
+
+	for ( auto& cp : *captures )
+		{
+		auto& cid = cp.Id();
+
+		if ( private_captures.count(cid.get()) == 0 && ! c->ID_IsReduced(cid) )
+			return NonReduced(this);
+		}
+
+	return true;
+	}
+
+bool LambdaExpr::HasReducedOps(Reducer* c) const
+	{
+	return IsReduced(c);
 	}
 
 ExprPtr LambdaExpr::Reduce(Reducer* c, StmtPtr& red_stmt)
 	{
 	if ( c->Optimizing() )
 		return ThisPtr();
-	else
+
-		return AssignToTemporary(c, red_stmt);
+	UpdateCaptures(c);
+
+	return AssignToTemporary(c, red_stmt);
+	}
+
+StmtPtr LambdaExpr::ReduceToSingletons(Reducer* c)
+	{
+	UpdateCaptures(c);
+	return nullptr;
+	}
+
+void LambdaExpr::UpdateCaptures(Reducer* c)
+	{
+	if ( captures )
+		{
+		for ( auto& cp : *captures )
+			{
+			auto& cid = cp.Id();
+
+			if ( private_captures.count(cid.get()) == 0 )
+				cp.SetID(c->UpdateID(cid));
+			}
+
+		c->UpdateIDs(&outer_ids);
+		}
 	}
 
 ExprPtr EventExpr::Duplicate()
@@ -2558,11 +2605,14 @@ StmtPtr ListExpr::ReduceToSingletons(Reducer* c)
 
 	loop_over_list(exprs, i)
 		{
-		if ( exprs[i]->IsSingleton(c) )
+		auto& e_i = exprs[i];
+
+		if ( e_i->IsSingleton(c) )
 			continue;
 
 		StmtPtr e_stmt;
-		auto old = exprs.replace(i, exprs[i]->Reduce(c, e_stmt).release());
+		auto new_e_i = e_i->ReduceToSingleton(c, e_stmt);
+		auto old = exprs.replace(i, new_e_i.release());
 		Unref(old);
 
 		if ( e_stmt )
@@ -2696,6 +2746,13 @@ void InlineExpr::ExprDescribe(ODesc* d) const
 		{
 		d->Add("inline(");
 		args->Describe(d);
+		d->Add(")(");
+		for ( auto& p : params )
+			{
+			if ( &p != &params[0] )
+				d->AddSP(",");
+			d->Add(p->Name());
+			}
 		d->Add("){");
 		body->Describe(d);
 		d->Add("}");

src/script_opt/GenIDDefs.cc

@@ -196,12 +196,6 @@ TraversalCode GenIDDefs::PreStmt(const Stmt* s)
 			return TC_ABORTSTMT;
 			}
 
-		case STMT_WHEN:
-			{
-			// ### punt on these for now, need to reflect on bindings.
-			return TC_ABORTSTMT;
-			}
-
 		default:
 			return TC_CONTINUE;
 		}

src/script_opt/Inline.cc

@@ -115,16 +115,21 @@ void Inliner::Analyze()
 		const auto& func = func_ptr.get();
 		const auto& body = f.Body();
 
+		if ( ! should_analyze(func_ptr, body) )
+			continue;
+
 		// Candidates are non-event, non-hook, non-recursive,
-		// non-compiled functions ... that don't use lambdas or when's,
+		// non-compiled functions ...
-		// since we don't currently compute the closures/frame
+		if ( func->Flavor() != FUNC_FLAVOR_FUNCTION )
-		// sizes for them correctly, and more fundamentally since
+			continue;
-		// we don't compile them and hence inlining them will
+
-		// make the parent non-compilable.
+		if ( non_recursive_funcs.count(func) == 0 )
-		if ( should_analyze(func_ptr, body) && func->Flavor() == FUNC_FLAVOR_FUNCTION &&
+			continue;
-		     non_recursive_funcs.count(func) > 0 && f.Profile()->NumLambdas() == 0 &&
+
-		     f.Profile()->NumWhenStmts() == 0 && body->Tag() != STMT_CPP )
+		if ( body->Tag() == STMT_CPP )
-			inline_ables.insert(func);
+			continue;
+
+		inline_ables.insert(func);
 		}
 
 	for ( auto& f : funcs )
@@ -165,6 +170,11 @@ ExprPtr Inliner::CheckForInlining(CallExprPtr c)
 		// We don't inline indirect calls.
 		return c;
 
+	if ( c->IsInWhen() )
+		// Don't inline these, as doing so requires propagating
+		// the in-when attribute to the inlined function body.
+		return c;
+
 	auto n = f->AsNameExpr();
 	auto func = n->Id();
 

src/script_opt/ProfileFunc.cc

@@ -28,7 +28,23 @@ ProfileFunc::ProfileFunc(const Func* func, const StmtPtr& body, bool _abs_rec_fi
 	profiled_func = func;
 	profiled_body = body.get();
 	abs_rec_fields = _abs_rec_fields;
-	Profile(func->GetType().get(), body);
+
+	auto ft = func->GetType()->AsFuncType();
+	auto& fcaps = ft->GetCaptures();
+
+	if ( fcaps )
+		{
+		int offset = 0;
+
+		for ( auto& c : *fcaps )
+			{
+			auto cid = c.Id().get();
+			captures.insert(cid);
+			captures_offsets[cid] = offset++;
+			}
+		}
+
+	Profile(ft, body);
 	}
 
 ProfileFunc::ProfileFunc(const Stmt* s, bool _abs_rec_fields)
@@ -48,10 +64,15 @@ ProfileFunc::ProfileFunc(const Expr* e, bool _abs_rec_fields)
 		{
 		auto func = e->AsLambdaExpr();
 
-		for ( auto oid : func->OuterIDs() )
+		int offset = 0;
-			captures.insert(oid);
 
-		Profile(func->GetType()->AsFuncType(), func->Ingredients().Body());
+		for ( auto oid : func->OuterIDs() )
+			{
+			captures.insert(oid);
+			captures_offsets[oid] = offset++;
+			}
+
+		Profile(func->GetType()->AsFuncType(), func->Ingredients()->Body());
 		}
 
 	else
@@ -91,9 +112,9 @@ TraversalCode ProfileFunc::PreStmt(const Stmt* s)
 
 			auto w = s->AsWhenStmt();
 			auto wi = w->Info();
-			auto wl = wi ? wi->Lambda() : nullptr;
+
-			if ( wl )
+			for ( auto wl : wi->WhenNewLocals() )
-				lambdas.push_back(wl.get());
+				when_locals.insert(wl);
 			}
 			break;
 
@@ -171,6 +192,11 @@ TraversalCode ProfileFunc::PreExpr(const Expr* e)
 			auto n = e->AsNameExpr();
 			auto id = n->Id();
 
+			// Turns out that NameExpr's can be constructed using a
+			// different Type* than that of the identifier itself,
+			// so be sure we track the latter too.
+			TrackType(id->GetType());
+
 			if ( id->IsGlobal() )
 				{
 				globals.insert(id);
@@ -179,30 +205,24 @@ TraversalCode ProfileFunc::PreExpr(const Expr* e)
 				const auto& t = id->GetType();
 				if ( t->Tag() == TYPE_FUNC && t->AsFuncType()->Flavor() == FUNC_FLAVOR_EVENT )
 					events.insert(id->Name());
+
+				break;
 				}
 
-			else
+			// This is a tad ugly.  Unfortunately due to the
-				{
+			// weird way that Zeek function *declarations* work,
-				// This is a tad ugly.  Unfortunately due to the
+			// there's no reliable way to get the list of
-				// weird way that Zeek function *declarations* work,
+			// parameters for a function *definition*, since
-				// there's no reliable way to get the list of
+			// they can have different names than what's present
-				// parameters for a function *definition*, since
+			// in the declaration.  So we identify them directly,
-				// they can have different names than what's present
+			// by knowing that they come at the beginning of the
-				// in the declaration.  So we identify them directly,
+			// frame ... and being careful to avoid misconfusing
-				// by knowing that they come at the beginning of the
+			// a lambda capture with a low frame offset as a
-				// frame ... and being careful to avoid misconfusing
+			// parameter.
-				// a lambda capture with a low frame offset as a
+			if ( captures.count(id) == 0 && id->Offset() < num_params )
-				// parameter.
+				params.insert(id);
-				if ( captures.count(id) == 0 && id->Offset() < num_params )
-					params.insert(id);
 
-				locals.insert(id);
+			locals.insert(id);
-				}
-
-			// Turns out that NameExpr's can be constructed using a
-			// different Type* than that of the identifier itself,
-			// so be sure we track the latter too.
-			TrackType(id->GetType());
 
 			break;
 			}
@@ -350,7 +370,12 @@ TraversalCode ProfileFunc::PreExpr(const Expr* e)
 					params.insert(i);
 				}
 
-			// Avoid recursing into the body.
+			// In general, we don't want to recurse into the body.
+			// However, we still want to *profile* it so we can
+			// identify calls within it.
+			ProfileFunc body_pf(l->Ingredients()->Body().get(), false);
+			script_calls.insert(body_pf.ScriptCalls().begin(), body_pf.ScriptCalls().end());
+
 			return TC_ABORTSTMT;
 			}
 

src/script_opt/ProfileFunc.h

@@ -100,6 +100,8 @@ public:
 	const IDSet& Globals() const { return globals; }
 	const IDSet& AllGlobals() const { return all_globals; }
 	const IDSet& Locals() const { return locals; }
+	const IDSet& Captures() const { return captures; }
+	const auto& CapturesOffsets() const { return captures_offsets; }
 	const IDSet& WhenLocals() const { return when_locals; }
 	const IDSet& Params() const { return params; }
 	const std::unordered_map<const ID*, int>& Assignees() const { return assignees; }
@@ -208,6 +210,9 @@ protected:
 	// If we're profiling a lambda function, this holds the captures.
 	IDSet captures;
 
+	// This maps capture identifiers to their offsets.
+	std::map<const ID*, int> captures_offsets;
+
 	// Constants seen in the function.
 	std::vector<const ConstExpr*> constants;
 
@@ -224,7 +229,8 @@ protected:
 	// The same, but in a deterministic order, with duplicates removed.
 	std::vector<const Type*> ordered_types;
 
-	// Script functions that this script calls.
+	// Script functions that this script calls.  Includes calls made
+	// by lambdas and when bodies, as the goal is to identify recursion.
 	std::unordered_set<ScriptFunc*> script_calls;
 
 	// Same for BiF's, though for them we record the corresponding global

src/script_opt/ScriptOpt.cc

@@ -36,15 +36,35 @@ static ZAMCompiler* ZAM = nullptr;
 static bool generating_CPP = false;
 static std::string CPP_dir; // where to generate C++ code
 
+static std::unordered_set<const ScriptFunc*> lambdas;
+static std::unordered_set<const ScriptFunc*> when_lambdas;
 static ScriptFuncPtr global_stmts;
 
 void analyze_func(ScriptFuncPtr f)
 	{
 	// Even if we're analyzing only a subset of the scripts, we still
 	// track all functions here because the inliner will need the full list.
+	ASSERT(f->GetScope());
 	funcs.emplace_back(f, f->GetScope(), f->CurrentBody(), f->CurrentPriority());
 	}
 
+void analyze_lambda(LambdaExpr* l)
+	{
+	auto& pf = l->PrimaryFunc();
+	analyze_func(pf);
+	lambdas.insert(pf.get());
+	}
+
+void analyze_when_lambda(LambdaExpr* l)
+	{
+	when_lambdas.insert(l->PrimaryFunc().get());
+	}
+
+bool is_when_lambda(const ScriptFunc* f)
+	{
+	return when_lambdas.count(f) > 0;
+	}
+
 const FuncInfo* analyze_global_stmts(Stmt* stmts)
 	{
 	// We ignore analysis_options.only_{files,funcs} - if they're in use, later
@@ -208,7 +228,8 @@ static void optimize_func(ScriptFunc* f, std::shared_ptr<ProfileFunc> pf, ScopeP
 
 	rc->SetReadyToOptimize();
 
-	auto ud = std::make_shared<UseDefs>(body, rc);
+	auto ft = cast_intrusive<FuncType>(f->GetType());
+	auto ud = std::make_shared<UseDefs>(body, rc, ft);
 	ud->Analyze();
 
 	if ( analysis_options.dump_uds )
@@ -216,6 +237,9 @@ static void optimize_func(ScriptFunc* f, std::shared_ptr<ProfileFunc> pf, ScopeP
 
 	new_body = ud->RemoveUnused();
 
+	if ( analysis_options.dump_xform )
+		printf("Post removal of unused: %s\n", obj_desc(new_body.get()).c_str());
+
 	if ( new_body != body )
 		{
 		f->ReplaceBody(body, new_body);
@@ -442,10 +466,10 @@ static void analyze_scripts_for_ZAM(std::unique_ptr<ProfileFuncs>& pfs)
 		}
 
 	// Re-profile the functions, now without worrying about compatibility
-	// with compilation to C++.  Note that the first profiling pass earlier
+	// with compilation to C++.
-	// may have marked some of the functions as to-skip, so first clear
+
-	// those markings.  Once we have full compile-to-C++ and ZAM support
+	// The first profiling pass earlier may have marked some of the
-	// for all Zeek language features, we can remove the re-profiling here.
+	// functions as to-skip, so clear those markings.
 	for ( auto& f : funcs )
 		f.SetSkip(false);
 
@@ -496,6 +520,7 @@ static void analyze_scripts_for_ZAM(std::unique_ptr<ProfileFuncs>& pfs)
 	for ( auto& f : funcs )
 		{
 		auto func = f.Func();
+		bool is_lambda = lambdas.count(func) > 0;
 
 		if ( ! analysis_options.only_funcs.empty() || ! analysis_options.only_files.empty() )
 			{
@@ -503,14 +528,17 @@ static void analyze_scripts_for_ZAM(std::unique_ptr<ProfileFuncs>& pfs)
 				continue;
 			}
 
-		else if ( ! analysis_options.compile_all && inl && inl->WasInlined(func) &&
+		else if ( ! analysis_options.compile_all && ! is_lambda && inl && inl->WasInlined(func) &&
 		          func_used_indirectly.count(func) == 0 )
+			{
 			// No need to compile as it won't be called directly.
 			continue;
+			}
 
 		auto new_body = f.Body();
 		optimize_func(func, f.ProfilePtr(), f.Scope(), new_body);
 		f.SetBody(new_body);
+
 		did_one = true;
 		}
 
@@ -593,6 +621,9 @@ void analyze_scripts(bool no_unused_warnings)
 	// At this point we're done with C++ considerations, so instead
 	// are compiling to ZAM.
 	analyze_scripts_for_ZAM(pfs);
+
+	if ( reporter->Errors() > 0 )
+		reporter->FatalError("Optimized script execution aborted due to errors");
 	}
 
 void profile_script_execution()

src/script_opt/ScriptOpt.h

@@ -138,7 +138,6 @@ public:
 	std::shared_ptr<ProfileFunc> ProfilePtr() const { return pf; }
 
 	void SetBody(StmtPtr new_body) { body = std::move(new_body); }
-	// void SetProfile(std::shared_ptr<ProfileFunc> _pf);
 	void SetProfile(std::shared_ptr<ProfileFunc> _pf) { pf = std::move(_pf); }
 
 	// The following provide a way of marking FuncInfo's as
@@ -168,6 +167,16 @@ extern std::unordered_set<const Func*> non_recursive_funcs;
 // Analyze a given function for optimization.
 extern void analyze_func(ScriptFuncPtr f);
 
+// Same, for lambdas.
+extern void analyze_lambda(LambdaExpr* f);
+
+// Same, for lambdas used in "when" statements.  For these, analyze_lambda()
+// has already been called.
+extern void analyze_when_lambda(LambdaExpr* f);
+
+// Whether a given script function is a "when" lambda.
+extern bool is_when_lambda(const ScriptFunc* f);
+
 // Analyze the given top-level statement(s) for optimization.  Returns
 // a pointer to a FuncInfo for an argument-less quasi-function that can
 // be Invoked, or its body executed directly, to execute the statements.

src/script_opt/Stmt.cc

@@ -646,21 +646,26 @@ StmtPtr ReturnStmt::Duplicate()
 
 ReturnStmt::ReturnStmt(ExprPtr arg_e, bool ignored) : ExprStmt(STMT_RETURN, std::move(arg_e)) { }
 
+bool ReturnStmt::IsReduced(Reducer* c) const
+	{
+	if ( ! e || e->IsSingleton(c) )
+		return true;
+
+	return NonReduced(e.get());
+	}
+
 StmtPtr ReturnStmt::DoReduce(Reducer* c)
 	{
 	if ( ! e )
 		return ThisPtr();
 
 	if ( c->Optimizing() )
-		{
 		e = c->OptExpr(e);
-		return ThisPtr();
-		}
 
-	if ( ! e->IsSingleton(c) )
+	else if ( ! e->IsSingleton(c) )
 		{
 		StmtPtr red_e_stmt;
-		e = e->Reduce(c, red_e_stmt);
+		e = e->ReduceToSingleton(c, red_e_stmt);
 
 		if ( red_e_stmt )
 			{
@@ -934,34 +939,85 @@ StmtPtr AssertStmt::DoReduce(Reducer* c)
 	return make_intrusive<NullStmt>();
 	}
 
-StmtPtr WhenStmt::Duplicate()
+bool WhenInfo::HasUnreducedIDs(Reducer* c) const
 	{
-	FuncType::CaptureList* cl_dup = nullptr;
+	for ( auto& cp : *cl )
-
-	if ( wi->Captures() )
 		{
-		cl_dup = new FuncType::CaptureList;
+		auto cid = cp.Id();
-		*cl_dup = *wi->Captures();
+
+		if ( when_new_locals.count(cid.get()) == 0 && ! c->ID_IsReduced(cp.Id()) )
+			return true;
 		}
 
-	auto new_wi = new WhenInfo(Cond(), cl_dup, IsReturn());
+	for ( auto& l : when_expr_locals )
-	new_wi->AddBody(Body());
+		if ( ! c->ID_IsReduced(l) )
-	new_wi->AddTimeout(TimeoutExpr(), TimeoutBody());
+			return true;
 
-	return SetSucc(new WhenStmt(wi));
+	return false;
 	}
 
-void WhenStmt::Inline(Inliner* inl)
+void WhenInfo::UpdateIDs(Reducer* c)
 	{
-	// Don't inline, since we currently don't correctly capture
+	for ( auto& cp : *cl )
-	// the frames of closures.
+		{
+		auto& cid = cp.Id();
+		if ( when_new_locals.count(cid.get()) == 0 )
+			cp.SetID(c->UpdateID(cid));
+		}
+
+	for ( auto& l : when_expr_locals )
+		l = c->UpdateID(l);
+	}
+
+StmtPtr WhenStmt::Duplicate()
+	{
+	return SetSucc(new WhenStmt(new WhenInfo(wi)));
 	}
 
 bool WhenStmt::IsReduced(Reducer* c) const
 	{
-	// We consider these always reduced because they're not
+	if ( wi->HasUnreducedIDs(c) )
-	// candidates for any further optimization.
+		return false;
-	return true;
+
+	if ( ! wi->Lambda()->IsReduced(c) )
+		return false;
+
+	if ( ! wi->TimeoutExpr() )
+		return true;
+
+	return wi->TimeoutExpr()->IsReduced(c);
+	}
+
+StmtPtr WhenStmt::DoReduce(Reducer* c)
+	{
+	if ( ! c->Optimizing() )
+		{
+		wi->UpdateIDs(c);
+		(void)wi->Lambda()->ReduceToSingletons(c);
+		}
+
+	auto e = wi->TimeoutExpr();
+
+	if ( ! e )
+		return ThisPtr();
+
+	if ( c->Optimizing() )
+		wi->SetTimeoutExpr(c->OptExpr(e));
+
+	else if ( ! e->IsSingleton(c) )
+		{
+		StmtPtr red_e_stmt;
+		auto new_e = e->ReduceToSingleton(c, red_e_stmt);
+		wi->SetTimeoutExpr(new_e);
+
+		if ( red_e_stmt )
+			{
+			auto s = make_intrusive<StmtList>(red_e_stmt, ThisPtr());
+			return TransformMe(s, c);
+			}
+		}
+
+	return ThisPtr();
 	}
 
 CatchReturnStmt::CatchReturnStmt(StmtPtr _block, NameExprPtr _ret_var) : Stmt(STMT_CATCH_RETURN)

src/script_opt/UseDefs.cc

@@ -17,10 +17,11 @@ void UseDefSet::Dump() const
 		printf(" %s", u->Name());
 	}
 
-UseDefs::UseDefs(StmtPtr _body, std::shared_ptr<Reducer> _rc)
+UseDefs::UseDefs(StmtPtr _body, std::shared_ptr<Reducer> _rc, FuncTypePtr _ft)
 	{
 	body = std::move(_body);
 	rc = std::move(_rc);
+	ft = std::move(_ft);
 	}
 
 void UseDefs::Analyze()
@@ -164,6 +165,13 @@ bool UseDefs::CheckIfUnused(const Stmt* s, const ID* id, bool report)
 	if ( id->IsGlobal() )
 		return false;
 
+	if ( auto& captures = ft->GetCaptures() )
+		{
+		for ( auto& c : *captures )
+			if ( c.Id() == id )
+				return false;
+		}
+
 	auto uds = FindSuccUsage(s);
 	if ( ! uds || ! uds->HasID(id) )
 		{
@@ -283,9 +291,7 @@ UDs UseDefs::PropagateUDs(const Stmt* s, UDs succ_UDs, const Stmt* succ_stmt, bo
 			auto true_UDs = PropagateUDs(i->TrueBranch(), succ_UDs, succ_stmt, second_pass);
 			auto false_UDs = PropagateUDs(i->FalseBranch(), succ_UDs, succ_stmt, second_pass);
 
-			auto uds = CreateUDs(s, UD_Union(cond_UDs, true_UDs, false_UDs));
+			return CreateUDs(s, UD_Union(cond_UDs, true_UDs, false_UDs));
-
-			return uds;
 			}
 
 		case STMT_INIT:
@@ -295,13 +301,17 @@ UDs UseDefs::PropagateUDs(const Stmt* s, UDs succ_UDs, const Stmt* succ_stmt, bo
 			return UseUDs(s, succ_UDs);
 
 		case STMT_WHEN:
-			// ### Once we support compiling functions with "when"
+			{
-			// statements in them, we'll need to revisit this.
+			auto w = s->AsWhenStmt();
-			// For now, we don't worry about it (because the current
+			auto wi = w->Info();
-			// "when" body semantics of deep-copy frames has different
+			auto uds = UD_Union(succ_UDs, ExprUDs(wi->Lambda().get()));
-			// implications than potentially switching those shallow-copy
+
-			// frames).
+			auto timeout = wi->TimeoutExpr();
-			return UseUDs(s, succ_UDs);
+			if ( timeout )
+				uds = UD_Union(uds, ExprUDs(timeout.get()));
+
+			return CreateUDs(s, uds);
+			}
 
 		case STMT_SWITCH:
 			{
@@ -450,6 +460,7 @@ UDs UseDefs::ExprUDs(const Expr* e)
 	switch ( e->Tag() )
 		{
 		case EXPR_NAME:
+		case EXPR_LAMBDA:
 			AddInExprUDs(uds, e);
 			break;
 
@@ -482,19 +493,23 @@ UDs UseDefs::ExprUDs(const Expr* e)
 			break;
 			}
 
-		case EXPR_CONST:
+		case EXPR_TABLE_CONSTRUCTOR:
-			break;
-
-		case EXPR_LAMBDA:
 			{
-			auto l = static_cast<const LambdaExpr*>(e);
+			auto t = static_cast<const TableConstructorExpr*>(e);
-			auto ids = l->OuterIDs();
+			AddInExprUDs(uds, t->GetOp1().get());
+
+			auto& t_attrs = t->GetAttrs();
+			auto def_attr = t_attrs ? t_attrs->Find(ATTR_DEFAULT) : nullptr;
+			auto& def_expr = def_attr ? def_attr->GetExpr() : nullptr;
+			if ( def_expr && def_expr->Tag() == EXPR_LAMBDA )
+				uds = ExprUDs(def_expr.get());
 
-			for ( const auto& id : ids )
-				AddID(uds, id);
 			break;
 			}
 
+		case EXPR_CONST:
+			break;
+
 		case EXPR_CALL:
 			{
 			auto c = e->AsCallExpr();
@@ -577,6 +592,14 @@ void UseDefs::AddInExprUDs(UDs uds, const Expr* e)
 			AddInExprUDs(uds, e->AsFieldExpr()->Op());
 			break;
 
+		case EXPR_LAMBDA:
+			{
+			auto outer_ids = e->AsLambdaExpr()->OuterIDs();
+			for ( auto& i : outer_ids )
+				AddID(uds, i);
+			break;
+			}
+
 		case EXPR_CONST:
 			// Nothing to do.
 			break;

src/script_opt/UseDefs.h

@@ -51,7 +51,7 @@ class Reducer;
 class UseDefs
 	{
 public:
-	UseDefs(StmtPtr body, std::shared_ptr<Reducer> rc);
+	UseDefs(StmtPtr body, std::shared_ptr<Reducer> rc, FuncTypePtr ft);
 
 	// Does a full pass over the function body's AST.  We can wind
 	// up doing this multiple times because when we use use-defs to
@@ -173,6 +173,7 @@ private:
 
 	StmtPtr body;
 	std::shared_ptr<Reducer> rc;
+	FuncTypePtr ft;
 	};
 
 	} // zeek::detail

src/script_opt/ZAM/AM-Opt.cc

@@ -258,9 +258,9 @@ bool ZAMCompiler::PruneUnused()
 			KillInst(i);
 			}
 
-		if ( inst->IsGlobalLoad() )
+		if ( inst->IsNonLocalLoad() )
 			{
-			// Any straight-line load of the same global
+			// Any straight-line load of the same global/capture
 			// is redundant.
 			for ( unsigned int j = i + 1; j < insts1.size(); ++j )
 				{
@@ -277,14 +277,14 @@ bool ZAMCompiler::PruneUnused()
 					// Inbound branch ends block.
 					break;
 
-				if ( i1->aux && i1->aux->can_change_globals )
+				if ( i1->aux && i1->aux->can_change_non_locals )
 					break;
 
-				if ( ! i1->IsGlobalLoad() )
+				if ( ! i1->IsNonLocalLoad() )
 					continue;
 
-				if ( i1->v2 == inst->v2 )
+				if ( i1->v2 == inst->v2 && i1->IsGlobalLoad() == inst->IsGlobalLoad() )
-					{ // Same global
+					{ // Same global/capture
 					did_prune = true;
 					KillInst(i1);
 					}
@@ -299,9 +299,10 @@ bool ZAMCompiler::PruneUnused()
 			// Variable is used, keep assignment.
 			continue;
 
-		if ( frame_denizens[slot]->IsGlobal() )
+		auto& id = frame_denizens[slot];
+		if ( id->IsGlobal() || IsCapture(id) )
 			{
-			// Extend the global's range to the end of the
+			// Extend the global/capture's range to the end of the
 			// function.
 			denizen_ending[slot] = insts1.back();
 			continue;
@@ -466,18 +467,30 @@ void ZAMCompiler::ComputeFrameLifetimes()
 				break;
 				}
 
+			case OP_LAMBDA_VV:
+				{
+				auto aux = inst->aux;
+				int n = aux->n;
+				auto& slots = aux->slots;
+				for ( int i = 0; i < n; ++i )
+					ExtendLifetime(slots[i], EndOfLoop(inst, 1));
+				break;
+				}
+
 			default:
 				// Look for slots in auxiliary information.
 				auto aux = inst->aux;
 				if ( ! aux || ! aux->slots )
 					break;
 
-				for ( auto j = 0; j < aux->n; ++j )
+				int n = aux->n;
+				auto& slots = aux->slots;
+				for ( auto j = 0; j < n; ++j )
 					{
-					if ( aux->slots[j] < 0 )
+					if ( slots[j] < 0 )
 						continue;
 
-					ExtendLifetime(aux->slots[j], EndOfLoop(inst, 1));
+					ExtendLifetime(slots[j], EndOfLoop(inst, 1));
 					}
 				break;
 			}
@@ -759,7 +772,6 @@ void ZAMCompiler::ReMapVar(const ID* id, int slot, zeek_uint_t inst)
 void ZAMCompiler::CheckSlotAssignment(int slot, const ZInstI* inst)
 	{
 	ASSERT(slot >= 0 && static_cast<zeek_uint_t>(slot) < frame_denizens.size());
-
 	// We construct temporaries such that their values are never used
 	// earlier than their definitions in loop bodies.  For other
 	// denizens, however, they can be, so in those cases we expand the
@@ -915,47 +927,6 @@ const ZInstI* ZAMCompiler::EndOfLoop(const ZInstI* inst, int depth) const
 	return insts1[i];
 	}
 
-bool ZAMCompiler::VarIsAssigned(int slot) const
-	{
-	for ( auto& inst : insts1 )
-		if ( inst->live && VarIsAssigned(slot, inst) )
-			return true;
-
-	return false;
-	}
-
-bool ZAMCompiler::VarIsAssigned(int slot, const ZInstI* i) const
-	{
-	// Special-case for table iterators, which assign to a bunch
-	// of variables but they're not immediately visible in the
-	// instruction layout.
-	if ( i->op == OP_NEXT_TABLE_ITER_VAL_VAR_VVV || i->op == OP_NEXT_TABLE_ITER_VV )
-		{
-		auto& iter_vars = i->aux->loop_vars;
-		for ( auto v : iter_vars )
-			if ( v == slot )
-				return true;
-
-		if ( i->op != OP_NEXT_TABLE_ITER_VAL_VAR_VVV )
-			return false;
-
-		// Otherwise fall through, since that flavor of iterate
-		// *does* also assign to slot 1.
-		}
-
-	if ( i->op == OP_NEXT_VECTOR_ITER_VAL_VAR_VVVV && i->v2 == slot )
-		return true;
-
-	if ( i->op_type == OP_VV_FRAME )
-		// We don't want to consider these as assigning to the
-		// variable, since the point of this method is to figure
-		// out which variables don't need storing to the frame
-		// because their internal value is never modified.
-		return false;
-
-	return i->AssignsToSlot1() && i->v1 == slot;
-	}
-
 bool ZAMCompiler::VarIsUsed(int slot) const
 	{
 	for ( auto& inst : insts1 )

src/script_opt/ZAM/Compile.h

@@ -92,6 +92,7 @@ private:
 	void Init();
 	void InitGlobals();
 	void InitArgs();
+	void InitCaptures();
 	void InitLocals();
 	void TrackMemoryManagement();
 
@@ -137,6 +138,7 @@ private:
 	const ZAMStmt CompileCatchReturn(const CatchReturnStmt* cr);
 	const ZAMStmt CompileStmts(const StmtList* sl);
 	const ZAMStmt CompileInit(const InitStmt* is);
+	const ZAMStmt CompileWhen(const WhenStmt* ws);
 
 	const ZAMStmt CompileNext() { return GenGoTo(nexts.back()); }
 	const ZAMStmt CompileBreak() { return GenGoTo(breaks.back()); }
@@ -219,10 +221,15 @@ private:
 	const ZAMStmt CompileInExpr(const NameExpr* n1, const ListExpr* l, const NameExpr* n2,
 	                            const ConstExpr* c);
 
-	const ZAMStmt CompileIndex(const NameExpr* n1, const NameExpr* n2, const ListExpr* l);
+	const ZAMStmt CompileIndex(const NameExpr* n1, const NameExpr* n2, const ListExpr* l,
-	const ZAMStmt CompileIndex(const NameExpr* n1, const ConstExpr* c, const ListExpr* l);
+	                           bool in_when);
+	const ZAMStmt CompileIndex(const NameExpr* n1, const ConstExpr* c, const ListExpr* l,
+	                           bool in_when);
 	const ZAMStmt CompileIndex(const NameExpr* n1, int n2_slot, const TypePtr& n2_type,
-	                           const ListExpr* l);
+	                           const ListExpr* l, bool in_when);
+
+	const ZAMStmt BuildLambda(const NameExpr* n, LambdaExpr* le);
+	const ZAMStmt BuildLambda(int n_slot, LambdaExpr* le);
 
 	// Second argument is which instruction slot holds the branch target.
 	const ZAMStmt GenCond(const Expr* e, int& branch_v);
@@ -350,8 +357,15 @@ private:
 
 	bool IsUnused(const IDPtr& id, const Stmt* where) const;
 
+	bool IsCapture(const IDPtr& id) const { return IsCapture(id.get()); }
+	bool IsCapture(const ID* id) const;
+
+	int CaptureOffset(const IDPtr& id) const { return IsCapture(id.get()); }
+	int CaptureOffset(const ID* id) const;
+
 	void LoadParam(const ID* id);
 	const ZAMStmt LoadGlobal(const ID* id);
+	const ZAMStmt LoadCapture(const ID* id);
 
 	int AddToFrame(const ID*);
 
@@ -445,14 +459,6 @@ private:
 	const ZInstI* BeginningOfLoop(const ZInstI* inst, int depth) const;
 	const ZInstI* EndOfLoop(const ZInstI* inst, int depth) const;
 
-	// True if any statement other than a frame sync assigns to the
-	// given slot.
-	bool VarIsAssigned(int slot) const;
-
-	// True if the given statement assigns to the given slot, and
-	// it's not a frame sync.
-	bool VarIsAssigned(int slot, const ZInstI* i) const;
-
 	// True if any statement other than a frame sync uses the given slot.
 	bool VarIsUsed(int slot) const;
 
@@ -599,8 +605,10 @@ private:
 
 	// Used for communication between Frame1Slot and a subsequent
 	// AddInst.  If >= 0, then upon adding the next instruction,
-	// it should be followed by Store-Global for the given slot.
+	// it should be followed by Store-Global or Store-Capture for
+	// the given slot.
 	int pending_global_store = -1;
+	int pending_capture_store = -1;
 	};
 
 // Invokes after compiling all of the function bodies.

src/script_opt/ZAM/Driver.cc

@@ -33,20 +33,9 @@ void ZAMCompiler::Init()
 	{
 	InitGlobals();
 	InitArgs();
+	InitCaptures();
 	InitLocals();
 
-#if 0
-	// Complain about unused aggregates ... but not if we're inlining,
-	// as that can lead to optimizations where they wind up being unused
-	// but the original logic for using them was sound.
-	if ( ! analysis_options.inliner )
-		for ( auto a : pf->Inits() )
-			{
-			if ( pf->Locals().find(a) == pf->Locals().end() )
-				reporter->Warning("%s unused", a->Name());
-			}
-#endif
-
 	TrackMemoryManagement();
 
 	non_recursive = non_recursive_funcs.count(func) > 0;
@@ -92,12 +81,25 @@ void ZAMCompiler::InitArgs()
 	pop_scope();
 	}
 
+void ZAMCompiler::InitCaptures()
+	{
+	for ( auto c : pf->Captures() )
+		(void)AddToFrame(c);
+	}
+
 void ZAMCompiler::InitLocals()
 	{
 	// Assign slots for locals (which includes temporaries).
 	for ( auto l : pf->Locals() )
 		{
+		if ( IsCapture(l) )
+			continue;
+
+		if ( pf->WhenLocals().count(l) > 0 )
+			continue;
+
 		auto non_const_l = const_cast<ID*>(l);
+
 		// Don't add locals that were already added because they're
 		// parameters.
 		//
@@ -203,11 +205,6 @@ StmtPtr ZAMCompiler::CompileBody()
 	// Could erase insts1 here to recover memory, but it's handy
 	// for debugging.
 
-#if 0
-	if ( non_recursive )
-		func->UseStaticFrame();
-#endif
-
 	auto zb = make_intrusive<ZBody>(func->Name(), this);
 	zb->SetInsts(insts2);
 

src/script_opt/ZAM/Expr.cc

@@ -4,6 +4,7 @@
 
 #include "zeek/Desc.h"
 #include "zeek/Reporter.h"
+#include "zeek/script_opt/ProfileFunc.h"
 #include "zeek/script_opt/ZAM/Compile.h"
 
 namespace zeek::detail
@@ -176,12 +177,6 @@ const ZAMStmt ZAMCompiler::CompileAssignExpr(const AssignExpr* e)
 	auto r2 = rhs->GetOp2();
 	auto r3 = rhs->GetOp3();
 
-	if ( rhs->Tag() == EXPR_LAMBDA )
-		{
-		// reporter->Error("lambda expressions not supported for compiling");
-		return ErrorStmt();
-		}
-
 	if ( rhs->Tag() == EXPR_NAME )
 		return AssignVV(lhs, rhs->AsNameExpr());
 
@@ -213,6 +208,9 @@ const ZAMStmt ZAMCompiler::CompileAssignExpr(const AssignExpr* e)
 	if ( rhs->Tag() == EXPR_ANY_INDEX )
 		return AnyIndexVVi(lhs, r1->AsNameExpr(), rhs->AsAnyIndexExpr()->Index());
 
+	if ( rhs->Tag() == EXPR_LAMBDA )
+		return BuildLambda(lhs, rhs->AsLambdaExpr());
+
 	if ( rhs->Tag() == EXPR_COND && r1->GetType()->Tag() == TYPE_VECTOR )
 		return Bool_Vec_CondVVVV(lhs, r1->AsNameExpr(), r2->AsNameExpr(), r3->AsNameExpr());
 
@@ -285,7 +283,18 @@ const ZAMStmt ZAMCompiler::CompileAssignToIndex(const NameExpr* lhs, const Index
 		                  : IndexVecIntSelectVVV(lhs, n, index);
 		}
 
-	return const_aggr ? IndexVCL(lhs, con, indexes_expr) : IndexVVL(lhs, n, indexes_expr);
+	if ( rhs->IsInsideWhen() )
+		{
+		if ( const_aggr )
+			return WhenIndexVCL(lhs, con, indexes_expr);
+		else
+			return WhenIndexVVL(lhs, n, indexes_expr);
+		}
+
+	if ( const_aggr )
+		return IndexVCL(lhs, con, indexes_expr);
+	else
+		return IndexVVL(lhs, n, indexes_expr);
 	}
 
 const ZAMStmt ZAMCompiler::CompileFieldLHSAssignExpr(const FieldLHSAssignExpr* e)
@@ -616,26 +625,28 @@ const ZAMStmt ZAMCompiler::CompileInExpr(const NameExpr* n1, const ListExpr* l,
 	return AddInst(z);
 	}
 
-const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, const NameExpr* n2, const ListExpr* l)
+const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, const NameExpr* n2, const ListExpr* l,
+                                        bool in_when)
 	{
-	return CompileIndex(n1, FrameSlot(n2), n2->GetType(), l);
+	return CompileIndex(n1, FrameSlot(n2), n2->GetType(), l, in_when);
 	}
 
-const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n, const ConstExpr* c, const ListExpr* l)
+const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n, const ConstExpr* c, const ListExpr* l,
+                                        bool in_when)
 	{
 	auto tmp = TempForConst(c);
-	return CompileIndex(n, tmp, c->GetType(), l);
+	return CompileIndex(n, tmp, c->GetType(), l, in_when);
 	}
 
 const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, int n2_slot, const TypePtr& n2t,
-                                        const ListExpr* l)
+                                        const ListExpr* l, bool in_when)
 	{
 	ZInstI z;
 
 	int n = l->Exprs().length();
 	auto n2tag = n2t->Tag();
 
-	if ( n == 1 )
+	if ( n == 1 && ! in_when )
 		{
 		auto ind = l->Exprs()[0];
 		auto var_ind = ind->Tag() == EXPR_NAME;
@@ -677,12 +688,28 @@ const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, int n2_slot, const T
 			if ( n3 )
 				{
 				int n3_slot = FrameSlot(n3);
-				auto zop = is_any ? OP_INDEX_ANY_VEC_VVV : OP_INDEX_VEC_VVV;
+
+				ZOp zop;
+				if ( in_when )
+					zop = OP_WHEN_INDEX_VEC_VVV;
+				else if ( is_any )
+					zop = OP_INDEX_ANY_VEC_VVV;
+				else
+					zop = OP_INDEX_VEC_VVV;
+
 				z = ZInstI(zop, Frame1Slot(n1, zop), n2_slot, n3_slot);
 				}
 			else
 				{
-				auto zop = is_any ? OP_INDEX_ANY_VECC_VVV : OP_INDEX_VECC_VVV;
+				ZOp zop;
+
+				if ( in_when )
+					zop = OP_WHEN_INDEX_VECC_VVV;
+				else if ( is_any )
+					zop = OP_INDEX_ANY_VECC_VVV;
+				else
+					zop = OP_INDEX_VECC_VVV;
+
 				z = ZInstI(zop, Frame1Slot(n1, zop), n2_slot, c);
 				z.op_type = OP_VVV_I3;
 				}
@@ -720,13 +747,13 @@ const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, int n2_slot, const T
 	switch ( n2tag )
 		{
 		case TYPE_VECTOR:
-			op = OP_INDEX_VEC_SLICE_VV;
+			op = in_when ? OP_WHEN_INDEX_VEC_SLICE_VV : OP_INDEX_VEC_SLICE_VV;
 			z = ZInstI(op, Frame1Slot(n1, op), n2_slot);
 			z.SetType(n2t);
 			break;
 
 		case TYPE_TABLE:
-			op = OP_TABLE_INDEX_VV;
+			op = in_when ? OP_WHEN_TABLE_INDEX_VV : OP_TABLE_INDEX_VV;
 			z = ZInstI(op, Frame1Slot(n1, op), n2_slot);
 			z.SetType(n1->GetType());
 			break;
@@ -747,6 +774,38 @@ const ZAMStmt ZAMCompiler::CompileIndex(const NameExpr* n1, int n2_slot, const T
 	return AddInst(z);
 	}
 
+const ZAMStmt ZAMCompiler::BuildLambda(const NameExpr* n, LambdaExpr* le)
+	{
+	return BuildLambda(Frame1Slot(n, OP1_WRITE), le);
+	}
+
+const ZAMStmt ZAMCompiler::BuildLambda(int n_slot, LambdaExpr* le)
+	{
+	auto& captures = le->GetCaptures();
+	int ncaptures = captures ? captures->size() : 0;
+
+	auto aux = new ZInstAux(ncaptures);
+	aux->primary_func = le->PrimaryFunc();
+	aux->lambda_name = le->Name();
+	aux->id_val = le->Ingredients()->GetID();
+
+	for ( int i = 0; i < ncaptures; ++i )
+		{
+		auto& id_i = (*captures)[i].Id();
+
+		if ( pf->WhenLocals().count(id_i.get()) > 0 )
+			aux->Add(i, nullptr);
+		else
+			aux->Add(i, FrameSlot(id_i), id_i->GetType());
+		}
+
+	auto z = ZInstI(OP_LAMBDA_VV, n_slot, le->PrimaryFunc()->FrameSize());
+	z.op_type = OP_VV_I2;
+	z.aux = aux;
+
+	return AddInst(z);
+	}
+
 const ZAMStmt ZAMCompiler::AssignVecElems(const Expr* e)
 	{
 	auto index_assign = e->AsIndexAssignExpr();
@@ -887,16 +946,17 @@ const ZAMStmt ZAMCompiler::AssignToCall(const ExprStmt* e)
 const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n)
 	{
 	auto func = c->Func()->AsNameExpr();
-	auto func_id = func->Id();
+	auto func_id = func->IdPtr();
 	auto& args = c->Args()->Exprs();
 
 	int nargs = args.length();
 	int call_case = nargs;
 
 	bool indirect = ! func_id->IsGlobal() || ! func_id->GetVal();
+	bool in_when = c->IsInWhen();
 
-	if ( indirect )
+	if ( indirect || in_when )
-		call_case = -1; // force default of CallN
+		call_case = -1; // force default of some flavor of CallN
 
 	auto nt = n ? n->GetType()->Tag() : TYPE_VOID;
 	auto n_slot = n ? Frame1Slot(n, OP1_WRITE) : -1;
@@ -973,8 +1033,17 @@ const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n)
 				break;
 
 			default:
-				if ( indirect )
+				if ( in_when )
+					{
+					if ( indirect )
+						op = OP_WHENINDCALLN_VV;
+					else
+						op = OP_WHENCALLN_V;
+					}
+
+				else if ( indirect )
 					op = n ? OP_INDCALLN_VV : OP_INDCALLN_V;
+
 				else
 					op = n ? OP_CALLN_V : OP_CALLN_X;
 				break;
@@ -982,7 +1051,9 @@ const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n)
 
 		if ( n )
 			{
-			op = AssignmentFlavor(op, nt);
+			if ( ! in_when )
+				op = AssignmentFlavor(op, nt);
+
 			auto n_slot = Frame1Slot(n, OP1_WRITE);
 
 			if ( indirect )
@@ -1023,10 +1094,13 @@ const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n)
 	if ( ! z.aux )
 		z.aux = new ZInstAux(0);
 
-	z.aux->can_change_globals = true;
+	z.aux->can_change_non_locals = true;
 
 	z.call_expr = c;
 
+	if ( in_when )
+		z.SetType(n->GetType());
+
 	if ( ! indirect || func_id->IsGlobal() )
 		{
 		z.aux->id_val = func_id;
@@ -1035,19 +1109,6 @@ const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n)
 			z.func = func_id->GetVal()->AsFunc();
 		}
 
-	if ( n )
-		{
-		auto id = n->Id();
-		if ( id->IsGlobal() )
-			{
-			AddInst(z);
-			auto global_slot = global_id_to_info[id];
-			z = ZInstI(OP_STORE_GLOBAL_V, global_slot);
-			z.op_type = OP_V_I1;
-			z.t = globalsI[global_slot].id->GetType();
-			}
-		}
-
 	return AddInst(z);
 	}
 
@@ -1062,6 +1123,31 @@ const ZAMStmt ZAMCompiler::ConstructTable(const NameExpr* n, const Expr* e)
 	z.t = tt;
 	z.attrs = e->AsTableConstructorExpr()->GetAttrs();
 
+	auto zstmt = AddInst(z);
+
+	auto def_attr = z.attrs ? z.attrs->Find(ATTR_DEFAULT) : nullptr;
+	if ( ! def_attr || def_attr->GetExpr()->Tag() != EXPR_LAMBDA )
+		return zstmt;
+
+	auto def_lambda = def_attr->GetExpr()->AsLambdaExpr();
+	auto dl_t = def_lambda->GetType()->AsFuncType();
+	auto& captures = dl_t->GetCaptures();
+
+	if ( ! captures )
+		return zstmt;
+
+	// What a pain.  The table's default value is a lambda that has
+	// captures.  The semantics of this are that the captures are
+	// evaluated at table-construction time.  We need to build the
+	// lambda and assign it as the table's default.
+
+	auto slot = NewSlot(true); // since func_val's are managed
+	(void)BuildLambda(slot, def_lambda);
+
+	z = GenInst(OP_SET_TABLE_DEFAULT_LAMBDA_VV, n, slot);
+	z.op_type = OP_VV;
+	z.t = def_lambda->GetType();
+
 	return AddInst(z);
 	}
 

src/script_opt/ZAM/Low-Level.cc

@@ -149,6 +149,9 @@ const ZAMStmt ZAMCompiler::AddInst(const ZInstI& inst, bool suppress_non_local)
 	if ( suppress_non_local )
 		return ZAMStmt(top_main_inst);
 
+	// Ensure we haven't confused ourselves about any pending stores.
+	ASSERT(pending_global_store == -1 || pending_capture_store == -1);
+
 	if ( pending_global_store >= 0 )
 		{
 		auto gs = pending_global_store;
@@ -161,6 +164,27 @@ const ZAMStmt ZAMCompiler::AddInst(const ZInstI& inst, bool suppress_non_local)
 		return AddInst(store_inst);
 		}
 
+	if ( pending_capture_store >= 0 )
+		{
+		auto cs = pending_capture_store;
+		pending_capture_store = -1;
+
+		auto& cv = *func->GetType()->AsFuncType()->GetCaptures();
+		auto& c_id = cv[cs].Id();
+
+		ZOp op;
+
+		if ( ZVal::IsManagedType(c_id->GetType()) )
+			op = OP_STORE_MANAGED_CAPTURE_VV;
+		else
+			op = OP_STORE_CAPTURE_VV;
+
+		auto store_inst = ZInstI(op, RawSlot(c_id.get()), cs);
+		store_inst.op_type = OP_VV_I2;
+
+		return AddInst(store_inst);
+		}
+
 	return ZAMStmt(top_main_inst);
 	}
 

src/script_opt/ZAM/Ops.in

@@ -12,7 +12,7 @@
 # The Gen-ZAM utility processes this file to generate numerous C++ inclusion
 # files that are then compiled into Zeek.  These files span the range of (1)
 # hooks that enable run-time generation of ZAM code to execute ASTs (which
-# have first been transformed to "reduced" form, (2) specifications of the
+# have first been transformed to "reduced" form), (2) specifications of the
 # properties of the different instructions, (3) code to evaluate (execute)
 # each instruction, and (4) macros (C++ #define's) to aid in writing that
 # code.  See Gen-ZAM.h for a list of the different inclusion files.
@@ -947,11 +947,19 @@ eval	EvalIndexVecIntSelect(z.c, frame[z.v2])
 
 op Index
 type VVL
-custom-method return CompileIndex(n1, n2, l);
+custom-method return CompileIndex(n1, n2, l, false);
 
 op Index
 type VCL
-custom-method return CompileIndex(n, c, l);
+custom-method return CompileIndex(n, c, l, false);
+
+op WhenIndex
+type VVL
+custom-method return CompileIndex(n1, n2, l, true);
+
+op WhenIndex
+type VCL
+custom-method return CompileIndex(n, c, l, true);
 
 internal-op Index-Vec
 type VVV
@@ -988,22 +996,51 @@ internal-op Index-Any-VecC
 type VVV
 eval	EvalIndexAnyVec(z.v3)
 
-internal-op Index-Vec-Slice
+macro	WhenIndexResCheck()
-type VV
+	auto& res = frame[z.v1].vector_val;
-eval	auto vec = frame[z.v2].vector_val;
+	if ( res && IndexExprWhen::evaluating > 0 )
+		IndexExprWhen::results.push_back({NewRef{}, res});
+
+internal-op When-Index-Vec
+type VVV
+eval	EvalIndexAnyVec(frame[z.v3].uint_val)
+	WhenIndexResCheck()
+
+internal-op When-Index-VecC
+type VVV
+eval	EvalIndexAnyVec(z.v3)
+	WhenIndexResCheck()
+
+macro	EvalVecSlice()
+	auto vec = frame[z.v2].vector_val;
 	auto lv = z.aux->ToListVal(frame);
 	auto v = index_slice(vec, lv.get());
 	Unref(frame[z.v1].vector_val);
 	frame[z.v1].vector_val = v.release();
 
+internal-op Index-Vec-Slice
+type VV
+eval	EvalVecSlice()
+
+internal-op When-Index-Vec-Slice
+type VV
+eval	EvalVecSlice()
+	WhenIndexResCheck()
+
 internal-op Table-Index
 type VV
 eval	EvalTableIndex(z.aux->ToListVal(frame))
 	AssignV1(BuildVal(v, z.t))
 
+internal-op When-Table-Index
+type VV
+eval	EvalTableIndex(z.aux->ToListVal(frame))
+	if ( IndexExprWhen::evaluating > 0 )
+		IndexExprWhen::results.emplace_back(v);
+	AssignV1(BuildVal(v, z.t))
+
 macro EvalTableIndex(index)
-	auto v2 = index;
+	auto v = frame[z.v2].table_val->FindOrDefault(index);
-	auto v = frame[z.v2].table_val->FindOrDefault(v2);
 	if ( ! v )
 		{
 		ZAM_run_time_error(z.loc, "no such index");
@@ -1092,6 +1129,15 @@ eval	ConstructTableOrSetPre()
 		}
 	ConstructTableOrSetPost()
 
+# When tables are constructed, if their &default is a lambda with captures
+# then we need to explicitly set up the default.
+internal-op Set-Table-Default-Lambda
+type VV
+op1-read
+eval	auto& tbl = frame[z.v1].table_val;
+	auto lambda = frame[z.v2].ToVal(z.t);
+	tbl->InitDefaultVal(lambda);
+
 direct-unary-op Set-Constructor ConstructSet
 
 internal-op Construct-Set
@@ -1527,6 +1573,49 @@ assign-val v
 indirect-call
 num-call-args n
 
+# A call made in a "when" context.  These always have assignment targets.
+# To keep things simple, we just use one generic flavor (for N arguments,
+# doing a less-streamlined-but-simpler Val-based assignment).
+macro WhenCall(func)
+	if ( ! func )
+		throw ZAMDelayedCallException();
+	auto& lhs = frame[z.v1];
+	auto trigger = f->GetTrigger();
+	Val* v = trigger ? trigger->Lookup(z.call_expr) : nullptr;
+	ValPtr vp;
+	if ( v )
+		vp = {NewRef{}, v};
+	else
+		{
+		auto aux = z.aux;
+		auto current_assoc = f->GetTriggerAssoc();
+		auto n = aux->n;
+		std::vector<ValPtr> args;
+		for ( auto i = 0; i < n; ++i )
+			args.push_back(aux->ToVal(frame, i));
+		f->SetCall(z.call_expr);
+		vp = func->Invoke(&args, f);
+		f->SetTriggerAssoc(current_assoc);
+		if ( ! vp )
+			throw ZAMDelayedCallException();
+		}
+	if ( z.is_managed )
+		ZVal::DeleteManagedType(lhs);
+	lhs = ZVal(vp, z.t);
+
+internal-op WhenCallN
+type V
+side-effects
+eval	WhenCall(z.func)
+
+internal-op WhenIndCallN
+type VV
+side-effects
+eval	auto sel = z.v2;
+	auto func = (sel < 0) ? z.aux->id_val->GetVal()->AsFunc() : frame[sel].AsFunc();
+	WhenCall(func)
+
+
 ########## Statements ##########
 
 macro EvalScheduleArgs(time, is_delta, build_args)
@@ -1956,6 +2045,35 @@ eval	auto tt = cast_intrusive<TableType>(z.t);
 	Unref(frame[z.v1].table_val);
 	frame[z.v1].table_val = t;
 
+op When
+type V
+op1-read
+eval	BuildWhen(-1.0)
+
+op When-Timeout
+type VV
+op1-read
+eval	BuildWhen(frame[z.v2].double_val)
+
+op When-Timeout
+type VC
+op1-read
+eval	BuildWhen(z.c.double_val)
+
+macro BuildWhen(timeout)
+	auto& aux = z.aux;
+	auto wi = aux->wi;
+	FuncPtr func{NewRef{}, frame[z.v1].func_val};
+	auto lambda = make_intrusive<FuncVal>(func);
+	wi->Instantiate(lambda);
+	std::vector<ValPtr> local_aggrs;
+	for ( int i = 0; i < aux->n; ++i )
+		{
+		auto v = aux->ToVal(frame, i);
+		if ( v )
+			local_aggrs.push_back(v);
+		}
+	new trigger::Trigger(wi, timeout, wi->WhenExprGlobals(), local_aggrs, f, z.loc);
 
 ########################################
 # Internal
@@ -1994,7 +2112,7 @@ assign-val v
 eval	auto v = globals[z.v2].id->GetVal();
 	if ( ! v )
 		{
-		ZAM_run_time_error(z.loc, "value used but not set", z.aux->id_val);
+		ZAM_run_time_error(z.loc, "value used but not set", z.aux->id_val.get());
 		break;
 		}
 
@@ -2007,12 +2125,41 @@ eval	auto& v = frame[z.v1].type_val;
 	auto t = globals[z.v2].id->GetType();
 	v = new TypeVal(t, true);
 
+internal-op Load-Capture
+type VV
+eval	frame[z.v1] = f->GetFunction()->GetCapturesVec()[z.v2];
+
+internal-op Load-Managed-Capture
+type VV
+eval	auto& lhs = frame[z.v1];
+	auto& rhs = f->GetFunction()->GetCapturesVec()[z.v2];
+	zeek::Ref(rhs.ManagedVal());
+	ZVal::DeleteManagedType(lhs);
+	lhs = rhs;
+
 internal-op Store-Global
 op1-internal
 type V
 eval	auto& g = globals[z.v1];
 	g.id->SetVal(frame[g.slot].ToVal(z.t));
 
+# Both of these have the LHS as v2 not v1, to keep with existing
+# conventions of OP_VV_I2 op type (as opposed to OP_VV_I1_V2, which doesn't
+# currently exist, and would be a pain to add).
+internal-op Store-Capture
+op1-read
+type VV
+eval	f->GetFunction()->GetCapturesVec()[z.v2] = frame[z.v1];
+
+internal-op Store-Managed-Capture
+op1-read
+type VV
+eval	auto& lhs = f->GetFunction()->GetCapturesVec()[z.v2];
+	auto& rhs = frame[z.v1];
+	zeek::Ref(rhs.ManagedVal());
+	ZVal::DeleteManagedType(lhs);
+	lhs = rhs;
+
 
 internal-op Copy-To
 type VC
@@ -2029,6 +2176,37 @@ eval	flow = FLOW_BREAK;
 	pc = end_pc;
 	continue;
 
+# Slot 2 gives frame size.
+internal-op Lambda
+type VV
+eval	auto& aux = z.aux;
+	auto& primary_func = aux->primary_func;
+	auto& body = primary_func->GetBodies()[0].stmts;
+	ASSERT(body->Tag() == STMT_ZAM);
+	auto lamb = make_intrusive<ScriptFunc>(aux->id_val);
+	lamb->AddBody(body, z.v2);
+	lamb->SetName(aux->lambda_name.c_str());
+	if ( aux->n > 0 )
+		{
+		auto captures = std::make_unique<std::vector<ZVal>>();
+		for ( auto i = 0; i < aux->n; ++i )
+			{
+			auto slot = aux->slots[i];
+			if ( slot >= 0 )
+				{
+				auto& cp = frame[aux->slots[i]];
+				if ( aux->is_managed[i] )
+					zeek::Ref(cp.ManagedVal());
+				captures->push_back(cp);
+				}
+			else
+				// Used for when-locals.
+				captures->push_back(ZVal());
+			}
+		lamb->CreateCaptures(std::move(captures));
+		}
+	ZVal::DeleteManagedType(frame[z.v1]);
+	frame[z.v1].func_val = lamb.release();
 
 ########################################
 # Built-in Functions

src/script_opt/ZAM/README.md

@@ -41,35 +41,29 @@ the _Logging Framework_, for example, and thus you won't see much improvement.
 * Those two factors add up to gains very often on the order of only 10-15%,
 rather than something a lot more dramatic.
 
-* In addition, there are some
-[types of scripts that currently can't be compiled](#Scripts-that-cannot-be-compiled),
-and thus will remain interpreted.  If your processing bottlenecks in such
-scripts, you won't see much in the way of gains.
-
 <br>
 
 ## Known Issues
 
 
 Here we list various issues with using script optimization, including both
-deficiencies (problems to eventually fix) and incompatibilities (differences
+deficiencies (things that don't work as well as you might like)
-in behavior from the default of script interpretation, not necessarily
+and incompatibilities (differences in behavior from the default
-fixable).  For each, the corresponding list is roughly ordered from
+of script interpretation).
-you're-most-likely-to-care-about-it to you're-less-likely-to-care, though
-of course this varies for different users.
 <br>
 
-### Deficiencies to eventually fix:
+### Deficiencies:
 
-* Error messages in compiled scripts have diminished identifying
+* Run-time error messages in compiled scripts have diminished identifying
 information.
 
 * The optimizer assumes you have ensured initialization of your variables.
 If your script uses a variable that hasn't been set, the compiled code may
 crash or behave aberrantly. You can use the `-u` command-line flag to find such potential usage issues.
 
-* Certain complex "when" expressions may fail to reevaluate when elements
+* When printing scripts (such as in some error messages), the names of
-of the expression are modified by compiled scripts.
+variables often reflect internal temporaries rather than the original
+variables.
 
 <br>
 
@@ -77,30 +71,11 @@ of the expression are modified by compiled scripts.
 
 * ZAM ignores `assert` statements.
 
-* When printing scripts (such as in some error messages), the names of
-variables often reflect internal temporaries rather than the original
-variables.
-
 * The `same_object()` BiF will always deem two non-container values as
 different.
 
 <br>
 
-### Scripts that cannot be compiled:
-
-The ZAM optimizer does not compile scripts that include "when" statements or
-lambda expressions.  These will take substantial work to support.  It also
-will not inline such scripts, nor will it inline scripts that are either
-directly or indirectly recursive.
-
-You can get a list of non-compilable scripts using
-`-O ZAM -O report-uncompilable`.  For recursive scripts, use
-`-O report-recursive` (no `-O ZAM` required, since it doesn't apply to the
-alternative optimization, `-O gen-C++`).
-
-<br>
-
-
 ## Script Optimization Options
 
 Users will generally simply use `-O ZAM` to invoke the script optimizer.
@@ -120,7 +95,7 @@ issues:
 |`optimize-AST`	|	Optimize the (transform) AST; implies `xform`.|
 |`profile-ZAM`	|	Generate to _stdout_ a ZAM execution profile. (Requires configuring with `--enable-debug`.)|
 |`report-recursive`	|	Report on recursive functions and exit.|
-|`report-uncompilable`	|	Report on uncompilable functions and exit.|
+|`report-uncompilable`	|	Report on uncompilable functions and exit. For ZAM, all functions should be compilable.|
 |`xform`		|	Transform scripts to "reduced" form.|
 
 <br>

src/script_opt/ZAM/Stmt.cc

@@ -60,6 +60,9 @@ const ZAMStmt ZAMCompiler::CompileStmt(const Stmt* s)
 		case STMT_INIT:
 			return CompileInit(static_cast<const InitStmt*>(s));
 
+		case STMT_WHEN:
+			return CompileWhen(static_cast<const WhenStmt*>(s));
+
 		case STMT_NULL:
 			return EmptyStmt();
 
@@ -1093,6 +1096,60 @@ const ZAMStmt ZAMCompiler::CompileInit(const InitStmt* is)
 	return last;
 	}
 
+const ZAMStmt ZAMCompiler::CompileWhen(const WhenStmt* ws)
+	{
+	auto wi = ws->Info();
+	auto timeout = wi->TimeoutExpr();
+
+	auto lambda = NewSlot(true);
+	(void)BuildLambda(lambda, wi->Lambda().get());
+
+	std::vector<IDPtr> local_aggr_slots;
+	for ( auto& l : wi->WhenExprLocals() )
+		if ( IsAggr(l->GetType()->Tag()) )
+			local_aggr_slots.push_back(l);
+
+	int n = local_aggr_slots.size();
+	auto aux = new ZInstAux(n);
+	aux->wi = wi;
+
+	for ( auto i = 0; i < n; ++i )
+		{
+		auto la = local_aggr_slots[i];
+		aux->Add(i, FrameSlot(la), la->GetType());
+		}
+
+	ZInstI z;
+
+	if ( timeout )
+		{
+		if ( timeout->Tag() == EXPR_NAME )
+			{
+			auto ns = FrameSlot(timeout->AsNameExpr());
+			z = ZInstI(OP_WHEN_TIMEOUT_VV, lambda, ns);
+			}
+		else
+			{
+			ASSERT(timeout->Tag() == EXPR_CONST);
+			z = ZInstI(OP_WHEN_TIMEOUT_VC, lambda, timeout->AsConstExpr());
+			}
+		}
+
+	else
+		z = ZInstI(OP_WHEN_V, lambda);
+
+	z.aux = aux;
+
+	if ( ws->IsReturn() )
+		{
+		(void)AddInst(z);
+		z = ZInstI(OP_RETURN_C);
+		z.c = ZVal();
+		}
+
+	return AddInst(z);
+	}
+
 const ZAMStmt ZAMCompiler::InitRecord(IDPtr id, RecordType* rt)
 	{
 	auto z = ZInstI(OP_INIT_RECORD_V, FrameSlot(id));

src/script_opt/ZAM/Support.cc

@@ -20,20 +20,6 @@ bool ZAM_error = false;
 
 bool is_ZAM_compilable(const ProfileFunc* pf, const char** reason)
 	{
-	if ( pf->NumLambdas() > 0 )
-		{
-		if ( reason )
-			*reason = "use of lambda";
-		return false;
-		}
-
-	if ( pf->NumWhenStmts() > 0 )
-		{
-		if ( reason )
-			*reason = "use of \"when\"";
-		return false;
-		}
-
 	auto b = pf->ProfiledBody();
 	auto is_hook = pf->ProfiledFunc()->Flavor() == FUNC_FLAVOR_HOOK;
 	if ( b && ! script_is_valid(b, is_hook) )

src/script_opt/ZAM/Vars.cc

@@ -24,6 +24,17 @@ bool ZAMCompiler::IsUnused(const IDPtr& id, const Stmt* where) const
 	return ! usage || ! usage->HasID(id.get());
 	}
 
+bool ZAMCompiler::IsCapture(const ID* id) const
+	{
+	const auto& c = pf->CapturesOffsets();
+	return c.find(id) != c.end();
+	}
+
+int ZAMCompiler::CaptureOffset(const ID* id) const
+	{
+	return pf->CapturesOffsets().find(id)->second;
+	}
+
 void ZAMCompiler::LoadParam(const ID* id)
 	{
 	if ( id->IsType() )
@@ -64,7 +75,24 @@ const ZAMStmt ZAMCompiler::LoadGlobal(const ID* id)
 
 	// We use the id_val for reporting used-but-not-set errors.
 	z.aux = new ZInstAux(0);
-	z.aux->id_val = id;
+	z.aux->id_val = {NewRef{}, const_cast<ID*>(id)};
+
+	return AddInst(z, true);
+	}
+
+const ZAMStmt ZAMCompiler::LoadCapture(const ID* id)
+	{
+	ZOp op;
+
+	if ( ZVal::IsManagedType(id->GetType()) )
+		op = OP_LOAD_MANAGED_CAPTURE_VV;
+	else
+		op = OP_LOAD_CAPTURE_VV;
+
+	auto slot = RawSlot(id);
+
+	ZInstI z(op, slot, CaptureOffset(id));
+	z.op_type = OP_VV_I2;
 
 	return AddInst(z, true);
 	}
@@ -83,6 +111,9 @@ int ZAMCompiler::FrameSlot(const ID* id)
 	if ( id->IsGlobal() )
 		(void)LoadGlobal(id);
 
+	else if ( IsCapture(id) )
+		(void)LoadCapture(id);
+
 	return slot;
 	}
 
@@ -103,6 +134,13 @@ int ZAMCompiler::Frame1Slot(const ID* id, ZAMOp1Flavor fl)
 	if ( id->IsGlobal() )
 		pending_global_store = global_id_to_info[id];
 
+	else if ( IsCapture(id) )
+		pending_capture_store = CaptureOffset(id);
+
+	// Make sure we don't think we're storing to both a global and
+	// a capture.
+	ASSERT(pending_global_store == -1 || pending_capture_store == -1);
+
 	return slot;
 	}
 

src/script_opt/ZAM/ZBody.cc

@@ -31,6 +31,11 @@ namespace zeek::detail
 
 using std::vector;
 
+// Thrown when a call inside a "when" delays.
+class ZAMDelayedCallException : public InterpreterException
+	{
+	};
+
 static bool did_init = false;
 
 // Count of how often each type of ZOP executed, and how much CPU it
@@ -188,10 +193,10 @@ ZBody::~ZBody()
 
 void ZBody::SetInsts(vector<ZInst*>& _insts)
 	{
-	ninst = _insts.size();
+	end_pc = _insts.size();
-	auto insts_copy = new ZInst[ninst];
+	auto insts_copy = new ZInst[end_pc];
 
-	for ( auto i = 0U; i < ninst; ++i )
+	for ( auto i = 0U; i < end_pc; ++i )
 		insts_copy[i] = *_insts[i];
 
 	insts = insts_copy;
@@ -201,10 +206,10 @@ void ZBody::SetInsts(vector<ZInst*>& _insts)
 
 void ZBody::SetInsts(vector<ZInstI*>& instsI)
 	{
-	ninst = instsI.size();
+	end_pc = instsI.size();
-	auto insts_copy = new ZInst[ninst];
+	auto insts_copy = new ZInst[end_pc];
 
-	for ( auto i = 0U; i < ninst; ++i )
+	for ( auto i = 0U; i < end_pc; ++i )
 		{
 		auto& iI = *instsI[i];
 		insts_copy[i] = iI;
@@ -223,7 +228,7 @@ void ZBody::InitProfile()
 		{
 		inst_count = new vector<int>;
 		inst_CPU = new vector<double>;
-		for ( auto i = 0U; i < ninst; ++i )
+		for ( auto i = 0U; i < end_pc; ++i )
 			{
 			inst_count->push_back(0);
 			inst_CPU->push_back(0.0);
@@ -240,7 +245,7 @@ ValPtr ZBody::Exec(Frame* f, StmtFlowType& flow)
 	double t = analysis_options.profile_ZAM ? util::curr_CPU_time() : 0.0;
 #endif
 
-	auto val = DoExec(f, 0, flow);
+	auto val = DoExec(f, flow);
 
 #ifdef DEBUG
 	if ( analysis_options.profile_ZAM )
@@ -250,10 +255,9 @@ ValPtr ZBody::Exec(Frame* f, StmtFlowType& flow)
 	return val;
 	}
 
-ValPtr ZBody::DoExec(Frame* f, int start_pc, StmtFlowType& flow)
+ValPtr ZBody::DoExec(Frame* f, StmtFlowType& flow)
 	{
-	int pc = start_pc;
+	int pc = 0;
-	const int end_pc = ninst;
 
 	// Return value, or nil if none.
 	const ZVal* ret_u = nullptr;
@@ -288,6 +292,9 @@ ValPtr ZBody::DoExec(Frame* f, int start_pc, StmtFlowType& flow)
 
 	flow = FLOW_RETURN; // can be over-written by a Hook-Break
 
+	// Clear any leftover error state.
+	ZAM_error = false;
+
 	while ( pc < end_pc && ! ZAM_error )
 		{
 		auto& z = insts[pc];
@@ -364,9 +371,6 @@ ValPtr ZBody::DoExec(Frame* f, int start_pc, StmtFlowType& flow)
 		delete[] frame;
 		}
 
-	// Clear any error state.
-	ZAM_error = false;
-
 	return result;
 	}
 
@@ -444,7 +448,7 @@ void ZBody::Dump() const
 
 	printf("Final code:\n");
 
-	for ( unsigned i = 0; i < ninst; ++i )
+	for ( unsigned i = 0; i < end_pc; ++i )
 		{
 		auto& inst = insts[i];
 		printf("%d: ", i);
@@ -467,25 +471,6 @@ TraversalCode ZBody::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-ValPtr ZAMResumption::Exec(Frame* f, StmtFlowType& flow)
-	{
-	return am->DoExec(f, xfer_pc, flow);
-	}
-
-void ZAMResumption::StmtDescribe(ODesc* d) const
-	{
-	d->Add("<resumption of compiled code>");
-	}
-
-TraversalCode ZAMResumption::Traverse(TraversalCallback* cb) const
-	{
-	TraversalCode tc = cb->PreStmt(this);
-	HANDLE_TC_STMT_PRE(tc);
-
-	tc = cb->PostStmt(this);
-	HANDLE_TC_STMT_POST(tc);
-	}
-
 // Unary vector operation of v1 <vec-op> v2.
 static void vec_exec(ZOp op, TypePtr t, VectorVal*& v1, const VectorVal* v2, const ZInst& z)
 	{

src/script_opt/ZAM/ZBody.h

@@ -52,12 +52,10 @@ public:
 	void ProfileExecution() const;
 
 protected:
-	friend class ZAMResumption;
-
 	// Initializes profiling information, if needed.
 	void InitProfile();
 
-	ValPtr DoExec(Frame* f, int start_pc, StmtFlowType& flow);
+	ValPtr DoExec(Frame* f, StmtFlowType& flow);
 
 	// Run-time checking for "any" type being consistent with
 	// expected typed.  Returns true if the type match is okay.
@@ -73,7 +71,7 @@ private:
 	const char* func_name = nullptr;
 
 	const ZInst* insts = nullptr;
-	unsigned int ninst = 0;
+	unsigned int end_pc = 0;
 
 	FrameReMap frame_denizens;
 	int frame_size;
@@ -117,29 +115,6 @@ private:
 	CaseMaps<std::string> str_cases;
 	};
 
-// This is a statement that resumes execution into a code block in a
-// ZBody.  Used for deferred execution for "when" statements.
-
-class ZAMResumption : public Stmt
-	{
-public:
-	ZAMResumption(ZBody* _am, int _xfer_pc) : Stmt(STMT_ZAM_RESUMPTION), am(_am), xfer_pc(_xfer_pc)
-		{
-		}
-
-	ValPtr Exec(Frame* f, StmtFlowType& flow) override;
-
-	StmtPtr Duplicate() override { return {NewRef{}, this}; }
-
-	void StmtDescribe(ODesc* d) const override;
-
-protected:
-	TraversalCode Traverse(TraversalCallback* cb) const override;
-
-	ZBody* am;
-	int xfer_pc = 0;
-	};
-
 // Prints the execution profile.
 extern void report_ZOP_profile();
 

src/script_opt/ZAM/ZInst.cc

@@ -383,16 +383,20 @@ bool ZInstI::IsDirectAssignment() const
 
 	switch ( op )
 		{
-		case OP_ASSIGN_VV_N:
 		case OP_ASSIGN_VV_A:
+		case OP_ASSIGN_VV_D:
+		case OP_ASSIGN_VV_F:
+		case OP_ASSIGN_VV_I:
+		case OP_ASSIGN_VV_L:
+		case OP_ASSIGN_VV_N:
 		case OP_ASSIGN_VV_O:
 		case OP_ASSIGN_VV_P:
 		case OP_ASSIGN_VV_R:
 		case OP_ASSIGN_VV_S:
-		case OP_ASSIGN_VV_F:
 		case OP_ASSIGN_VV_T:
+		case OP_ASSIGN_VV_U:
 		case OP_ASSIGN_VV_V:
-		case OP_ASSIGN_VV_L:
+		case OP_ASSIGN_VV_a:
 		case OP_ASSIGN_VV_f:
 		case OP_ASSIGN_VV_t:
 		case OP_ASSIGN_VV:
@@ -403,6 +407,21 @@ bool ZInstI::IsDirectAssignment() const
 		}
 	}
 
+bool ZInstI::HasCaptures() const
+	{
+	switch ( op )
+		{
+		case OP_LAMBDA_VV:
+		case OP_WHEN_V:
+		case OP_WHEN_TIMEOUT_VV:
+		case OP_WHEN_TIMEOUT_VC:
+			return true;
+
+		default:
+			return false;
+		}
+	}
+
 bool ZInstI::HasSideEffects() const
 	{
 	return op_side_effects[op];
@@ -647,6 +666,11 @@ bool ZInstI::IsGlobalLoad() const
 	return global_ops.count(op) > 0;
 	}
 
+bool ZInstI::IsCaptureLoad() const
+	{
+	return op == OP_LOAD_CAPTURE_VV || op == OP_LOAD_MANAGED_CAPTURE_VV;
+	}
+
 void ZInstI::InitConst(const ConstExpr* ce)
 	{
 	auto v = ce->ValuePtr();

src/script_opt/ZAM/ZInst.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include "zeek/Desc.h"
+#include "zeek/Func.h"
 #include "zeek/script_opt/ZAM/BuiltInSupport.h"
 #include "zeek/script_opt/ZAM/Support.h"
 #include "zeek/script_opt/ZAM/ZOp.h"
@@ -220,6 +221,9 @@ public:
 	// True if this instruction is of the form "v1 = v2".
 	bool IsDirectAssignment() const;
 
+	// True if this instruction includes captures in its aux slots.
+	bool HasCaptures() const;
+
 	// True if this instruction has side effects when executed, so
 	// should not be pruned even if it has a dead assignment.
 	bool HasSideEffects() const;
@@ -247,9 +251,17 @@ public:
 	// the ZAM frame.
 	bool IsGlobalLoad() const;
 
+	// True if the instruction corresponds to loading a capture into
+	// the ZAM frame.
+	bool IsCaptureLoad() const;
+
+	// True if the instruction does not correspond to a load from the
+	// ZAM frame.
+	bool IsNonLocalLoad() const { return IsGlobalLoad() || IsCaptureLoad(); }
+
 	// True if the instruction corresponds to some sort of load,
-	// either from the interpreter frame or of a global.
+	// either from the interpreter frame or of a global/capture.
-	bool IsLoad() const { return op_type == OP_VV_FRAME || IsGlobalLoad(); }
+	bool IsLoad() const { return op_type == OP_VV_FRAME || IsNonLocalLoad(); }
 
 	// True if the instruction corresponds to storing a global.
 	bool IsGlobalStore() const { return op == OP_STORE_GLOBAL_V; }
@@ -317,6 +329,7 @@ public:
 			slots = ints = new int[n];
 			constants = new ValPtr[n];
 			types = new TypePtr[n];
+			is_managed = new bool[n];
 			}
 		}
 
@@ -325,6 +338,7 @@ public:
 		delete[] ints;
 		delete[] constants;
 		delete[] types;
+		delete[] is_managed;
 		delete[] cat_args;
 		}
 
@@ -384,6 +398,7 @@ public:
 		ints[i] = slot;
 		constants[i] = nullptr;
 		types[i] = t;
+		is_managed[i] = t ? ZVal::IsManagedType(t) : false;
 		}
 
 	// Same but for constants.
@@ -392,6 +407,7 @@ public:
 		ints[i] = -1;
 		constants[i] = c;
 		types[i] = nullptr;
+		is_managed[i] = false;
 		}
 
 	// Member variables.  We could add accessors for manipulating
@@ -404,24 +420,37 @@ public:
 	// if not, it's nil).
 	//
 	// We track associated types, too, enabling us to use
-	// ZVal::ToVal to convert frame slots or constants to ValPtr's.
+	// ZVal::ToVal to convert frame slots or constants to ValPtr's;
+	// and, as a performance optimization, whether those types
+	// indicate the slot needs to be managed.
 
 	int n; // size of arrays
 	int* slots = nullptr; // either nil or points to ints
 	int* ints = nullptr;
 	ValPtr* constants = nullptr;
 	TypePtr* types = nullptr;
+	bool* is_managed = nullptr;
+
+	// Ingredients associated with lambdas ...
+	ScriptFuncPtr primary_func;
+
+	// ... and its name.
+	std::string lambda_name;
+
+	// For "when" statements.  Needs to be non-const so we can
+	// Instantiate() it as needed.
+	WhenInfo* wi;
 
 	// A parallel array for the cat() built-in replacement.
 	std::unique_ptr<CatArg>* cat_args = nullptr;
 
 	// Used for accessing function names.
-	const ID* id_val = nullptr;
+	IDPtr id_val = nullptr;
 
-	// Whether the instruction can lead to globals changing.
+	// Whether the instruction can lead to globals/captures changing.
-	// Currently only needed by the optimizer, but convenient
+	// Currently only needed by the optimizer, but convenient to
-	// to store here.
+	// store here.
-	bool can_change_globals = false;
+	bool can_change_non_locals = false;
 
 	// The following is only used for OP_CONSTRUCT_KNOWN_RECORD_V,
 	// to map elements in slots/constants/types to record field offsets.

testing/btest/Baseline.zam/bifs.from_json-11/.stderr

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in <...>/from_json.zeek, line 8: required field Foo$hello is missing in JSON (from_json({"t":null}, <internal>::#0))
+error in <...>/from_json.zeek, line 8: required field Foo$hello is missing in JSON (from_json({"t":null}, <internal>::#0, from_json_default_key_mapper))
-error in <...>/from_json.zeek, line 9: required field Foo$hello is null in JSON (from_json({"hello": null, "t": true}, <internal>::#2))
+error in <...>/from_json.zeek, line 9: required field Foo$hello is null in JSON (from_json({"hello": null, "t": true}, <internal>::#2, from_json_default_key_mapper))

testing/btest/Baseline.zam/bifs.from_json-13/.stderr

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in <...>/from_json.zeek, line 7: required field Foo$id_field is missing in JSON (from_json({"id-field": "Hello!"}, <internal>::#0, from_json_default_key_mapper))

testing/btest/Baseline.zam/bifs.from_json-13/.stdout

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[v=<uninitialized>, valid=F]
+[v=[id_field=Hello!], valid=T]

testing/btest/Baseline.zam/bifs.from_json-14/.stderr

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline.zam/bifs.from_json-14/.stdout

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+T
+[ietf_mud_mud=[mud_version=1, mud_url=https://lighting.example.com/lightbulb2000, last_update=2019-01-28T11:20:51+01:00, cache_validity=48, is_supported=T, systeminfo=The BMS Example Light Bulb, from_device_policy=[access_lists=[access_list=[[name=mud-76100-v6fr]]]], to_device_policy=[access_lists=[access_list=[[name=mud-76100-v6to]]]]], ietf_access_control_list_acls=[acl=[[name=mud-76100-v6to, _type=ipv6-acl-type, aces=[ace=[[name=cl0-todev, matches=[ipv6=[ietf_acldns_dst_dnsname=<uninitialized>, ietf_mud_direction_initiated=<uninitialized>, destination_port=<uninitialized>], tcp=[ietf_acldns_dst_dnsname=<uninitialized>, ietf_mud_direction_initiated=from-device, destination_port=<uninitialized>]]]]]], [name=mud-76100-v6fr, _type=ipv6-acl-type, aces=[ace=[[name=cl0-frdev, matches=[ipv6=[ietf_acldns_dst_dnsname=test.example.com, ietf_mud_direction_initiated=<uninitialized>, destination_port=<uninitialized>], tcp=[ietf_acldns_dst_dnsname=<uninitialized>, ietf_mud_direction_initiated=from-device, destination_port=[operator=eq, _port=443]]]]]]]]]]

testing/btest/Baseline.zam/bifs.from_json-2/.stderr

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in <...>/from_json.zeek, line 4: from_json() requires a type argument (from_json([], 10))
+error in <...>/from_json.zeek, line 4: from_json() requires a type argument (from_json([], 10, from_json_default_key_mapper))

testing/btest/Baseline.zam/bifs.from_json-3/.stderr

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in <...>/from_json.zeek, line 4: JSON parse error: Missing a closing quotation mark in string. Offset: 5 (from_json({"hel, <internal>::#0))
+error in <...>/from_json.zeek, line 4: JSON parse error: Missing a closing quotation mark in string. Offset: 5 (from_json({"hel, <internal>::#0, from_json_default_key_mapper))

testing/btest/Baseline.zam/bifs.from_json-4/.stderr

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in <...>/from_json.zeek, line 9: cannot convert JSON type 'array' to Zeek type 'bool' (from_json([], <internal>::#0))
+error in <...>/from_json.zeek, line 9: cannot convert JSON type 'array' to Zeek type 'bool' (from_json([], <internal>::#0, from_json_default_key_mapper))
-error in <...>/from_json.zeek, line 10: cannot convert JSON type 'string' to Zeek type 'bool' (from_json({"a": "hello"}, <internal>::#2))
+error in <...>/from_json.zeek, line 10: cannot convert JSON type 'string' to Zeek type 'bool' (from_json({"a": "hello"}, <internal>::#2, from_json_default_key_mapper))

testing/btest/Baseline.zam/bifs.from_json-5/.stderr

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in <...>/from_json.zeek, line 4: tables are not supported (from_json([], <internal>::#0))
+error in <...>/from_json.zeek, line 4: tables are not supported (from_json([], <internal>::#0, from_json_default_key_mapper))

testing/btest/Baseline.zam/bifs.from_json-6/.stderr

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in <...>/from_json.zeek, line 5: wrong port format, must be <...>/(tcp|udp|icmp|unknown)/ (from_json("80", <internal>::#0))
+error in <...>/from_json.zeek, line 5: wrong port format, must be <...>/(tcp|udp|icmp|unknown)/ (from_json("80", <internal>::#0, from_json_default_key_mapper))

testing/btest/Baseline.zam/bifs.from_json-7/.stderr

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in <...>/from_json.zeek, line 5: index type doesn't match (from_json([[1, false], [2]], <internal>::#0))
+error in <...>/from_json.zeek, line 5: index type doesn't match (from_json([[1, false], [2]], <internal>::#0, from_json_default_key_mapper))
-error in <...>/from_json.zeek, line 6: cannot convert JSON type 'number' to Zeek type 'bool' (from_json([[1, false], [2, 1]], <internal>::#2))
+error in <...>/from_json.zeek, line 6: cannot convert JSON type 'number' to Zeek type 'bool' (from_json([[1, false], [2, 1]], <internal>::#2, from_json_default_key_mapper))

testing/btest/Baseline.zam/bifs.from_json-8/.stderr

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 error: error compiling pattern /^?(.|\n)*(([[:print:]]{-}[[:alnum:]]foo))/
-error in <...>/from_json.zeek, line 5: error compiling pattern (from_json("/([[:print:]]{-}[[:alnum:]]foo)/", <internal>::#0))
+error in <...>/from_json.zeek, line 5: error compiling pattern (from_json("/([[:print:]]{-}[[:alnum:]]foo)/", <internal>::#0, from_json_default_key_mapper))

testing/btest/Baseline.zam/bifs.from_json-9/.stderr

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in <...>/from_json.zeek, line 7: 'Yellow' is not a valid enum for 'Color'. (from_json("Yellow", <internal>::#0))
+error in <...>/from_json.zeek, line 7: 'Yellow' is not a valid enum for 'Color'. (from_json("Yellow", <internal>::#0, from_json_default_key_mapper))

testing/btest/Baseline.zam/broker.store.create-failure/zeek.err

@@ -2,8 +2,8 @@
 error in <...>/store.zeek, line 780: Failed to attach master store backend_failure: (Broker::__create_master(../fail, Broker::SQLITE, Broker::options.2))
 error in <...>/store.zeek, line 780: Could not create Broker master store '../fail' (Broker::__create_master(../fail, Broker::SQLITE, Broker::options.2))
 error in <...>/create-failure.zeek, line 49: invalid Broker store handle (Broker::keys(s) and broker::store::{})
-error in <...>/store.zeek, line 794: invalid Broker store handle (Broker::__close(Broker::h) and broker::store::{})
+error in <...>/store.zeek, line 794: invalid Broker store handle (Broker::__close(Broker::h.12) and broker::store::{})
-error in <...>/store.zeek, line 794: invalid Broker store handle (Broker::__close(Broker::h) and broker::store::{})
+error in <...>/store.zeek, line 794: invalid Broker store handle (Broker::__close(Broker::h.21) and broker::store::{})
 error in <...>/create-failure.zeek, line 49: invalid Broker store handle (Broker::keys(s) and broker::store::{})
 error in <...>/create-failure.zeek, line 49: invalid Broker store handle (Broker::keys(s) and broker::store::{})
 error in <...>/create-failure.zeek, line 49: invalid Broker store handle (Broker::keys(s) and broker::store::{})

testing/btest/Baseline.zam/core.when-interpreter-exceptions/zeek.output

@@ -0,0 +1,15 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+### NOTE: This file has been sorted with diff-sort.
+[f(F)]
+[f(T)]
+[zeek_init()]
+f() done, no exception, T
+g() done, no exception, T
+received termination signal
+runtime error in <...>/when-interpreter-exceptions.zeek, line 103: field value missing: $notset
+runtime error in <...>/when-interpreter-exceptions.zeek, line 47: field value missing: $notset
+runtime error in <...>/when-interpreter-exceptions.zeek, line 72: field value missing: $notset
+runtime error in <...>/when-interpreter-exceptions.zeek, line 91: field value missing: $notset
+timeout
+timeout g(), F
+timeout g(), T

testing/btest/Baseline.zam/language.when-capture-errors/out

@@ -0,0 +1,8 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in <...>/when-capture-errors.zeek, lines 19-22: orig1 is used inside "when" statement but not captured (when (0 < g) { print orig1})
+error in <...>/when-capture-errors.zeek, lines 25-28: orig3 is used inside "when" statement but not captured (when (0 < g || orig3) { print g})
+error in <...>/when-capture-errors.zeek, lines 34-38: orig1 is used inside "when" statement but not captured (when (0 < g) { print g} timeout 1.0 sec { print orig1})
+error in <...>/when-capture-errors.zeek, lines 66-70: orig2 is used inside "when" statement but not captured (when [orig1](0 < g) { print orig1} timeout 1.0 sec { print orig2})
+error in <...>/when-capture-errors.zeek, lines 76-80: orig3 is captured but not used inside "when" statement (when [orig1, orig2, orig3](0 < g) { print orig1} timeout 1.0 sec { print orig2})
+error in <...>/when-capture-errors.zeek, line 83: no such local identifier: l1
+error in <...>/when-capture-errors.zeek, line 89: no such local identifier: l2

testing/btest/Baseline/language.uninitialized-local/out

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/uninitialized-local.zeek, line 16: value used but not set (my_string)
+expression error in <...>/uninitialized-local.zeek, line 19: value used but not set (my_string)

testing/btest/Baseline/language.when-capture-errors/out

@@ -2,7 +2,7 @@
 error in <...>/when-capture-errors.zeek, lines 19-22: orig1 is used inside "when" statement but not captured (when (0 < g) { print orig1})
 error in <...>/when-capture-errors.zeek, lines 25-28: orig3 is used inside "when" statement but not captured (when (0 < g || orig3) { print g})
 error in <...>/when-capture-errors.zeek, lines 34-38: orig1 is used inside "when" statement but not captured (when (0 < g) { print g} timeout 1.0 sec { print orig1})
-error in <...>/when-capture-errors.zeek, lines 66-70: orig2 is used inside "when" statement but not captured (when (0 < g) { print orig1} timeout 1.0 sec { print orig2})
+error in <...>/when-capture-errors.zeek, lines 66-70: orig2 is used inside "when" statement but not captured (when [orig1](0 < g) { print orig1} timeout 1.0 sec { print orig2})
-error in <...>/when-capture-errors.zeek, lines 76-80: orig3 is captured but not used inside "when" statement (when (0 < g) { print orig1} timeout 1.0 sec { print orig2})
+error in <...>/when-capture-errors.zeek, lines 76-80: orig3 is captured but not used inside "when" statement (when [orig1, orig2, orig3](0 < g) { print orig1} timeout 1.0 sec { print orig2})
 error in <...>/when-capture-errors.zeek, line 83: no such local identifier: l1
 error in <...>/when-capture-errors.zeek, line 89: no such local identifier: l2

testing/btest/Baseline/language.when-unitialized-rhs/out

@@ -1,6 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/when-unitialized-rhs.zeek, line 12: value used but not set (crashMe)
+expression error in <...>/when-unitialized-rhs.zeek, line 13: value used but not set (crashMe)
-expression error in <...>/when-unitialized-rhs.zeek, line 17: value used but not set (x)
+expression error in <...>/when-unitialized-rhs.zeek, line 18: value used but not set (x)
 1
 2
 3

testing/btest/language/uninitialized-local.ZAM.zeek

@@ -0,0 +1,23 @@
+# A version of uninitialized-local.zeek suitable for ZAM's behavior.
+# @TEST-REQUIRES: test "${ZEEK_ZAM}" == "1"
+# @TEST-EXEC-FAIL: zeek -b %INPUT >out 2>&1
+
+event testit() &priority=10
+	{
+	local my_count: count = 10;
+	}
+
+event testit()
+	{
+	# my_string's value occupies same Frame offset as my_count's from above
+	# handler, but execution of this handler body should still "initialize"
+	# it to a null value instead of referring to a left-over value of my_count.
+	local my_string: string;
+	local my_vector: vector of string;
+	my_vector[0] = my_string;
+	}
+
+event zeek_init()
+	{
+	event testit();
+	}

testing/btest/language/uninitialized-local.zeek

@@ -1,3 +1,6 @@
+# Not suitable for ZAM since it spots the error at compile time and exits
+# with an error status code.
+# @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1"
 # @TEST-EXEC: zeek -b %INPUT >out 2>&1
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
 

testing/btest/language/when-unitialized-rhs.zeek

@@ -1,5 +1,6 @@
-# C++-compiled scripts don't check for missing functions - not clear it's
+# Compiled scripts don't fully check for missing functions - not clear it's
 # worth fixing that.
+# @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1"
 # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
 # @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT >out 2>&1
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out

testing/external/commit-hash.zeek-testing

@@ -1 +1 @@
-d59caff708b41db11fa0cbfe0b1f95b46c3e700e
+828845c99306c6d5d6811fa42987de5b16f530b9

testing/external/commit-hash.zeek-testing-private

@@ -1 +1 @@
-7162c907aa25e155ea841710ef30b65afb578c3f
+b121bfe4d869f1f5e334505b970cd456558ef6a1