support for discarding ASTs once compiled via ZAM script optimization

auxil/gen-zam

@@ -1 +1 @@
-Subproject commit 116be85c38f2f5dabd01f42b8b53e38004df41c0
+Subproject commit cbba05dbaa58fdabe863f4e8a122ca92809b52d6

src/ID.cc

@@ -132,8 +132,8 @@ ID::ID(const char* arg_name, IDScope arg_scope, bool arg_is_export)
 
 ID::~ID()
 	{
+	ClearOptInfo();
 	delete[] name;
-	delete opt_info;
 	}
 
 std::string ID::ModuleName() const
@@ -687,6 +687,12 @@ std::vector<Func*> ID::GetOptionHandlers() const
 	return v;
 	}
 
+void ID::ClearOptInfo()
+	{
+	delete opt_info;
+	opt_info = nullptr;
+	}
+
 	} // namespace detail
 
 	} // namespace zeek

src/ID.h

@@ -151,6 +151,7 @@ public:
 	std::vector<Func*> GetOptionHandlers() const;
 
 	IDOptInfo* GetOptInfo() const { return opt_info; }
+	void ClearOptInfo();
 
 protected:
 	void EvalFunc(ExprPtr ef, ExprPtr ev);

src/script_opt/Expr.cc

@@ -1770,7 +1770,7 @@ ExprPtr AssignExpr::Reduce(Reducer* c, StmtPtr& red_stmt)
 
 		red_stmt = MergeStmts(rhs_reduce, lhs_stmt, rhs_stmt);
 
-		auto field_name = field_e->FieldName();
+		auto field_name = util::copy_string(field_e->FieldName());
 		auto field = field_e->Field();
 		auto field_assign = make_intrusive<FieldLHSAssignExpr>(lhs_e, rhs_e, field_name, field);
 
@@ -1909,7 +1909,7 @@ ExprPtr IndexExprWhen::Duplicate()
 
 ExprPtr FieldExpr::Duplicate()
 	{
-	return SetSucc(new FieldExpr(op->Duplicate(), field_name));
+	return SetSucc(new FieldExpr(op->Duplicate(), util::copy_string(field_name)));
 	}
 
 ExprPtr HasFieldExpr::Duplicate()

src/script_opt/IDOptInfo.h

@@ -148,6 +148,7 @@ public:
 	// Returns a list of the initialization expressions seen for all
 	// globals, ordered by when they were processed.
 	static auto& GetGlobalInitExprs() { return global_init_exprs; }
+	static void ClearGlobalInitExprs() { global_init_exprs.clear(); }
 
 	// Associated constant expression, if any.  This is only set
 	// for identifiers that are aliases for a constant (i.e., there

src/script_opt/ScriptOpt.cc

@@ -31,8 +31,6 @@ void (*CPP_activation_hook)() = nullptr;
 // Tracks all of the loaded functions (including event handlers and hooks).
 static std::vector<FuncInfo> funcs;
 
-static ZAMCompiler* ZAM = nullptr;
-
 static bool generating_CPP = false;
 static std::string CPP_dir; // where to generate C++ code
 
@@ -78,12 +76,10 @@ const FuncInfo* analyze_global_stmts(Stmt* stmts)
 
 	auto sc = current_scope();
 	std::vector<IDPtr> empty_inits;
-	StmtPtr stmts_p{NewRef{}, stmts};
 	global_stmts = make_intrusive<ScriptFunc>(id);
-	global_stmts->AddBody(stmts_p, empty_inits, sc->Length());
-
-	funcs.emplace_back(global_stmts, sc, stmts_p, 0);
+	global_stmts->AddBody(stmts->ThisPtr(), empty_inits, sc->Length());
 
+	funcs.emplace_back(global_stmts, sc, stmts->ThisPtr(), 0);
 	return &funcs.back();
 	}
 
@@ -253,15 +249,15 @@ static void optimize_func(ScriptFunc* f, std::shared_ptr<ProfileFunc> pf, ScopeP
 
 	if ( analysis_options.gen_ZAM_code )
 		{
-		ZAM = new ZAMCompiler(f, pf, scope, new_body, ud, rc);
+		ZAMCompiler ZAM(f, pf, scope, new_body, ud, rc);
 
-		new_body = ZAM->CompileBody();
+		new_body = ZAM.CompileBody();
 
 		if ( reporter->Errors() > 0 )
 			return;
 
 		if ( analysis_options.dump_ZAM )
-			ZAM->Dump();
+			ZAM.Dump();
 
 		f->ReplaceBody(body, new_body);
 		body = new_body;
@@ -544,6 +540,25 @@ static void analyze_scripts_for_ZAM(std::unique_ptr<ProfileFuncs>& pfs)
 	finalize_functions(funcs);
 	}
 
+void clear_script_analysis()
+	{
+	IDOptInfo::ClearGlobalInitExprs();
+
+	// We need to explicitly clear out the optimization information
+	// associated with identifiers.  They have reference loops with
+	// the parent identifier that will prevent reclamation of the
+	// identifiers (and the optimization information) upon Unref'ing
+	// when discarding the scopes and ASTs.
+	for ( auto& f : funcs )
+		for ( auto& id : f.Scope()->OrderedVars() )
+			id->ClearOptInfo();
+
+	funcs.clear();
+	non_recursive_funcs.clear();
+	lambdas.clear();
+	when_lambdas.clear();
+	}
+
 void analyze_scripts(bool no_unused_warnings)
 	{
 	init_options();

src/script_opt/ScriptOpt.h

@@ -196,6 +196,10 @@ extern bool should_analyze(const ScriptFuncPtr& f, const StmtPtr& body);
 // suppressed by the flag) and optimization.
 extern void analyze_scripts(bool no_unused_warnings);
 
+// Called when all script processing is complete and we can discard
+// unused ASTs and associated state.
+extern void clear_script_analysis();
+
 // Called when Zeek is terminating.
 extern void finish_script_execution();
 

src/script_opt/ZAM/Compile.h

@@ -60,6 +60,7 @@ class ZAMCompiler
 public:
 	ZAMCompiler(ScriptFunc* f, std::shared_ptr<ProfileFunc> pf, ScopePtr scope, StmtPtr body,
 	            std::shared_ptr<UseDefs> ud, std::shared_ptr<Reducer> rd);
+	~ZAMCompiler();
 
 	StmtPtr CompileBody();
 

src/script_opt/ZAM/Driver.cc

@@ -29,6 +29,14 @@ ZAMCompiler::ZAMCompiler(ScriptFunc* f, std::shared_ptr<ProfileFunc> _pf, ScopeP
 	Init();
 	}
 
+ZAMCompiler::~ZAMCompiler()
+	{
+	curr_stmt = nullptr;
+
+	for ( auto i : insts1 )
+		delete i;
+	}
+
 void ZAMCompiler::Init()
 	{
 	InitGlobals();

src/script_opt/ZAM/Expr.cc

@@ -1096,7 +1096,7 @@ const ZAMStmt ZAMCompiler::DoCall(const CallExpr* c, const NameExpr* n)
 
 	z.aux->can_change_non_locals = true;
 
-	z.call_expr = c;
+	z.call_expr = {NewRef{}, const_cast<CallExpr*>(c)};
 
 	if ( in_when )
 		z.SetType(n->GetType());

src/script_opt/ZAM/Ops.in

@@ -1581,7 +1581,7 @@ macro WhenCall(func)
 		throw ZAMDelayedCallException();
 	auto& lhs = frame[z.v1];
 	auto trigger = f->GetTrigger();
-	Val* v = trigger ? trigger->Lookup(z.call_expr) : nullptr;
+	Val* v = trigger ? trigger->Lookup(z.call_expr.get()) : nullptr;
 	ValPtr vp;
 	if ( v )
 		vp = {NewRef{}, v};
@@ -1593,7 +1593,7 @@ macro WhenCall(func)
 		std::vector<ValPtr> args;
 		for ( auto i = 0; i < n; ++i )
 			args.push_back(aux->ToVal(frame, i));
-		f->SetCall(z.call_expr);
+		f->SetCall(z.call_expr.get());
 		/* It's possible that this function will call another that
 		 * itself returns null because *it* is the actual blocker.
 		 * That will set ZAM_error, which we need to ignore.

src/script_opt/ZAM/ZInst.h

@@ -130,7 +130,7 @@ public:
 
 	// Interpreter call expression associated with this instruction,
 	// for error reporting and stack backtraces.
-	const CallExpr* call_expr = nullptr;
+	CallExprPtr call_expr = nullptr;
 
 	// Whether v1 represents a frame slot type for which we
 	// explicitly manage the memory.

src/zeek-setup.cc

@@ -1114,6 +1114,8 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 			g_frame_stack.pop_back();
 			}
 
+		clear_script_analysis();
+
 		if ( zeek_script_loaded )
 			{
 			// Queue events reporting loaded scripts.