Merge remote-tracking branch 'origin/topic/vern/use-defs'

- Added explicit test case of "unused assignement" warning and
  &is_used suppression during merge.

* origin/topic/vern/use-defs:
  activate &is_used
  removed unnecessary statement flagged by Coverity (thanks, Jon!)
  Adjust reference/move nitpicks in use-def/reduce code
  Adjust some whitespace in UseDefs.cc
  updates to "usage" test suite alternative now that more warnings are generated
  suppress usage warning in baseline script
  "xform" alternative baseline update, needed for recent change to master
  baseline update due to shift in number of lines in base intel framework script
  environment variable (which has precedence) not flags for baseline usage test
  splitting out "usage" test suite alternative into -u/-uu versions
  adding &is_used attribute for base scripts - not actually needed yet, but will be once optimization is added
  removing unused assignments from base scripts
  driver glue for invoking use-def construction
  classes for managing and propagating use-defs
  enhancements/changes to the Reduce class in preparation for use-defs
  some tidying with smart pointers
  flag/environment variable for dumping use-defs
  &is_used attribute to suppress set-but-not-used usage warnings
  whitespace micro-preening

CHANGES

@@ -1,4 +1,31 @@
 
+4.1.0-dev.252 | 2021-02-23 12:01:52 -0800
+
+  * "xform" btest alternative baseline update (Vern Paxson, Corelight)
+
+  * Test baselines updates related to "unused assignment" warnings (Vern Paxson, Corelight)
+
+  * USE ZEEK_USAGE_ISSUES environment variable in tests rather than -u/-uu (Vern Paxson, Corelight)
+
+    Since it has precedence over the flag arguments and don't want btest
+    alternatives to override it.
+
+  * splitting out "usage" test suite alternative into -u/-uu versions (Vern Paxson, Corelight)
+
+  * adding &is_used attribute for base scripts (Vern Paxson, Corelight)
+
+    Not actually needed yet, but will be once optimization is added.
+
+  * removing unused assignments from base scripts (Vern Paxson, Corelight)
+
+  * enhancements/changes to the Reduce class in preparation for use-defs (Vern Paxson, Corelight)
+
+  * Add "-Odump-uds" and "ZEEK_DUMP_UDS" flag/environment variable for dumping use-defs (Vern Paxson, Corelight)
+
+  * Add &is_used attribute to suppress set-but-not-used usage warnings (Vern Paxson, Corelight)
+
+  * Support for "unused assignment" dead-code detection/warning via -u/-uu analysis (Vern Paxson, Corelight)
+
 4.1.0-dev.229 | 2021-02-16 12:32:23 -0800
 
   * Update Broker submodule for performance/credit-tracking fixes (Jon Siwek, Corelight)

NEWS

@@ -34,6 +34,10 @@ New Functionality
   analysis to report instances where record fields might be used without
   having previously been set.
 
+  The ``zeek -u`` option can also identify assigned-to variables that aren't
+  subsequently used (i.e. "dead code") and issues a warning.  A new ``is_used``
+  attribute can be used situationally to suppress such warnings.
+
   The base scripts have some places where the static analysis lacks sufficient
   power to tell that values are being used safely (guaranteed to have been
   identified). In order to enable users to employ ``zeek -u`` on their own

VERSION

@@ -1 +1 @@
-4.1.0-dev.229
+4.1.0-dev.252

doc

@@ -1 +1 @@
-Subproject commit 05709b177952d62893858bb4d2b20f77033c0459
+Subproject commit 5583f544a287d7e7638c76f75b458dfe90910d1a

scripts/base/frameworks/intel/main.zeek

@@ -471,8 +471,12 @@ function insert_meta_data(item: Item): bool
 # indicates whether the item might be new for other nodes.
 function _insert(item: Item, first_dispatch: bool &default = T)
 	{
-	# Assume that the item is new by default.
+	# Assume that the item is new by default.  The &is_used attribute
-	local is_new: bool = T;
+	# is because if have_full_data isn't redef'd to F, then constant
+	# propagation will cause the definition here to be shadowed by
+	# the one below.  Alternatively, we could skip initializing here
+	# and instead do so in the "else" branch for the have_full_data test.
+	local is_new: bool = T &is_used;
 
 	# All intelligence is case insensitive at the moment.
 	local lower_indicator = to_lower(item$indicator);

scripts/base/frameworks/netcontrol/plugins/openflow.zeek

@@ -361,7 +361,6 @@ event OpenFlow::flow_mod_success(name: string, match: OpenFlow::ofp_match, flow_
 
 	local r = of_messages[id,flow_mod$command]$r;
 	local p = of_messages[id,flow_mod$command]$p;
-	local c = of_messages[id,flow_mod$command]$c;
 
 	if ( r$entity$ty == ADDRESS || r$entity$ty == MAC )
 		{

scripts/base/protocols/smb/smb1-main.zeek

@@ -23,7 +23,6 @@ event smb1_message(c: connection, hdr: SMB1::Header, is_orig: bool) &priority=5
 	local smb_state = c$smb_state;
 	local tid = hdr$tid;
 	local uid = hdr$uid;
-	local pid = hdr$pid;
 	local mid = hdr$mid;
 
 	if ( uid in smb_state$uid_map )

scripts/base/protocols/smb/smb2-main.zeek

@@ -22,9 +22,7 @@ event smb2_message(c: connection, hdr: SMB2::Header, is_orig: bool) &priority=5
 	
 	local smb_state = c$smb_state;
 	local tid = hdr$tree_id;
-	local pid = hdr$process_id;
 	local mid = hdr$message_id;
-	local sid = hdr$session_id;
 
 	if ( mid !in smb_state$pending_cmds )
 		{

src/Attr.cc

@@ -22,7 +22,7 @@ const char* attr_name(AttrTag t)
 		"&group", "&log", "&error_handler", "&type_column",
 		"(&tracked)", "&on_change", "&broker_store",
 		"&broker_allow_complex_type", "&backend", "&deprecated",
-		"&is_set",
+		"&is_assigned", "&is_used",
 	};
 
 	return attr_names[int(t)];
@@ -291,6 +291,7 @@ void Attributes::CheckAttr(Attr* a)
 	case ATTR_DEPRECATED:
 	case ATTR_REDEF:
 	case ATTR_IS_ASSIGNED:
+	case ATTR_IS_USED:
 		break;
 
 	case ATTR_OPTIONAL:

src/Attr.h

@@ -46,7 +46,8 @@ enum AttrTag {
 	ATTR_BROKER_STORE_ALLOW_COMPLEX, // for Broker store backed tables
 	ATTR_BACKEND, // for Broker store backed tables
 	ATTR_DEPRECATED,
-	ATTR_IS_ASSIGNED, // to suppress warnings
+	ATTR_IS_ASSIGNED, // to suppress usage warnings
+	ATTR_IS_USED, // to suppress usage warnings
 	NUM_ATTRS // this item should always be last
 };
 

src/CMakeLists.txt

@@ -336,6 +336,7 @@ set(MAIN_SRCS
     script_opt/ScriptOpt.cc
     script_opt/Stmt.cc
     script_opt/TempVar.cc
+    script_opt/UseDefs.cc
 
     nb_dns.c
     digest.h

src/Options.cc

@@ -157,6 +157,7 @@ static void set_analysis_option(const char* opt, Options& opts)
 	if ( util::streq(opt, "help") )
 		{
 		fprintf(stderr, "--optimize options:\n");
+		fprintf(stderr, "    dump-uds	dump use-defs to stdout; implies xform\n");
 		fprintf(stderr, "    dump-xform	dump transformed scripts to stdout; implies xform\n");
 		fprintf(stderr, "    help	print this list\n");
 		fprintf(stderr, "    inline	inline function calls\n");
@@ -167,7 +168,9 @@ static void set_analysis_option(const char* opt, Options& opts)
 
 	auto& a_o = opts.analysis_options;
 
-	if ( util::streq(opt, "dump-xform") )
+	if ( util::streq(opt, "dump-uds") )
+		a_o.activate = a_o.dump_uds = true;
+	else if ( util::streq(opt, "dump-xform") )
 		a_o.activate = a_o.dump_xform = true;
 	else if ( util::streq(opt, "inline") )
 		a_o.inliner = true;

src/parse.y

@@ -5,7 +5,7 @@
 // Switching parser table type fixes ambiguity problems.
 %define lr.type ielr
 
-%expect 135
+%expect 141
 
 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY
 %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF
@@ -28,7 +28,7 @@
 %token TOK_ATTR_BROKER_STORE_ALLOW_COMPLEX TOK_ATTR_BACKEND
 %token TOK_ATTR_PRIORITY TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER
 %token TOK_ATTR_TYPE_COLUMN TOK_ATTR_DEPRECATED
-%token TOK_ATTR_IS_ASSIGNED
+%token TOK_ATTR_IS_ASSIGNED TOK_ATTR_IS_USED
 
 %token TOK_DEBUG
 
@@ -1474,6 +1474,8 @@ attr:
 			{ $$ = new zeek::detail::Attr(zeek::detail::ATTR_REDEF); }
 	|	TOK_ATTR_IS_ASSIGNED
 			{ $$ = new zeek::detail::Attr(zeek::detail::ATTR_IS_ASSIGNED); }
+	|	TOK_ATTR_IS_USED
+			{ $$ = new zeek::detail::Attr(zeek::detail::ATTR_IS_USED); }
 	|	TOK_ATTR_ADD_FUNC '=' expr
 			{ $$ = new zeek::detail::Attr(zeek::detail::ATTR_ADD_FUNC, {zeek::AdoptRef{}, $3}); }
 	|	TOK_ATTR_DEL_FUNC '=' expr

src/scan.l

@@ -286,6 +286,7 @@ when	return TOK_WHEN;
 &log		return TOK_ATTR_LOG;
 &optional	return TOK_ATTR_OPTIONAL;
 &is_assigned	return TOK_ATTR_IS_ASSIGNED;
+&is_used	return TOK_ATTR_IS_USED;
 &priority	return TOK_ATTR_PRIORITY;
 &type_column	return TOK_ATTR_TYPE_COLUMN;
 &read_expire	return TOK_ATTR_EXPIRE_READ;

src/script_opt/GenRDs.cc

@@ -430,10 +430,7 @@ void RD_Decorate::DoLoopConfluence(const Stmt* s, const Stmt* top,
 		}
 
 	for ( const auto& post : bd->PostRDs() )
-		{
 		mgr.MergeIntoPost(body, post);
-		mgr.MergeIntoPre(s, post);
-		}
 
 	// Freshen due to mergers.
 	loop_pre = mgr.GetPreMaxRDs(top);

src/script_opt/GenRDs.h

@@ -50,7 +50,7 @@ private:
 
 class RD_Decorate : public TraversalCallback {
 public:
-	RD_Decorate(const ProfileFunc* _pf) : pf(_pf)
+	RD_Decorate(std::shared_ptr<ProfileFunc> _pf) : pf(_pf)
 		{ }
 
 	// Traverses the given function body, using the first two
@@ -133,7 +133,7 @@ private:
 
 	// Profile for the function.  Currently, all we actually need from
 	// this is the list of globals.
-	const ProfileFunc* pf;
+	std::shared_ptr<ProfileFunc> pf;
 
 	// Whether the Func is an event/hook/function.  We currently only
 	// need to know whether it's a hook, so we correctly interpret an

src/script_opt/Reduce.cc

@@ -257,11 +257,11 @@ StmtPtr Reducer::MergeStmts(const NameExpr* lhs, ExprPtr rhs, Stmt* succ_stmt)
 
 	return make_intrusive<ExprStmt>(merge_e);
 	}
-                 
+
 void Reducer::TrackExprReplacement(const Expr* orig, const Expr* e)
 	{
 	new_expr_to_orig[e] = orig;
-	}       
+	}
 
 
 const Expr* non_reduced_perp;

src/script_opt/Reduce.h

@@ -2,10 +2,10 @@
 
 #pragma once
 
-#include "zeek/IntrusivePtr.h"
 #include "zeek/Scope.h"
 #include "zeek/Expr.h"
 #include "zeek/Stmt.h"
+#include "zeek/script_opt/DefSetsMgr.h"
 
 namespace zeek::detail {
 
@@ -23,6 +23,9 @@ public:
 		return s->Reduce(this);
 		}
 
+	const DefSetsMgr* GetDefSetsMgr() const		{ return mgr; }
+	void SetDefSetsMgr(const DefSetsMgr* _mgr)	{ mgr = _mgr; }
+
 	ExprPtr GenTemporaryExpr(const TypePtr& t, ExprPtr rhs);
 
 	NameExprPtr UpdateName(NameExprPtr n);
@@ -70,25 +73,50 @@ public:
 	bool IsTemporary(const ID* id) const
 		{ return FindTemporary(id) != nullptr; }
 
-	// This is a stub for now, since it's not relevant for AST
+	bool IsConstantVar(const ID* id) const
-	// reduction by itself.  However, many of the Reduce methods
+		{ return constant_vars.find(id) != constant_vars.end(); }
-	// ultimately will call this predicate to control how they
-	// function during the second traversal used to optimize
-	// the reduced form, so we provide the hook now.
-	bool Optimizing() const	{ return false; }
 
-	// A stub for now, but ultimately a predicate that indicates whether
+	// True if the Reducer is being used in the context of a second
-	// a given reduction pass is being made to prune unused statements.
+	// pass over for AST optimization.
-	bool IsPruning() const		{ return false; }
+	bool Optimizing() const
+		{ return ! IsPruning() && mgr != nullptr; }
 
-	// A stub for now, ultimately a predicate that returns true if
+	// A predicate that indicates whether a given reduction pass
-	// the given statement should be removed due to AST optimization.
+	// is being made to prune unused statements.
-	bool ShouldOmitStmt(const StmtPtr& s) const	{ return false; }
+	bool IsPruning() const		{ return omitted_stmts.size() > 0; }
 
-	// A stub for now, ultimately provides a replacement for the
+	// A predicate that returns true if the given statement should
-	// given statement due to AST optimization, or nil if there's
+	// be removed due to AST optimization.
-	// no replacement.
+	bool ShouldOmitStmt(const Stmt* s) const
-	StmtPtr ReplacementStmt(const StmtPtr& s) const	{ return nullptr; }
+		{ return omitted_stmts.find(s) != omitted_stmts.end(); }
+
+	// Provides a replacement for the given statement due to
+	// AST optimization, or nil if there's no replacement.
+	StmtPtr ReplacementStmt(const StmtPtr& s) const
+		{
+		auto repl = replaced_stmts.find(s.get());
+		if ( repl == replaced_stmts.end() )
+			return nullptr;
+		else
+			return repl->second;
+		}
+
+	// Tells the reducer to prune the given statement during the
+	// next reduction pass.
+	void AddStmtToOmit(const Stmt* s)	{ omitted_stmts.insert(s); }
+
+	// Tells the reducer to replace the given statement during the
+	// next reduction pass.
+	void AddStmtToReplace(const Stmt* s_old, StmtPtr s_new)
+		{ replaced_stmts[s_old] = std::move(s_new); }
+
+	// Tells the reducer that it can reclaim the storage associated
+	// with the omitted statements.
+	void ResetAlteredStmts()
+		{
+		omitted_stmts.clear();
+		replaced_stmts.clear();
+		}
 
 	// NOT YET IMPLEMENTED, SO CURRENTLY A STUB:
 	// Given the LHS and RHS of an assignment, returns true
@@ -162,6 +190,9 @@ protected:
 	std::unordered_set<ID*> new_locals;
 	std::unordered_map<const ID*, IDPtr> orig_to_new_locals;
 
+	std::unordered_set<const Stmt*> omitted_stmts;
+	std::unordered_map<const Stmt*, StmtPtr> replaced_stmts;
+
 	// Tracks whether we're inside an inline block, and if so then
 	// how deeply.
 	int inline_block_level = 0;
@@ -172,10 +203,16 @@ protected:
 	// exponentially.
 	int bifurcation_level = 0;
 
+	// Tracks which (non-temporary) variables had constant
+	// values used for constant propagation.
+	std::unordered_set<const ID*> constant_vars;
+
 	// For a new expression we've created, map it to the expression
 	// it's replacing.  This allows us to locate the RDs associated
 	// with the usage.
 	std::unordered_map<const Expr*, const Expr*> new_expr_to_orig;
+
+	const DefSetsMgr* mgr = nullptr;
 };
 
 // Used for debugging, to communicate which expression wasn't

src/script_opt/ScriptOpt.cc

@@ -8,6 +8,7 @@
 #include "zeek/script_opt/Inline.h"
 #include "zeek/script_opt/Reduce.h"
 #include "zeek/script_opt/GenRDs.h"
+#include "zeek/script_opt/UseDefs.h"
 
 
 namespace zeek::detail {
@@ -21,8 +22,9 @@ std::unordered_set<const Func*> non_recursive_funcs;
 static std::vector<FuncInfo> funcs;
 
 
-void optimize_func(ScriptFunc* f, ProfileFunc* pf, ScopePtr scope_ptr,
+void optimize_func(ScriptFunc* f, std::shared_ptr<ProfileFunc> pf,
-			StmtPtr& body, AnalyOpt& analysis_options)
+			ScopePtr scope_ptr, StmtPtr& body,
+			AnalyOpt& analysis_options)
 	{
 	if ( reporter->Errors() > 0 )
 		return;
@@ -47,7 +49,7 @@ void optimize_func(ScriptFunc* f, ProfileFunc* pf, ScopePtr scope_ptr,
 	auto scope = scope_ptr.release();
 	push_existing_scope(scope);
 
-	auto rc = std::make_unique<Reducer>(scope);
+	auto rc = std::make_shared<Reducer>(scope);
 	auto new_body = rc->Reduce(body);
 
 	if ( reporter->Errors() > 0 )
@@ -75,11 +77,25 @@ void optimize_func(ScriptFunc* f, ProfileFunc* pf, ScopePtr scope_ptr,
 
 	f->ReplaceBody(body, new_body);
 	body = new_body;
-	body->Traverse(pf);
 
+	// Profile the new body.
+	pf = std::make_shared<ProfileFunc>(false);
+	body->Traverse(pf.get());
+
+	// Compute its reaching definitions.
 	RD_Decorate reduced_rds(pf);
 	reduced_rds.TraverseFunction(f, scope, body);
 
+	rc->SetDefSetsMgr(reduced_rds.GetDefSetsMgr());
+
+	auto ud = std::make_unique<UseDefs>(body, rc);
+	ud->Analyze();
+
+	if ( analysis_options.dump_uds )
+		ud->Dump();
+
+	ud->RemoveUnused();
+
 	int new_frame_size =
 		scope->Length() + rc->NumTemps() + rc->NumNewLocals();
 
@@ -94,7 +110,7 @@ FuncInfo::FuncInfo(ScriptFuncPtr _func, ScopePtr _scope, StmtPtr _body)
 		: func(std::move(_func)), scope(std::move(_scope)), body(std::move(_body))
 	{}
 
-void FuncInfo::SetProfile(std::unique_ptr<ProfileFunc> _pf)
+void FuncInfo::SetProfile(std::shared_ptr<ProfileFunc> _pf)
 	{ pf = std::move(_pf); }
 
 void analyze_func(ScriptFuncPtr f)
@@ -115,6 +131,7 @@ void analyze_scripts()
 	if ( ! did_init )
 		{
 		check_env_opt("ZEEK_DUMP_XFORM", analysis_options.dump_xform);
+		check_env_opt("ZEEK_DUMP_UDS", analysis_options.dump_uds);
 		check_env_opt("ZEEK_INLINE", analysis_options.inliner);
 		check_env_opt("ZEEK_XFORM", analysis_options.activate);
 
@@ -142,12 +159,13 @@ void analyze_scripts()
 
 	// Now that everything's parsed and BiF's have been initialized,
 	// profile the functions.
-	std::unordered_map<const ScriptFunc*, const ProfileFunc*> func_profs;
+	std::unordered_map<const ScriptFunc*, std::shared_ptr<ProfileFunc>>
+		func_profs;
 
 	for ( auto& f : funcs )
 		{
-		f.SetProfile(std::make_unique<ProfileFunc>(true));
+		f.SetProfile(std::make_shared<ProfileFunc>(true));
-		f.Body()->Traverse(f.Profile());
+		f.Body()->Traverse(f.Profile().get());
 		func_profs[f.Func()] = f.Profile();
 		}
 

src/script_opt/ScriptOpt.h

@@ -27,6 +27,9 @@ struct AnalyOpt {
 	// them.  Always done if only_func is set.
 	bool dump_xform = false;
 
+	// If true, dump out the use-defs for each analyzed function.
+	bool dump_uds = false;
+
 	// If non-nil, then only analyze the given function/event/hook.
 	std::optional<std::string> only_func;
 
@@ -62,18 +65,18 @@ public:
 	ScriptFuncPtr FuncPtr()	{ return func; }
 	ScopePtr Scope()	{ return scope; }
 	StmtPtr Body()		{ return body; }
-	ProfileFunc* Profile()	{ return pf.get(); }
+	std::shared_ptr<ProfileFunc> Profile()	{ return pf; }
 	const std::string& SaveFile()	{ return save_file; }
 
 	void SetBody(StmtPtr new_body)	{ body = std::move(new_body); }
-	void SetProfile(std::unique_ptr<ProfileFunc> _pf);
+	void SetProfile(std::shared_ptr<ProfileFunc> _pf);
 	void SetSaveFile(std::string _sf)	{ save_file = std::move(_sf); }
 
 protected:
 	ScriptFuncPtr func;
 	ScopePtr scope;
 	StmtPtr body;
-	std::unique_ptr<ProfileFunc> pf;
+	std::shared_ptr<ProfileFunc> pf;
 
 	// If we're saving this function in a file, this is the name
 	// of the file to use.

src/script_opt/Stmt.cc

@@ -27,7 +27,7 @@ StmtPtr Stmt::Reduce(Reducer* c)
 	if ( repl )
 		return repl;
 
-	if ( c->ShouldOmitStmt(this_ptr) )
+	if ( c->ShouldOmitStmt(this) )
 		{
 		auto null = make_intrusive<NullStmt>();
 		null->SetOriginal(this_ptr);

src/script_opt/UseDefs.cc

@@ -0,0 +1,690 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/Stmt.h"
+#include "zeek/Desc.h"
+#include "zeek/Reporter.h"
+#include "zeek/script_opt/Reduce.h"
+#include "zeek/script_opt/UseDefs.h"
+#include "zeek/script_opt/ScriptOpt.h"
+
+
+namespace zeek::detail {
+
+
+void UseDefSet::Dump() const
+	{
+	for ( const auto& u : IterateOver() )
+		printf(" %s", u->Name());
+	}
+
+
+UseDefs::UseDefs(StmtPtr _body, std::shared_ptr<Reducer> _rc)
+	{
+	body = std::move(_body);
+	rc = std::move(_rc);
+	}
+
+
+void UseDefs::Analyze()
+	{
+	// Start afresh.
+	use_defs_map.clear();
+	UDs_are_copies.clear();
+	stmts.clear();
+	successor.clear();
+	successor2.clear();
+
+	(void) PropagateUDs(body, nullptr, nullptr, false);
+	}
+
+void UseDefs::RemoveUnused()
+	{
+	int iter = 0;
+	while ( RemoveUnused(++iter) )
+		{
+		body = rc->Reduce(body);
+		Analyze();
+
+		if ( reporter->Errors() > 0 )
+			break;
+		}
+	}
+
+void UseDefs::Dump()
+	{
+	for ( int i = stmts.size(); --i >= 0; )
+		{
+		const auto& s = stmts[i];
+		auto uds = FindUsage(s);
+		auto are_copies =
+			(UDs_are_copies.find(s) != UDs_are_copies.end());
+
+		printf("UDs (%s) for %s:\n", are_copies ? "copy" : "orig",
+			obj_desc(s).c_str());
+
+		if ( uds )
+			uds->Dump();
+		else
+			printf(" <none>");
+
+		printf("\n\n");
+		}
+	}
+
+bool UseDefs::RemoveUnused(int iter)
+	{
+	rc->ResetAlteredStmts();
+
+	bool did_omission = false;
+
+	for ( unsigned int i = 0; i < stmts.size(); ++i )
+		{
+		const auto& s = stmts[i];
+
+		if ( s->Tag() == STMT_INIT )
+			{
+			auto init = s->AsInitStmt();
+			const auto& inits = init->Inits();
+			std::vector<IDPtr> used_ids;
+
+			for ( auto id : inits )
+				if ( is_atomic_type(id->GetType()) ||
+				     ! CheckIfUnused(s, id.get(), false) )
+					used_ids.emplace_back(id);
+
+			if ( used_ids.size() == 0 )
+				{ // There aren't any ID's to keep.
+				rc->AddStmtToOmit(s);
+				continue;
+				}
+
+			if ( used_ids.size() < inits.size() )
+				{
+				// Need to replace the current Init statement
+				// with one that only includes the actually
+				// used identifiers.
+
+				auto new_init = make_intrusive<InitStmt>(used_ids);
+				rc->AddStmtToReplace(s, std::move(new_init));
+				}
+
+			continue;
+			}
+
+		// The only other statements we might revise or remove
+		// are assignments.
+
+		if ( s->Tag() != STMT_EXPR )
+			continue;
+
+		auto s_e = s->AsExprStmt();
+		auto e = s_e->StmtExpr();
+
+		if ( e->Tag() != EXPR_ASSIGN )
+			continue;
+
+		auto a = e->AsAssignExpr();
+		auto r = a->GetOp1();
+
+		// Because we're dealing with reduced statements, the
+		// assignment expression should be to a simple variable.
+		if ( r->Tag() != EXPR_REF )
+			reporter->InternalError("lhs ref inconsistency in UseDefs::FindUnused");
+
+		auto n = r->AsRefExprPtr()->GetOp1();
+		if ( n->Tag() != EXPR_NAME )
+			reporter->InternalError("lhs name inconsistency in UseDefs::FindUnused");
+
+		auto id = n->AsNameExpr()->Id();
+
+		auto rhs = a->GetOp2();
+		auto rt = rhs->Tag();
+
+		if ( rt == EXPR_CALL && ! rhs->IsPure() )
+			// Need to do the call for the side effects.
+			// Could prune out the assignment and just
+			// keep the call, but not clear that that's
+			// worth the complexity.
+			continue;
+
+		if ( rt == EXPR_EVENT || rt == EXPR_SCHEDULE )
+			// These always have side effects.
+			continue;
+
+		// Check for degenerate assignment "x = x".
+		bool degen = rt == EXPR_NAME && id == rhs->AsNameExpr()->Id();
+
+		if ( CheckIfUnused(s, id, iter == 1) || degen )
+			{
+			rc->AddStmtToOmit(s);
+			did_omission = true;
+			}
+		}
+
+	return did_omission;
+	}
+
+bool UseDefs::CheckIfUnused(const Stmt* s, const ID* id, bool report)
+	{
+	if ( id->IsGlobal() )
+		return false;
+
+	auto uds = FindSuccUsage(s);
+	if ( ! uds || ! uds->HasID(id) )
+		{
+		if ( report && analysis_options.usage_issues > 0 &&
+		     ! rc->IsTemporary(id) && ! rc->IsConstantVar(id) &&
+		     ! rc->IsNewLocal(id) && ! id->GetAttr(ATTR_IS_USED) )
+			reporter->Warning("%s assignment unused: %s",
+					id->Name(), obj_desc(s).c_str());
+
+		return true;
+		}
+
+	return false;
+	}
+
+UDs UseDefs::PropagateUDs(const Stmt* s, UDs succ_UDs,
+				const Stmt* succ_stmt, bool second_pass)
+	{
+	if ( ! second_pass )
+		stmts.push_back(s);
+
+	switch ( s->Tag() ) {
+	case STMT_LIST:
+		{
+		auto sl = s->AsStmtList();
+		const auto& stmts = sl->Stmts();
+
+		for ( int i = stmts.length(); --i >= 0; )
+			{
+			auto s_i = stmts[i];
+
+			const Stmt* succ;
+
+			if ( i == stmts.length() - 1 )
+				{ // Very last statement.
+				succ = succ_stmt;
+				if ( successor2.find(s) != successor2.end() )
+					successor2[s_i] = successor2[s];
+				}
+			else
+				succ = stmts[i + 1];
+
+			succ_UDs = PropagateUDs(s_i, succ_UDs, succ, second_pass);
+			}
+
+		return UseUDs(s, succ_UDs);
+		}
+
+	case STMT_CATCH_RETURN:
+		{
+		auto cr = s->AsCatchReturnStmt();
+		auto block = cr->Block();
+
+		auto uds = PropagateUDs(block.get(), succ_UDs, succ_stmt,
+					second_pass);
+
+		return UseUDs(s, uds);
+		}
+
+	case STMT_NULL:
+	case STMT_NEXT:
+	case STMT_BREAK:
+	case STMT_FALLTHROUGH:
+		// When we back up to one of these, its successor isn't
+		// actually succ_stmt (other than for STMT_NULL).  However,
+		// in the contexts in which these can occur, it doesn't
+		// actually do any harm to use the successor anyway.
+		return UseUDs(s, succ_UDs);
+
+	case STMT_PRINT:
+		return CreateExprUDs(s, s->AsPrintStmt()->ExprList(), succ_UDs);
+
+	case STMT_EVENT:
+	case STMT_CHECK_ANY_LEN:
+	case STMT_ADD:
+	case STMT_DELETE:
+	case STMT_RETURN:
+		{
+		auto e = static_cast<const ExprStmt*>(s)->StmtExpr();
+
+		if ( e )
+			return CreateExprUDs(s, e, succ_UDs);
+		else
+			return UseUDs(s, succ_UDs);
+		}
+
+	case STMT_EXPR:
+		{
+		auto e = s->AsExprStmt()->StmtExpr();
+
+		if ( e->Tag() != EXPR_ASSIGN )
+			return CreateExprUDs(s, e, succ_UDs);
+
+		// Change in use-defs as here we have a definition.
+		auto a = e->AsAssignExpr();
+		auto lhs_ref = a->GetOp1();
+
+		if ( lhs_ref->Tag() != EXPR_REF )
+			// Since we're working on reduced form ...
+			reporter->InternalError("lhs inconsistency in UseDefs::ExprUDs");
+
+		auto lhs_var = lhs_ref->GetOp1();
+		auto lhs_id = lhs_var->AsNameExpr()->Id();
+		auto lhs_UDs = RemoveID(lhs_id, succ_UDs);
+		auto rhs_UDs = ExprUDs(a->GetOp2().get());
+		auto uds = UD_Union(lhs_UDs, rhs_UDs);
+
+		if ( ! second_pass )
+			successor[s] = succ_stmt;
+
+		return CreateUDs(s, uds);
+		}
+
+	case STMT_IF:
+		{
+		auto i = s->AsIfStmt();
+		auto cond = i->StmtExpr();
+
+		auto cond_UDs = ExprUDs(cond);
+		auto true_UDs = PropagateUDs(i->TrueBranch(), succ_UDs,
+						succ_stmt, second_pass);
+		auto false_UDs = PropagateUDs(i->FalseBranch(), succ_UDs,
+						succ_stmt, second_pass);
+
+		auto uds = CreateUDs(s, UD_Union(cond_UDs, true_UDs, false_UDs));
+
+		return uds;
+		}
+
+	case STMT_INIT:
+		if ( ! second_pass )
+			successor[s] = succ_stmt;
+
+		return UseUDs(s, succ_UDs);
+
+	case STMT_WHEN:
+		// ### Once we support compiling functions with "when"
+		// statements in them, we'll need to revisit this.
+		// For now, we don't worry about it (because the current
+		// "when" body semantics of deep-copy frames has different
+		// implications than potentially switching those shallow-copy
+		// frames).
+		return UseUDs(s, succ_UDs);
+
+	case STMT_SWITCH:
+		{
+		auto sw_UDs = std::make_shared<UseDefSet>();
+
+		auto sw = s->AsSwitchStmt();
+		auto cases = sw->Cases();
+
+		for ( const auto& c : *cases )
+			{
+			auto body = c->Body();
+			auto uds = PropagateUDs(body, succ_UDs, succ_stmt,
+						second_pass);
+
+			auto exprs = c->ExprCases();
+			if ( exprs )
+				{
+				auto e_UDs = ExprUDs(exprs);
+				uds = UD_Union(uds, e_UDs);
+				}
+
+			auto type_ids = c->TypeCases();
+			if ( type_ids )
+				for ( const auto& id : *type_ids )
+					uds = RemoveID(id, uds);
+
+			FoldInUDs(sw_UDs, uds);
+			}
+
+		auto e_UDs = ExprUDs(sw->StmtExpr());
+
+		if ( sw->HasDefault() )
+			FoldInUDs(sw_UDs, e_UDs);
+		else
+			// keep successor definitions in the mix
+			FoldInUDs(sw_UDs, succ_UDs, e_UDs);
+
+		return CreateUDs(s, sw_UDs);
+		}
+
+	case STMT_FOR:
+		{
+		auto f = s->AsForStmt();
+		auto body = f->LoopBody();
+
+		// The loop body has two potential successors, itself
+		// and the successor of the entire "for" statement.
+		successor2[body] = succ_stmt;
+		auto body_UDs = PropagateUDs(body, succ_UDs, body, second_pass);
+
+		auto e = f->LoopExpr();
+		auto f_UDs = ExprUDs(e);
+		FoldInUDs(f_UDs, body_UDs);
+
+		// Confluence: loop the top UDs back around to the bottom.
+		auto bottom_UDs = UD_Union(f_UDs, succ_UDs);
+		(void) PropagateUDs(body, bottom_UDs, body, true);
+
+		auto ids = f->LoopVars();
+		for ( const auto& id : *ids )
+			RemoveUDFrom(f_UDs, id);
+
+		auto val_var = f->ValueVar();
+		if ( val_var )
+			RemoveUDFrom(f_UDs, val_var.get());
+
+		// The loop might not execute at all.
+		FoldInUDs(f_UDs, succ_UDs);
+
+		return CreateUDs(s, f_UDs);
+		}
+
+	case STMT_WHILE:
+		{
+		auto w = s->AsWhileStmt();
+		auto body = w->Body();
+		auto cond_stmt = w->CondPredStmt();
+
+		// See note above for STMT_FOR regarding propagating
+		// around the loop.
+		auto succ = cond_stmt ? cond_stmt : body;
+		successor2[body.get()] = succ_stmt;
+		auto body_UDs = PropagateUDs(body.get(), succ_UDs, succ.get(), second_pass);
+
+		const auto& cond = w->Condition();
+		auto w_UDs = UD_Union(ExprUDs(cond.get()), body_UDs);
+		FoldInUDs(w_UDs, body_UDs);
+
+		if ( cond_stmt )
+			{
+			// Create a successor for the cond_stmt
+			// that has the correct UDs associated with it.
+			const auto& c_as_s = w->ConditionAsStmt();
+			auto c_as_s_UDs = std::make_shared<UseDefSet>(w_UDs);
+			CreateUDs(c_as_s.get(), c_as_s_UDs);
+
+			w_UDs = PropagateUDs(cond_stmt, w_UDs, c_as_s,
+						second_pass);
+			}
+
+		// Confluence: loop the top UDs back around to the bottom.
+		auto bottom_UDs = UD_Union(w_UDs, succ_UDs);
+		(void) PropagateUDs(body, bottom_UDs, succ, true);
+
+		// The loop might not execute at all.
+		FoldInUDs(w_UDs, succ_UDs);
+
+		return CreateUDs(s, w_UDs);
+		}
+
+	default:
+		reporter->InternalError("non-reduced statement in use-def analysis");
+	}
+	}
+
+UDs UseDefs::FindUsage(const Stmt* s) const
+	{
+	auto s_map = use_defs_map.find(s);
+
+	if ( s_map == use_defs_map.end() )
+		reporter->InternalError("missing use-defs");
+
+	return s_map->second;
+	}
+
+UDs UseDefs::FindSuccUsage(const Stmt* s) const
+	{
+	auto succ = successor.find(s);
+	auto no_succ = (succ == successor.end() || ! succ->second);
+	auto uds = no_succ ? nullptr : FindUsage(succ->second);
+
+	auto succ2 = successor2.find(s);
+	auto no_succ2 = (succ2 == successor.end() || ! succ2->second);
+	auto uds2 = no_succ2 ? nullptr : FindUsage(succ2->second);
+
+	if ( uds && uds2 )
+		return UD_Union(uds, uds2);
+	else if ( uds )
+		return uds;
+	else
+		return uds2;
+	}
+
+UDs UseDefs::ExprUDs(const Expr* e)
+	{
+	auto uds = std::make_shared<UseDefSet>();
+
+	switch ( e->Tag() ) {
+	case EXPR_NAME:
+		AddInExprUDs(uds, e);
+		break;
+
+	case EXPR_FIELD_LHS_ASSIGN:
+		{
+		AddInExprUDs(uds, e->GetOp1().get());
+		auto rhs_UDs = ExprUDs(e->GetOp2().get());
+		uds = UD_Union(uds, rhs_UDs);
+		break;
+		}
+
+	case EXPR_INCR:
+	case EXPR_DECR:
+		AddInExprUDs(uds, e->GetOp1()->AsRefExprPtr()->GetOp1().get());
+		break;
+
+	case EXPR_RECORD_CONSTRUCTOR:
+		{
+		auto r = static_cast<const RecordConstructorExpr*>(e);
+		AddInExprUDs(uds, r->Op());
+		break;
+		}
+
+	case EXPR_CONST:
+		break;
+
+	case EXPR_LAMBDA:
+		{
+		auto l = static_cast<const LambdaExpr*>(e);
+		auto ids = l->OuterIDs();
+
+		for ( const auto& id : ids )
+			AddID(uds, id);
+		break;
+		}
+
+	case EXPR_CALL:
+		{
+		auto c = e->AsCallExpr();
+		AddInExprUDs(uds, c->Func());
+		AddInExprUDs(uds, c->Args());
+		break;
+		}
+
+	case EXPR_LIST:
+		{
+		auto l = e->AsListExpr();
+		for ( const auto& l_e : l->Exprs() )
+			AddInExprUDs(uds, l_e);
+
+		break;
+		}
+
+	default:
+		auto op1 = e->GetOp1();
+		auto op2 = e->GetOp2();
+		auto op3 = e->GetOp3();
+
+		if ( ! op1 )
+			reporter->InternalError("expression inconsistency in UseDefs::ExprUDs");
+
+		AddInExprUDs(uds, op1.get());
+		if ( op2 ) AddInExprUDs(uds, op2.get());
+		if ( op3 ) AddInExprUDs(uds, op3.get());
+
+		break;
+	}
+
+	return uds;
+	}
+
+void UseDefs::AddInExprUDs(UDs uds, const Expr* e)
+	{
+	switch ( e->Tag() ) {
+	case EXPR_NAME:
+		AddID(uds, e->AsNameExpr()->Id());
+		break;
+
+	case EXPR_LIST:
+		{
+		auto l = e->AsListExpr();
+		for ( const auto& l_e : l->Exprs() )
+			AddInExprUDs(uds, l_e);
+		}
+		break;
+
+	case EXPR_EVENT:
+		AddInExprUDs(uds, e->GetOp1().get());
+		break;
+
+	case EXPR_INCR:
+	case EXPR_DECR:
+		AddInExprUDs(uds, e->GetOp1()->AsRefExprPtr()->GetOp1().get());
+		break;
+
+	case EXPR_ASSIGN:
+		// These occur inside of table constructors.
+		AddInExprUDs(uds, e->GetOp1().get());
+		AddInExprUDs(uds, e->GetOp2().get());
+		break;
+
+	case EXPR_FIELD_ASSIGN:
+		AddInExprUDs(uds, static_cast<const FieldAssignExpr*>(e)->Op());
+		break;
+
+	case EXPR_FIELD:
+		// This happens for append-to-field.
+		AddInExprUDs(uds, e->AsFieldExpr()->Op());
+		break;
+
+	case EXPR_CONST:
+		// Nothing to do.
+		break;
+
+	default:
+		reporter->InternalError("bad tag in UseDefs::AddInExprUDs");
+		break;
+	}
+	}
+
+void UseDefs::AddID(UDs uds, const ID* id) const
+	{
+	uds->Add(id);
+	}
+
+UDs UseDefs::RemoveID(const ID* id, const UDs& uds)
+	{
+	if ( ! uds )
+		return nullptr;
+
+	UDs new_uds = std::make_shared<UseDefSet>();
+
+	new_uds->Replicate(uds);
+	new_uds->Remove(id);
+
+	return new_uds;
+	}
+
+void UseDefs::RemoveUDFrom(UDs uds, const ID* id)
+	{
+	if ( uds )
+		uds->Remove(id);
+	}
+
+void UseDefs::FoldInUDs(UDs& main_UDs, const UDs& u1, const UDs& u2)
+	{
+	auto old_main = main_UDs;
+	main_UDs = std::make_shared<UseDefSet>();
+
+	if ( old_main )
+		main_UDs->Replicate(old_main);
+
+	if ( u1 )
+		for ( auto ud : u1->IterateOver() )
+			main_UDs->Add(ud);
+
+	if ( u2 )
+		for ( auto ud : u2->IterateOver() )
+			main_UDs->Add(ud);
+	}
+
+void UseDefs::UpdateUDs(const Stmt* s, const UDs& uds)
+	{
+	auto curr_uds = FindUsage(s);
+
+	if ( ! curr_uds || UDs_are_copies.find(s) != UDs_are_copies.end() )
+		{
+		// Copy-on-write.
+		auto new_uds = std::make_shared<UseDefSet>();
+
+		if ( curr_uds )
+			new_uds->Replicate(curr_uds);
+
+		CreateUDs(s, new_uds);
+
+		curr_uds = new_uds;
+		}
+
+	if ( uds )
+		{
+		for ( auto u : uds->IterateOver() )
+			curr_uds->Add(u);
+		}
+	}
+
+UDs UseDefs::UD_Union(const UDs& u1, const UDs& u2, const UDs& u3) const
+	{
+	auto new_uds = std::make_shared<UseDefSet>();
+
+	if ( u1 )
+		new_uds->Replicate(u1);
+
+	if ( u2 )
+		for ( auto& u : u2->IterateOver() )
+			AddID(new_uds, u);
+
+	if ( u3 )
+		for ( auto& u : u3->IterateOver() )
+			AddID(new_uds, u);
+
+	return new_uds;
+	}
+
+UDs UseDefs::UseUDs(const Stmt* s, UDs uds)
+	{
+	use_defs_map[s] = uds;
+	UDs_are_copies.insert(s);
+	return uds;
+	}
+
+UDs UseDefs::CreateExprUDs(const Stmt* s, const Expr* e, const UDs& uds)
+	{
+	auto e_UDs = ExprUDs(e);
+	auto new_UDs = UD_Union(uds, e_UDs);
+
+	return CreateUDs(s, new_UDs);
+	}
+
+UDs UseDefs::CreateUDs(const Stmt* s, UDs uds)
+	{
+	use_defs_map[s] = uds;
+	UDs_are_copies.erase(s);
+	return uds;
+	}
+
+
+} // zeek::detail

src/script_opt/UseDefs.h

@@ -0,0 +1,177 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include <vector>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "zeek/Expr.h"
+
+namespace zeek::detail {
+
+// UseDefs track which variables (identifiers) are used at or subsequent
+// to a given (reduced) Statement.  They allow us to determine unproductive
+// variable assignments (both to warn the user, and to prune temporaries),
+// and also accesses to globals (so we know which ones need to be synchronized
+// across function calls).
+
+class UseDefSet;
+typedef std::shared_ptr<UseDefSet> UDs;
+
+class UseDefSet {
+public:
+	UseDefSet()	{ }
+	UseDefSet(const UDs& uds)	{ Replicate(uds); }
+
+	void Replicate(const UDs& from)
+		{ use_defs = from->use_defs; }
+
+	bool HasID(const ID* id)
+		{ return use_defs.find(id) != use_defs.end(); }
+
+	void Add(const ID* id)		{ use_defs.insert(id); }
+	void Remove(const ID* id)	{ use_defs.erase(id); }
+
+	const std::unordered_set<const ID*>& IterateOver() const
+		{ return use_defs; }
+
+	void Dump() const;
+	void DumpNL() const	{ Dump(); printf("\n"); }
+
+protected:
+	std::unordered_set<const ID*> use_defs;
+};
+
+class Reducer;
+
+class UseDefs {
+public:
+	UseDefs(StmtPtr body, std::shared_ptr<Reducer> rc);
+
+	// Does a full pass over the function body's AST.  We can wind
+	// up doing this multiple times because when we use use-defs to
+	// prune AST nodes, that in turn can change other use-defs.
+	void Analyze();
+
+	// True if we've computed use-defs for the given statement.
+	bool HasUsage(const Stmt* s) const
+		{ return use_defs_map.find(s) != use_defs_map.end(); }
+
+	// Returns the use-defs for the given statement.
+	UDs GetUsage(const Stmt* s) const	{ return FindUsage(s); }
+
+	// Removes assignments corresponding to unused temporaries.
+	// In the process, reports on locals that are assigned
+	// but never used.
+	void RemoveUnused();
+
+	void Dump();
+
+private:
+	// Makes one pass over the statements, removing assignments
+	// corresponding to temporaries (because those can be propagted).
+	// "iter" is the iteration count of how often we've done such passes,
+	// with the first pass being numbered 1.
+	//
+	// Returns true if something was removed, false if not.
+	bool RemoveUnused(int iter);
+
+	// For a given identifier defined at a given statement, returns
+	// whether it is unused.  If "report" is true, also reports
+	// this fact.
+	bool CheckIfUnused(const Stmt* s, const ID* id, bool report);
+
+	// Propagates use-defs (backwards) across statement s,
+	// given its successor's UDs.
+	//
+	// succ_stmt is the successor statement to this statement.
+	// We only care about it for potential assignment statements,
+	// (see the "successor" map below).
+	//
+	// second_pass is true when we revisit a set of statements
+	// to propagate additional UDs generated by loop confluence.
+	// If true, it prevents some redundant bookkeeping from occurring.
+	UDs PropagateUDs(const StmtPtr& s, UDs succ_UDs,
+				const StmtPtr& succ_stmt, bool second_pass)
+		{
+		return PropagateUDs(s.get(), std::move(succ_UDs), succ_stmt.get(),
+					second_pass);
+		}
+	UDs PropagateUDs(const Stmt* s, UDs succ_UDs,
+				const Stmt* succ_stmt, bool second_pass);
+
+	UDs FindUsage(const Stmt* s) const;
+	UDs FindSuccUsage(const Stmt* s) const;
+
+	// Returns a new use-def corresponding to the variables
+	// referenced in e.
+	UDs ExprUDs(const Expr* e);
+
+	// Helper method that adds in an expression's use-defs (if any)
+	// to an existing set of UDs.
+	void AddInExprUDs(UDs uds, const Expr* e);
+
+	// Add an ID into an existing set of UDs.
+	void AddID(UDs uds, const ID* id) const;
+
+	// Returns a new use-def corresonding to the given one, but
+	// with the definition of "id" removed.
+	UDs RemoveID(const ID* id, const UDs& uds);
+
+	// Similar, but updates the UDs in place.
+	void RemoveUDFrom(UDs uds, const ID* id);
+
+	// Adds in the additional UDs to the main UDs.  Always creates
+	// a new use_def and updates main_UDs to point to it.
+	void FoldInUDs(UDs& main_UDs, const UDs& u1, const UDs& u2 = nullptr);
+
+	// Adds in the given UDs to those already associated with s.
+	void UpdateUDs(const Stmt* s, const UDs& uds);
+
+	// Returns a new use-def corresponding to the union of 2 or 3 UDs.
+	UDs UD_Union(const UDs& u1, const UDs& u2,
+			const UDs& u3 = nullptr) const;
+
+	// Associate a (shallow) copy of the given UDs with the given
+	// statement.
+	UDs UseUDs(const Stmt* s, UDs uds);
+
+	// Sets the given statement's UDs to a new UD set corresponding
+	// to the union of the given UDs and those associated with the
+	// given expression.
+	UDs CreateExprUDs(const Stmt* s, const Expr* e, const UDs& uds);
+
+	// The given statement takes ownership of the given UDs.
+	UDs CreateUDs(const Stmt* s, UDs uds);
+
+
+	// Maps each statement to its associated use-def identifiers
+	// (which could be nil).
+	std::unordered_map<const Stmt*, UDs> use_defs_map;
+
+	// The following tracks statements whose use-defs are
+	// currently copies of some other statement's use-defs.
+	std::unordered_set<const Stmt*> UDs_are_copies;
+
+	// Track the statements we've processed.  This lets us dump
+	// things out in order, even though the main map is unordered.
+	std::vector<const Stmt*> stmts;
+
+	// For a given expression statement, maps it to its successor
+	// (the statement that will execute after it).  We need this
+	// because we track UDs present at the *beginning* of
+	// a statement, not at its end; those at the end are
+	// the same as those at the beginning of the successor.
+	std::unordered_map<const Stmt*, const Stmt*> successor;
+
+	// Loop bodies have two successors, and it's important to
+	// track both because sometimes a relevant UD will be present
+	// in only one or the other.
+	std::unordered_map<const Stmt*, const Stmt*> successor2;
+
+	StmtPtr body;
+	std::shared_ptr<Reducer> rc;
+};
+
+} // zeek::detail

testing/btest/Baseline.usage/language.deprecated/no-warnings.out

@@ -0,0 +1,32 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning in ./no-warnings.zeek, line 27: deprecated (ONE)
+warning in ./no-warnings.zeek, line 28: deprecated (TWO)
+warning in ./no-warnings.zeek, line 30: deprecated (GREEN)
+warning in ./no-warnings.zeek, line 31: deprecated (BLUE)
+warning in ./no-warnings.zeek, line 33: deprecated (blah)
+warning in ./no-warnings.zeek, line 37: deprecated (my_event)
+warning in ./no-warnings.zeek, line 38: deprecated (my_event)
+warning in ./no-warnings.zeek, line 39: deprecated (my_hook)
+warning in ./no-warnings.zeek, line 41: deprecated (my_record$b)
+warning in ./no-warnings.zeek, line 42: deprecated (my_record$b)
+warning in ./no-warnings.zeek, line 43: deprecated (my_record$b)
+warning in ./no-warnings.zeek, line 45: deprecated (my_record?$b)
+warning in ./no-warnings.zeek, line 46: deprecated (my_record$b)
+warning in ./no-warnings.zeek, line 49: deprecated (my_record$b)
+warning in ./no-warnings.zeek, line 52: deprecated (my_event)
+warning in ./no-warnings.zeek, line 57: deprecated (my_hook)
+warning in ./no-warnings.zeek, line 62: deprecated (blah)
+warning in ./no-warnings.zeek, line 71: deprecated (dont_use_me)
+warning in ./no-warnings.zeek, line 76: deprecated (dont_use_me_either)
+warning: mr assignment unused: mr = (coerce <internal>::#1 to record { a:count; b:string; }); ./no-warnings.zeek, line 42
+warning: mr assignment unused: mr = (coerce <internal>::#0 to record { a:count; b:string; }); ./no-warnings.zeek, line 41
+warning: l assignment unused: l = testing; ./no-warnings.zeek, line 33
+ZERO
+ONE
+TWO
+RED
+GREEN
+BLUE
+generate my_hook please
+generate my_event please
+schedule my_event please

testing/btest/Baseline.usage/language.deprecated/warnings.out

@@ -0,0 +1,32 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning in ./warnings.zeek, line 27: deprecated (ONE): one warning
+warning in ./warnings.zeek, line 28: deprecated (TWO): two warning
+warning in ./warnings.zeek, line 30: deprecated (GREEN): green warning
+warning in ./warnings.zeek, line 31: deprecated (BLUE): red warning
+warning in ./warnings.zeek, line 33: deprecated (blah): type warning
+warning in ./warnings.zeek, line 37: deprecated (my_event): event warning
+warning in ./warnings.zeek, line 38: deprecated (my_event): event warning
+warning in ./warnings.zeek, line 39: deprecated (my_hook): hook warning
+warning in ./warnings.zeek, line 41: deprecated (my_record$b): record warning
+warning in ./warnings.zeek, line 42: deprecated (my_record$b): record warning
+warning in ./warnings.zeek, line 43: deprecated (my_record$b): record warning
+warning in ./warnings.zeek, line 45: deprecated (my_record?$b): record warning
+warning in ./warnings.zeek, line 46: deprecated (my_record$b): record warning
+warning in ./warnings.zeek, line 49: deprecated (my_record$b): record warning
+warning in ./warnings.zeek, line 52: deprecated (my_event): event warning
+warning in ./warnings.zeek, line 57: deprecated (my_hook): hook warning
+warning in ./warnings.zeek, line 62: deprecated (blah): type warning
+warning in ./warnings.zeek, line 71: deprecated (dont_use_me): global function warning
+warning in ./warnings.zeek, line 76: deprecated (dont_use_me_either): function warning
+warning: mr assignment unused: mr = (coerce <internal>::#1 to record { a:count; b:string; }); ./warnings.zeek, line 42
+warning: mr assignment unused: mr = (coerce <internal>::#0 to record { a:count; b:string; }); ./warnings.zeek, line 41
+warning: l assignment unused: l = testing; ./warnings.zeek, line 33
+ZERO
+ONE
+TWO
+RED
+GREEN
+BLUE
+generate my_hook please
+generate my_event please
+schedule my_event please

testing/btest/Baseline.usage/language.type-coerce-numerics/double_convert_failure1.out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in ./double_convert_failure1.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=5.0] to myrecord) and double)

testing/btest/Baseline.usage/language.type-coerce-numerics/double_convert_failure2.out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in ./double_convert_failure2.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=-5.0] to myrecord) and double)

testing/btest/Baseline.usage/language.type-coerce-numerics/first_set.out

@@ -0,0 +1,16 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning: overflow assignment unused: overflow = (coerce <internal>::#0 to record { ii:int; cc:count; dd:double; }); ./first_set.zeek, line 46
+error in int and ./first_set.zeek, line 46: overflow promoting from unsigned/double to signed arithmetic value (int and 9223372036854775808)
+expression error in ./first_set.zeek, line 46: Failed type conversion ((coerce <internal>::#0 to record { ii:int; cc:count; dd:double; }))
+3
+int
+4
+int
+5
+int
+6
+int
+7.0
+double
+-5.0
+double

testing/btest/Baseline.usage/language.type-coerce-numerics/int_convert_failure.out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in ./int_convert_failure.zeek, line 7 and int: type clash for field "cc" ((coerce [$cc=-5] to myrecord) and int)

testing/btest/Baseline.usage/language.type-coerce-numerics/vectors.out

@@ -0,0 +1,19 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+vector of count
+vector of count
+vector of count
+[1, 2]
+[3, 4]
+[4, 6]
+vector of int
+vector of int
+vector of int
+[1, 2]
+[3, 4]
+[4, 6]
+vector of double
+vector of double
+vector of double
+[1.0, 2.0]
+[3.0, 4.0]
+[4.0, 6.0]

testing/btest/Baseline.usage/language.uninitialized-local/out

@@ -1,3 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning: my_count assignment unused: my_count = 10; <...>/uninitialized-local.zeek, line 6
 warning in <...>/uninitialized-local.zeek, line 16: possibly used without definition (my_string)
 expression error in <...>/uninitialized-local.zeek, line 16: value used but not set (my_string)

testing/btest/Baseline.usage/language.uninitialized-local2/out

@@ -1,4 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning: var_a assignment unused: var_a = foo; <...>/uninitialized-local2.zeek, line 6
 warning in <...>/uninitialized-local2.zeek, line 19: possibly used without definition (var_b)
 expression error in <...>/uninitialized-local2.zeek, line 19: value used but not set (var_b)
 var_a is, baz

testing/btest/Baseline.xform/plugins.hooks/output

@@ -562,6 +562,7 @@
 0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_ETHERNET, 34887, PacketAnalyzer::ANALYZER_MPLS)) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_ETHERNET, 34916, PacketAnalyzer::ANALYZER_PPPOE)) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_ETHERNET, 34984, PacketAnalyzer::ANALYZER_VLAN)) -> <no result>
+0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_ETHERNET, 35110, PacketAnalyzer::ANALYZER_VNTAG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_ETHERNET, 37120, PacketAnalyzer::ANALYZER_VLAN)) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_IEEE802_11, 2048, PacketAnalyzer::ANALYZER_IP)) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_IEEE802_11, 2054, PacketAnalyzer::ANALYZER_ARP)) -> <no result>
@@ -601,6 +602,9 @@
 0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VLAN, 34525, PacketAnalyzer::ANALYZER_IP)) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VLAN, 34887, PacketAnalyzer::ANALYZER_MPLS)) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VLAN, 34916, PacketAnalyzer::ANALYZER_PPPOE)) -> <no result>
+0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VNTAG, 33024, PacketAnalyzer::ANALYZER_VLAN)) -> <no result>
+0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VNTAG, 34984, PacketAnalyzer::ANALYZER_VLAN)) -> <no result>
+0.000000   MetaHookPost  CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VNTAG, 37120, PacketAnalyzer::ANALYZER_VLAN)) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketFilter::build, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketFilter::combine_filters, <frame>, (ip or not ip, and, )) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketFilter::install, <frame>, ()) -> <no result>
@@ -976,6 +980,7 @@
 0.000000   MetaHookPost  LoadFile(0, base<...>/utils, <...>/utils.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/version, <...>/version.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/vlan, <...>/vlan) -> -1
+0.000000   MetaHookPost  LoadFile(0, base<...>/vntag, <...>/vntag) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/weird, <...>/weird.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/x509, <...>/x509) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/xmpp, <...>/xmpp) -> -1
@@ -1560,6 +1565,7 @@
 0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_ETHERNET, 34887, PacketAnalyzer::ANALYZER_MPLS))
 0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_ETHERNET, 34916, PacketAnalyzer::ANALYZER_PPPOE))
 0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_ETHERNET, 34984, PacketAnalyzer::ANALYZER_VLAN))
+0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_ETHERNET, 35110, PacketAnalyzer::ANALYZER_VNTAG))
 0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_ETHERNET, 37120, PacketAnalyzer::ANALYZER_VLAN))
 0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_IEEE802_11, 2048, PacketAnalyzer::ANALYZER_IP))
 0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_IEEE802_11, 2054, PacketAnalyzer::ANALYZER_ARP))
@@ -1599,6 +1605,9 @@
 0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VLAN, 34525, PacketAnalyzer::ANALYZER_IP))
 0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VLAN, 34887, PacketAnalyzer::ANALYZER_MPLS))
 0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VLAN, 34916, PacketAnalyzer::ANALYZER_PPPOE))
+0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VNTAG, 33024, PacketAnalyzer::ANALYZER_VLAN))
+0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VNTAG, 34984, PacketAnalyzer::ANALYZER_VLAN))
+0.000000   MetaHookPre   CallFunction(PacketAnalyzer::register_packet_analyzer, <frame>, (PacketAnalyzer::ANALYZER_VNTAG, 37120, PacketAnalyzer::ANALYZER_VLAN))
 0.000000   MetaHookPre   CallFunction(PacketFilter::build, <frame>, ())
 0.000000   MetaHookPre   CallFunction(PacketFilter::combine_filters, <frame>, (ip or not ip, and, ))
 0.000000   MetaHookPre   CallFunction(PacketFilter::install, <frame>, ())
@@ -1974,6 +1983,7 @@
 0.000000   MetaHookPre   LoadFile(0, base<...>/utils, <...>/utils.zeek)
 0.000000   MetaHookPre   LoadFile(0, base<...>/version, <...>/version.zeek)
 0.000000   MetaHookPre   LoadFile(0, base<...>/vlan, <...>/vlan)
+0.000000   MetaHookPre   LoadFile(0, base<...>/vntag, <...>/vntag)
 0.000000   MetaHookPre   LoadFile(0, base<...>/weird, <...>/weird.zeek)
 0.000000   MetaHookPre   LoadFile(0, base<...>/x509, <...>/x509)
 0.000000   MetaHookPre   LoadFile(0, base<...>/xmpp, <...>/xmpp)
@@ -2557,6 +2567,7 @@
 0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 34887, PacketAnalyzer::ANALYZER_MPLS)
 0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 34916, PacketAnalyzer::ANALYZER_PPPOE)
 0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 34984, PacketAnalyzer::ANALYZER_VLAN)
+0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 35110, PacketAnalyzer::ANALYZER_VNTAG)
 0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 37120, PacketAnalyzer::ANALYZER_VLAN)
 0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IEEE802_11, 2048, PacketAnalyzer::ANALYZER_IP)
 0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IEEE802_11, 2054, PacketAnalyzer::ANALYZER_ARP)
@@ -2596,6 +2607,9 @@
 0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_VLAN, 34525, PacketAnalyzer::ANALYZER_IP)
 0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_VLAN, 34887, PacketAnalyzer::ANALYZER_MPLS)
 0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_VLAN, 34916, PacketAnalyzer::ANALYZER_PPPOE)
+0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_VNTAG, 33024, PacketAnalyzer::ANALYZER_VLAN)
+0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_VNTAG, 34984, PacketAnalyzer::ANALYZER_VLAN)
+0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_VNTAG, 37120, PacketAnalyzer::ANALYZER_VLAN)
 0.000000 | HookCallFunction PacketFilter::build()
 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, )
 0.000000 | HookCallFunction PacketFilter::install()
@@ -2983,6 +2997,7 @@
 0.000000 | HookLoadFile  base<...>/utils <...>/utils.zeek
 0.000000 | HookLoadFile  base<...>/version <...>/version.zeek
 0.000000 | HookLoadFile  base<...>/vlan <...>/vlan
+0.000000 | HookLoadFile  base<...>/vntag <...>/vntag
 0.000000 | HookLoadFile  base<...>/weird <...>/weird.zeek
 0.000000 | HookLoadFile  base<...>/x509 <...>/x509
 0.000000 | HookLoadFile  base<...>/xmpp <...>/xmpp

testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.out

@@ -1 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 error in ./double_convert_failure2.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=-5.0] to myrecord) and double)

testing/btest/Baseline/language.unused-assignement/out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning: please_warn assignment unused: please_warn = test; <...>/unused-assignement.zeek, line 7

testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output

@@ -7,6 +7,6 @@
 #open XXXX-XX-XX-XX-XX-XX
 #fields	ts	level	message	location
 #types	time	enum	string	string
-XXXXXXXXXX.XXXXXX	Reporter::INFO	Tried to remove non-existing item '192.168.1.1' (Intel::ADDR).	<...>/main.zeek, lines 566-567
+XXXXXXXXXX.XXXXXX	Reporter::INFO	Tried to remove non-existing item '192.168.1.1' (Intel::ADDR).	<...>/main.zeek, lines 570-571
 XXXXXXXXXX.XXXXXX	Reporter::INFO	received termination signal	(empty)
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/btest.cfg

@@ -65,11 +65,18 @@ BTEST_BASELINE_DIR=%(testbase)s/Baseline.inline:%(testbase)s/Baseline
 ZEEK_XFORM=1
 BTEST_BASELINE_DIR=%(testbase)s/Baseline.xform:%(testbase)s/Baseline
 
-# The following is used for testing -uu functionality (which supercedes
+# The following is used for testing -u functionality.  We set $ZEEK_XFORM,
-# -u functionality).  We set $ZEEK_XFORM, too, because the analysis is
+# too, because the analysis is done on transformed ASTs, and some tests
-# done on transformed ASTs, and some tests might be sensitive to that fact.
+# might be sensitive to that fact.  For the same reason, we first fall
-# For the same reason, we first fall back to Baseline.xform.
+# back to Baseline.xform.
 [environment-usage]
 ZEEK_XFORM=1
+ZEEK_USAGE_ISSUES=1
+BTEST_BASELINE_DIR=%(testbase)s/Baseline.usage:%(testbase)s/Baseline.xform:%(testbase)s/Baseline
+
+# The same, but for -uu (which supercedes -u functionality).  We keep them
+# separate because -uu is a lot slower.
+[environment-usage2]
+ZEEK_XFORM=1
 ZEEK_USAGE_ISSUES=2
 BTEST_BASELINE_DIR=%(testbase)s/Baseline.usage:%(testbase)s/Baseline.xform:%(testbase)s/Baseline

testing/btest/language/deprecated.zeek

@@ -39,7 +39,7 @@ event zeek_init()
 
 	local l: blah = "testing";
 
-	local ls: string = " test";
+	local ls: string = " test" &is_used;
 
 	event my_event("generate my_event please");
 	schedule 1sec { my_event("schedule my_event please") };
@@ -119,7 +119,7 @@ event zeek_init()
 
 	local l: blah = "testing";
 
-	local ls: string = " test";
+	local ls: string = " test" &is_used;
 
 	event my_event("generate my_event please");
 	schedule 1sec { my_event("schedule my_event please") };

testing/btest/language/uninitialized-local3.zeek

@@ -1,4 +1,4 @@
-# @TEST-EXEC: zeek -uu -b %INPUT >out 2>&1
+# @TEST-EXEC: ZEEK_USAGE_ISSUES=2 zeek -b %INPUT >out 2>&1
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
 
 type r: record {

testing/btest/language/unused-assignement.zeek

@@ -0,0 +1,9 @@
+# @TEST-EXEC: ZEEK_USAGE_ISSUES=1 zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-DOC: The "-u" flags should warn about unused assignments and &is_used suppresses it.
+
+event zeek_init()
+	{
+	local please_warn: string = "test";
+	local please_no_warning: string = "test" &is_used;
+	}