Merge remote-tracking branch 'origin/topic/vern/cpp-incr'

* origin/topic/vern/cpp-incr:
  tweak to keep clang-format happy
  minor note regarding improving performance of C++-generated code
  factoring of generating C++ initializations, no semantic changes
  restored support for incremental compilation of scripts to C++
  fixes for -O gen-standalone-C++
  new ZEEK_FILE_ONLY and ZEEK_FUNC_ONLY environment variables for debugging script optimization - replaces ZEEK_ONLY
  fix for compiling record constructors to C++
  fixes for compiling vector operations to C++
  fixed for profiling missing some profile elements
  removing vestigial methods
  minor efficiency tweak for ZAM record construction

CHANGES

@@ -1,3 +1,21 @@
+4.2.0-dev.468 | 2021-12-14 11:34:47 -0700
+
+  * factoring of generating C++ initializations, no semantic changes (Vern Paxson, Corelight)
+
+  * restored support for incremental compilation of scripts to C++ (Vern Paxson, Corelight)
+
+  * fixes for -O gen-standalone-C++ (Vern Paxson, Corelight)
+
+  * new ZEEK_FILE_ONLY and ZEEK_FUNC_ONLY environment variables for debugging script optimization - replaces ZEEK_ONLY (Vern Paxson, Corelight)
+
+  * fix for compiling record constructors to C++ (Vern Paxson, Corelight)
+
+  * fixes for compiling vector operations to C++ (Vern Paxson, Corelight)
+
+  * fixed for profiling missing some profile elements (Vern Paxson, Corelight)
+
+  * minor efficiency tweak for ZAM record construction (Vern Paxson, Corelight)
+
 4.2.0-dev.456 | 2021-12-14 09:23:47 -0700
 
   * GH-1860: Add double_to_int() bif (Tim Wojtulewicz, Corelight)

VERSION

@@ -1 +1 @@
-4.2.0-dev.456
+4.2.0-dev.468

src/script_opt/CPP/CPP-load.bif

@@ -39,8 +39,5 @@ function load_CPP%(h: count%): bool
 	// compiled scripts.
 	detail::standalone_activations.push_back(cb->second);
 
-	// Proceed with activation.
-	(*detail::CPP_init_hook)();
-
 	return zeek::val_mgr->True();
 	%}

src/script_opt/CPP/Compile.h

@@ -301,6 +301,20 @@ private:
 	// in support of run-time initialization of various dynamic values.
 	void GenEpilog();
 
+	// Generate the main method of the CPPDynStmt class, doing dynamic
+	// dispatch for function invocation.
+	void GenCPPDynStmt();
+
+	// Generate a function to load BiFs.
+	void GenLoadBiFs();
+
+	// Generate the main initialization function, which finalizes
+	// the run-time environment.
+	void GenFinishInit();
+
+	// Generate the function that registers compiled script bodies.
+	void GenRegisterBodies();
+
 	// True if the given function (plus body and profile) is one
 	// that should be compiled.  If non-nil, sets reason to the
 	// the reason why, if there's a fundamental problem.  If however
@@ -585,10 +599,6 @@ private:
 	// function name, and maps to the C++ name.
 	std::unordered_map<std::string, std::string> compiled_simple_funcs;
 
-	// Maps those to their associated files - used to make add-C++ body
-	// hashes distinct.
-	std::unordered_map<std::string, std::string> cf_locs;
-
 	// Maps function bodies to the names we use for them.
 	std::unordered_map<const Stmt*, std::string> body_names;
 
@@ -943,10 +953,6 @@ private:
 	// the set of attributes.
 	void BuildAttrs(const AttributesPtr& attrs, std::string& attr_tags, std::string& attr_vals);
 
-	// Generates code to create the given attributes at run-time.
-	void GenAttrs(const AttributesPtr& attrs);
-	std::string GenAttrExpr(const ExprPtr& e);
-
 	// Returns a string representation of the name associated with
 	// different attribute tags (e.g., "ATTR_DEFAULT").
 	static const char* AttrName(AttrTag t);

src/script_opt/CPP/DeclFunc.cc

@@ -99,9 +99,6 @@ void CPPCompile::CreateFunction(const FuncTypePtr& ft, const ProfileFunc* pf, co
 		// for lambdas that don't take any arguments, but that
 		// seems potentially more confusing than beneficial.
 		compiled_funcs.emplace(fname);
-
-		auto loc_f = script_specific_filename(body);
-		cf_locs[fname] = loc_f;
 		}
 
 	auto h = pf->HashVal();

src/script_opt/CPP/Driver.cc

@@ -197,10 +197,10 @@ void CPPCompile::Compile(bool report_uncompilable)
 
 void CPPCompile::GenProlog()
 	{
-	if ( addl_tag == 0 )
+	if ( addl_tag <= 1 )
-		{
+		// This is either a compilation via gen-C++, or
+		// one using add-C++ and an empty CPP-gen.cc file.
 		Emit("#include \"zeek/script_opt/CPP/Runtime.h\"\n");
-		}
 
 	Emit("namespace zeek::detail { //\n");
 	Emit("namespace CPP_%s { // %s\n", Fmt(addl_tag), working_dir);
@@ -292,13 +292,6 @@ void CPPCompile::RegisterCompiledBody(const string& f)
 
 	events = string("{") + events + "}";
 
-	if ( addl_tag > 0 )
-		// Hash in the location associated with this compilation
-		// pass, to get a final hash that avoids conflicts with
-		// identical-but-in-a-different-context function bodies
-		// when compiling potentially conflicting additional code.
-		h = merge_p_hashes(h, p_hash(cf_locs[f]));
-
 	auto fi = func_index.find(f);
 	ASSERT(fi != func_index.end());
 	auto type_signature = casting_index[fi->second];
@@ -313,9 +306,65 @@ void CPPCompile::GenEpilog()
 		GenInitExpr(ii.second);
 
 	NL();
-	Emit("ValPtr CPPDynStmt::Exec(Frame* f, StmtFlowType& flow)");
+	GenCPPDynStmt();
+
+	NL();
+	for ( auto gi : all_global_info )
+		gi->GenerateInitializers(this);
+
+	NL();
+	InitializeEnumMappings();
+
+	NL();
+	InitializeFieldMappings();
+
+	NL();
+	InitializeBiFs();
+
+	NL();
+	indices_mgr.Generate(this);
+
+	NL();
+	InitializeStrings();
+
+	NL();
+	InitializeHashes();
+
+	NL();
+	InitializeConsts();
+
+	NL();
+	GenLoadBiFs();
+
+	NL();
+	GenFinishInit();
+
+	NL();
+	GenRegisterBodies();
+
+	NL();
+	Emit("void init__CPP()");
 	StartBlock();
+	Emit("register_bodies__CPP();");
+	EndBlock();
+
+	if ( standalone )
+		GenStandaloneActivation();
+
+	GenInitHook();
+
+	Emit("} // %s\n\n", scope_prefix(addl_tag));
+	Emit("} // zeek::detail");
+	}
+
+void CPPCompile::GenCPPDynStmt()
+	{
+	Emit("ValPtr CPPDynStmt::Exec(Frame* f, StmtFlowType& flow)");
+
+	StartBlock();
+
 	Emit("flow = FLOW_RETURN;");
+
 	Emit("switch ( type_signature )");
 	StartBlock();
 	for ( auto i = 0U; i < func_casting_glue.size(); ++i )
@@ -350,41 +399,29 @@ void CPPCompile::GenEpilog()
 
 	EndBlock();
 	EndBlock();
+	}
 
-	NL();
+void CPPCompile::GenLoadBiFs()
+	{
+	Emit("void load_BiFs__CPP()");
+	StartBlock();
+	Emit("for ( auto& b : CPP__BiF_lookups__ )");
+	Emit("\tb.ResolveBiF();");
+	EndBlock();
+	}
 
-	for ( auto gi : all_global_info )
+void CPPCompile::GenFinishInit()
-		gi->GenerateInitializers(this);
+	{
-
+	Emit("void finish_init__CPP()");
-	if ( standalone )
-		GenStandaloneActivation();
-
-	NL();
-	InitializeEnumMappings();
-
-	NL();
-	InitializeFieldMappings();
-
-	NL();
-	InitializeBiFs();
-
-	NL();
-	indices_mgr.Generate(this);
-
-	NL();
-	InitializeStrings();
-
-	NL();
-	InitializeHashes();
-
-	NL();
-	InitializeConsts();
-
-	NL();
-	Emit("void init__CPP()");
 
 	StartBlock();
 
+	Emit("static bool did_init = false;");
+	Emit("if ( did_init )");
+	Emit("\treturn;");
+	Emit("did_init = true;");
+
+	NL();
 	Emit("std::vector<std::vector<int>> InitIndices;");
 	Emit("generate_indices_set(CPP__Indices__init, InitIndices);");
 
@@ -401,13 +438,6 @@ void CPPCompile::GenEpilog()
 	Emit("InitsManager im(CPP__ConstVals, InitConsts, InitIndices, CPP__Strings, CPP__Hashes, "
 	     "CPP__Type__, CPP__Attributes__, CPP__Attr__, CPP__CallExpr__);");
 
-	NL();
-	Emit("for ( auto& b : CPP__bodies_to_register )");
-	StartBlock();
-	Emit("auto f = make_intrusive<CPPDynStmt>(b.func_name.c_str(), b.func, b.type_signature);");
-	Emit("register_body__CPP(f, b.priority, b.h, b.events);");
-	EndBlock();
-
 	NL();
 	int max_cohort = 0;
 	for ( auto gi : all_global_info )
@@ -418,10 +448,6 @@ void CPPCompile::GenEpilog()
 			if ( gi->CohortSize(c) > 0 )
 				Emit("%s.InitializeCohort(&im, %s);", gi->InitializersName(), Fmt(c));
 
-	NL();
-	Emit("for ( auto& b : CPP__BiF_lookups__ )");
-	Emit("\tb.ResolveBiF();");
-
 	// Populate mappings for dynamic offsets.
 	NL();
 	Emit("for ( auto& em : CPP__enum_mappings__ )");
@@ -430,15 +456,24 @@ void CPPCompile::GenEpilog()
 	Emit("for ( auto& fm : CPP__field_mappings__ )");
 	Emit("\tfield_mapping.push_back(fm.ComputeOffset(&im));");
 
-	if ( standalone )
+	NL();
-		Emit("standalone_init__CPP();");
+	Emit("load_BiFs__CPP();");
 
-	EndBlock(true);
+	EndBlock();
+	}
 
-	GenInitHook();
+void CPPCompile::GenRegisterBodies()
+	{
+	Emit("void register_bodies__CPP()");
+	StartBlock();
 
-	Emit("} // %s\n\n", scope_prefix(addl_tag));
+	Emit("for ( auto& b : CPP__bodies_to_register )");
-	Emit("} // zeek::detail");
+	StartBlock();
+	Emit("auto f = make_intrusive<CPPDynStmt>(b.func_name.c_str(), b.func, b.type_signature);");
+	Emit("register_body__CPP(f, b.priority, b.h, b.events, finish_init__CPP);");
+	EndBlock();
+
+	EndBlock();
 	}
 
 bool CPPCompile::IsCompilable(const FuncInfo& func, const char** reason)

src/script_opt/CPP/Exprs.cc

@@ -580,13 +580,13 @@ string CPPCompile::GenArithCoerceExpr(const Expr* e, GenType gt)
 	if ( same_type(t, op->GetType()) )
 		return GenExpr(op, gt);
 
-	bool is_vec = t->Tag() == TYPE_VECTOR;
+	if ( t->Tag() == TYPE_VECTOR )
-
+		return string("vector_coerce_to__CPP(") + GenExpr(op, GEN_NATIVE) + ", " + GenTypeName(t) +
-	auto coerce_t = is_vec ? t->Yield() : t;
+		       ")";
 
 	string cast_name;
 
-	switch ( coerce_t->InternalType() )
+	switch ( t->InternalType() )
 		{
 		case TYPE_INTERNAL_INT:
 			cast_name = "bro_int_t";
@@ -602,10 +602,6 @@ string CPPCompile::GenArithCoerceExpr(const Expr* e, GenType gt)
 			reporter->InternalError("bad type in arithmetic coercion");
 		}
 
-	if ( is_vec )
-		return string("vec_coerce_to_") + cast_name + "__CPP(" + GenExpr(op, GEN_NATIVE) + ", " +
-		       GenTypeName(t) + ")";
-
 	return NativeToGT(cast_name + "(" + GenExpr(op, GEN_NATIVE) + ")", t, gt);
 	}
 
@@ -1115,10 +1111,12 @@ string CPPCompile::GenListAssign(const ExprPtr& lhs, const ExprPtr& rhs)
 
 string CPPCompile::GenVectorOp(const Expr* e, string op, const char* vec_op)
 	{
-	auto gen = string("vec_op_") + vec_op + "__CPP(" + op + ")";
+	auto t = e->GetType();
+	auto gen_t = GenTypeName(t);
+	auto gen = string("vec_op_") + vec_op + "__CPP(" + op + ", " + gen_t + ")";
 
-	if ( ! IsArithmetic(e->GetType()->Yield()->Tag()) )
+	if ( ! IsArithmetic(t->Yield()->Tag()) )
-		gen = string("vector_coerce_to__CPP(") + gen + ", " + GenTypeName(e->GetType()) + ")";
+		gen = string("vector_coerce_to__CPP(") + gen + ", " + gen_t + ")";
 
 	return gen;
 	}

src/script_opt/CPP/Func.h

@@ -105,6 +105,7 @@ struct CompiledScript
 	CPPStmtPtr body;
 	int priority;
 	std::vector<std::string> events;
+	void (*finish_init_func)();
 	};
 
 // Maps hashes to compiled information.

src/script_opt/CPP/Inits.cc

@@ -213,17 +213,11 @@ void CPPCompile::GenStandaloneActivation()
 	{
 	NL();
 
-#if 0
 	Emit("void standalone_activation__CPP()");
 	StartBlock();
-	for ( auto& a : activations )
-		Emit(a);
-	EndBlock();
-#endif
 
+	Emit("finish_init__CPP();");
 	NL();
-	Emit("void standalone_init__CPP()");
-	StartBlock();
 
 	// For events and hooks, we need to add each compiled body *unless*
 	// it's already there (which could be the case if the standalone
@@ -283,10 +277,14 @@ void CPPCompile::GenStandaloneActivation()
 		     GenTypeName(ft), hashes);
 		}
 
+	EndBlock();
+
 	NL();
+	Emit("void standalone_init__CPP()");
+	StartBlock();
+	Emit("init__CPP();");
 	Emit("CPP_activation_funcs.push_back(standalone_activation__CPP);");
 	Emit("CPP_activation_hook = activate__CPPs;");
-
 	EndBlock();
 	}
 
@@ -300,15 +298,6 @@ void CPPCompile::GenLoad()
 
 	Emit("register_scripts__CPP(%s, standalone_init__CPP);", Fmt(total_hash));
 
-	// Spit out the placeholder script, and any associated module
-	// definitions.
-	for ( const auto& m : module_names )
-		if ( m != "GLOBAL" )
-			printf("module %s;\n", m.c_str());
-
-	if ( module_names.size() > 0 )
-		printf("module GLOBAL;\n\n");
-
 	printf("global init_CPP_%llu = load_CPP(%llu);\n", total_hash, total_hash);
 	}
 

src/script_opt/CPP/InitsInfo.cc

@@ -239,14 +239,14 @@ void FuncConstInfo::InitializerVals(std::vector<std::string>& ivs) const
 	{
 	auto f = fv->AsFunc();
 	const auto& fn = f->Name();
+	const auto& bodies = f->GetBodies();
 
 	ivs.emplace_back(Fmt(type));
 	ivs.emplace_back(Fmt(c->TrackString(fn)));
+	ivs.emplace_back(to_string(bodies.size()));
 
 	if ( ! c->NotFullyCompilable(fn) )
 		{
-		const auto& bodies = f->GetBodies();
-
 		for ( const auto& b : bodies )
 			{
 			auto h = c->BodyHash(b.stmts.get());

src/script_opt/CPP/README.md

@@ -185,7 +185,7 @@ particularly difficult to fix.
 
 * A number of steps could be taken to increase the performance of
 the optimized code.  These include:
-	1. Switching the generated code to use the new ZVal-related interfaces.
+	1. Switching the generated code to use the new ZVal-related interfaces, including for vector operations.
 	2. Directly calling BiFs rather than using the `Invoke()` method to do so.  This relates to the broader question of switching BiFs to be based on a notion of "inlined C++" code in Zeek functions, rather than using the standalone `bifcl` BiF compiler.
 	3. Switching the Event Engine over to queuing events with `ZVal` arguments rather than `ValPtr` arguments.
 	4. Making the compiler aware of certain BiFs that can be directly inlined (e.g., `network_time()`), a technique employed effectively by the ZAM compiler.

src/script_opt/CPP/RuntimeInitSupport.cc

@@ -62,9 +62,10 @@ void register_type__CPP(TypePtr t, const string& name)
 	id->MakeType();
 	}
 
-void register_body__CPP(CPPStmtPtr body, int priority, p_hash_type hash, vector<string> events)
+void register_body__CPP(CPPStmtPtr body, int priority, p_hash_type hash, vector<string> events,
+                        void (*finish_init)())
 	{
-	compiled_scripts[hash] = {move(body), priority, move(events)};
+	compiled_scripts[hash] = {move(body), priority, move(events), finish_init};
 	}
 
 void register_lambda__CPP(CPPStmtPtr body, p_hash_type hash, const char* name, TypePtr t,
@@ -89,7 +90,7 @@ void register_lambda__CPP(CPPStmtPtr body, p_hash_type hash, const char* name, T
 	if ( ! has_captures )
 		// Note, no support for lambdas that themselves refer
 		// to events.
-		register_body__CPP(body, 0, hash, {});
+		register_body__CPP(body, 0, hash, {}, nullptr);
 	}
 
 void register_scripts__CPP(p_hash_type h, void (*callback)())
@@ -190,11 +191,12 @@ Func* lookup_bif__CPP(const char* bif)
 	return b ? b->GetVal()->AsFunc() : nullptr;
 	}
 
-FuncValPtr lookup_func__CPP(string name, vector<p_hash_type> hashes, const TypePtr& t)
+FuncValPtr lookup_func__CPP(string name, int num_bodies, vector<p_hash_type> hashes,
+                            const TypePtr& t)
 	{
 	auto ft = cast_intrusive<FuncType>(t);
 
-	if ( hashes.empty() )
+	if ( static_cast<int>(hashes.size()) < num_bodies )
 		{
 		// This happens for functions that have at least one
 		// uncompilable body.

src/script_opt/CPP/RuntimeInitSupport.h

@@ -37,7 +37,7 @@ extern void register_type__CPP(TypePtr t, const std::string& name);
 // relevant for the function body, which should be registered if the
 // function body is going to be used.
 extern void register_body__CPP(CPPStmtPtr body, int priority, p_hash_type hash,
-                               std::vector<std::string> events);
+                               std::vector<std::string> events, void (*finish_init)());
 
 // Registers a lambda body as associated with the given hash.  Includes
 // the name of the lambda (so it can be made available as a quasi-global
@@ -67,7 +67,8 @@ extern Func* lookup_bif__CPP(const char* bif);
 // returns an associated FuncVal.  It's a fatal error for the hash
 // not to exist, because this function should only be called by compiled
 // code that has ensured its existence.
-extern FuncValPtr lookup_func__CPP(std::string name, std::vector<p_hash_type> h, const TypePtr& t);
+extern FuncValPtr lookup_func__CPP(std::string name, int num_bodies, std::vector<p_hash_type> h,
+                                   const TypePtr& t);
 
 // Returns the record corresponding to the given name, as long as the
 // name is indeed a record type.  Otherwise (or if the name is nil)

src/script_opt/CPP/RuntimeInits.cc

@@ -147,13 +147,14 @@ void CPP_IndexedInits<T>::Generate(InitsManager* im, std::vector<FuncValPtr>& iv
 	auto iv_end = init_vals.end();
 	auto t = *(iv_it++);
 	auto fn = im->Strings(*(iv_it++));
+	auto num_bodies = *(iv_it++);
 
 	std::vector<p_hash_type> hashes;
 
 	while ( iv_it != iv_end )
 		hashes.push_back(im->Hashes(*(iv_it++)));
 
-	ivec[offset] = lookup_func__CPP(fn, hashes, im->Types(t));
+	ivec[offset] = lookup_func__CPP(fn, num_bodies, hashes, im->Types(t));
 	}
 
 template <class T>

src/script_opt/CPP/RuntimeOps.cc

@@ -215,22 +215,43 @@ TableValPtr table_constructor__CPP(vector<ValPtr> indices, vector<ValPtr> vals,
 
 RecordValPtr record_constructor__CPP(vector<ValPtr> vals, RecordTypePtr t)
 	{
-	auto rv = make_intrusive<RecordVal>(move(t));
+	auto rv = make_intrusive<RecordVal>(t);
 	auto n = vals.size();
 
 	for ( auto i = 0u; i < n; ++i )
-		rv->Assign(i, vals[i]);
+		{
+		auto& v_i = vals[i];
+
+		if ( v_i && v_i->GetType()->Tag() == TYPE_VECTOR && v_i->AsVectorVal()->Size() == 0 )
+			{
+			const auto& t_ind = t->GetFieldType(i);
+			v_i->AsVectorVal()->Concretize(t_ind->Yield());
+			}
+
+		rv->Assign(i, v_i);
+		}
 
 	return rv;
 	}
 
 RecordValPtr record_constructor_map__CPP(vector<ValPtr> vals, vector<int> map, RecordTypePtr t)
 	{
-	auto rv = make_intrusive<RecordVal>(move(t));
+	auto rv = make_intrusive<RecordVal>(t);
 	auto n = vals.size();
 
 	for ( auto i = 0u; i < n; ++i )
-		rv->Assign(map[i], vals[i]);
+		{
+		auto& v_i = vals[i];
+		auto ind = map[i];
+
+		if ( v_i && v_i->GetType()->Tag() == TYPE_VECTOR && v_i->AsVectorVal()->Size() == 0 )
+			{
+			const auto& t_ind = t->GetFieldType(ind);
+			v_i->AsVectorVal()->Concretize(t_ind->Yield());
+			}
+
+		rv->Assign(ind, v_i);
+		}
 
 	return rv;
 	}

src/script_opt/CPP/RuntimeVec.cc

@@ -47,8 +47,9 @@ static VectorTypePtr base_vector_type__CPP(const VectorTypePtr& vt)
 #define VEC_OP1_KERNEL(accessor, type, op)                                                         \
 	for ( unsigned int i = 0; i < v->Size(); ++i )                                                 \
 		{                                                                                          \
-		auto v_i = v->ValAt(i)->accessor();                                                        \
+		auto v_i = v->ValAt(i);                                                                    \
-		v_result->Assign(i, make_intrusive<type>(op v_i));                                         \
+		if ( v_i )                                                                                 \
+			v_result->Assign(i, make_intrusive<type>(op v_i->accessor()));                         \
 		}
 
 // A macro (since it's beyond my templating skillz to deal with the
@@ -58,9 +59,9 @@ static VectorTypePtr base_vector_type__CPP(const VectorTypePtr& vt)
 // is "double".  It needs to be optional because C++ will (rightfully)
 // complain about applying certain C++ unary operations to doubles.
 #define VEC_OP1(name, op, double_kernel)                                                           \
-	VectorValPtr vec_op_##name##__CPP(const VectorValPtr& v)                                       \
+	VectorValPtr vec_op_##name##__CPP(const VectorValPtr& v, const TypePtr& t)                     \
 		{                                                                                          \
-		auto vt = base_vector_type__CPP(v->GetType<VectorType>());                                 \
+		auto vt = base_vector_type__CPP(cast_intrusive<VectorType>(t));                            \
 		auto v_result = make_intrusive<VectorVal>(vt);                                             \
                                                                                                    \
 		switch ( vt->Yield()->InternalType() )                                                     \
@@ -105,9 +106,10 @@ VEC_OP1(comp, ~, )
 #define VEC_OP2_KERNEL(accessor, type, op)                                                         \
 	for ( unsigned int i = 0; i < v1->Size(); ++i )                                                \
 		{                                                                                          \
-		auto v1_i = v1->ValAt(i)->accessor();                                                      \
+		auto v1_i = v1->ValAt(i);                                                                  \
-		auto v2_i = v2->ValAt(i)->accessor();                                                      \
+		auto v2_i = v2->ValAt(i);                                                                  \
-		v_result->Assign(i, make_intrusive<type>(v1_i op v2_i));                                   \
+		if ( v1_i && v2_i )                                                                        \
+			v_result->Assign(i, make_intrusive<type>(v1_i->accessor() op v2_i->accessor()));       \
 		}
 
 // Analogous to VEC_OP1, instantiates a function for a given binary operation,
@@ -387,27 +389,42 @@ VectorValPtr vector_coerce_to__CPP(const VectorValPtr& v, const TypePtr& targ)
 	for ( unsigned int i = 0; i < n; ++i )
 		{
 		ValPtr v_i = v->ValAt(i);
+		if ( ! v_i )
+			continue;
+
 		ValPtr r_i;
 		switch ( ytag )
 			{
 			case TYPE_BOOL:
-				r_i = val_mgr->Bool(v_i->AsBool());
+				r_i = val_mgr->Bool(v_i->CoerceToInt() != 0);
+				break;
+
+			case TYPE_INT:
+				r_i = val_mgr->Int(v_i->CoerceToInt());
+				break;
+
+			case TYPE_COUNT:
+				r_i = val_mgr->Count(v_i->CoerceToUnsigned());
 				break;
 
 			case TYPE_ENUM:
-				r_i = yt->AsEnumType()->GetEnumVal(v_i->AsInt());
+				r_i = yt->AsEnumType()->GetEnumVal(v_i->CoerceToInt());
 				break;
 
 			case TYPE_PORT:
-				r_i = make_intrusive<PortVal>(v_i->AsCount());
+				r_i = make_intrusive<PortVal>(v_i->CoerceToUnsigned());
+				break;
+
+			case TYPE_DOUBLE:
+				r_i = make_intrusive<DoubleVal>(v_i->CoerceToDouble());
 				break;
 
 			case TYPE_INTERVAL:
-				r_i = make_intrusive<IntervalVal>(v_i->AsDouble());
+				r_i = make_intrusive<IntervalVal>(v_i->CoerceToDouble());
 				break;
 
 			case TYPE_TIME:
-				r_i = make_intrusive<TimeVal>(v_i->AsDouble());
+				r_i = make_intrusive<TimeVal>(v_i->CoerceToDouble());
 				break;
 
 			default:
@@ -420,40 +437,10 @@ VectorValPtr vector_coerce_to__CPP(const VectorValPtr& v, const TypePtr& targ)
 	return v_result;
 	}
 
-VectorValPtr vec_coerce_to_bro_int_t__CPP(const VectorValPtr& v, TypePtr targ)
+VectorValPtr vec_scalar_mixed_with_vector()
 	{
-	auto res_t = cast_intrusive<VectorType>(targ);
+	reporter->CPPRuntimeError("vector-mixed-with-scalar operations not supported");
-	auto v_result = make_intrusive<VectorVal>(move(res_t));
+	return nullptr;
-	auto n = v->Size();
-
-	for ( unsigned int i = 0; i < n; ++i )
-		v_result->Assign(i, val_mgr->Int(v->IntAt(i)));
-
-	return v_result;
-	}
-
-VectorValPtr vec_coerce_to_bro_uint_t__CPP(const VectorValPtr& v, TypePtr targ)
-	{
-	auto res_t = cast_intrusive<VectorType>(targ);
-	auto v_result = make_intrusive<VectorVal>(move(res_t));
-	auto n = v->Size();
-
-	for ( unsigned int i = 0; i < n; ++i )
-		v_result->Assign(i, val_mgr->Count(v->CountAt(i)));
-
-	return v_result;
-	}
-
-VectorValPtr vec_coerce_to_double__CPP(const VectorValPtr& v, TypePtr targ)
-	{
-	auto res_t = cast_intrusive<VectorType>(targ);
-	auto v_result = make_intrusive<VectorVal>(move(res_t));
-	auto n = v->Size();
-
-	for ( unsigned int i = 0; i < n; ++i )
-		v_result->Assign(i, make_intrusive<DoubleVal>(v->DoubleAt(i)));
-
-	return v_result;
 	}
 
 	} // namespace zeek::detail

src/script_opt/CPP/RuntimeVec.h

@@ -22,10 +22,10 @@ inline ValPtr vector_append__CPP(VectorValPtr v1, ValPtr v2)
 	}
 
 // Unary vector operations.
-extern VectorValPtr vec_op_pos__CPP(const VectorValPtr& v);
+extern VectorValPtr vec_op_pos__CPP(const VectorValPtr& v, const TypePtr& t);
-extern VectorValPtr vec_op_neg__CPP(const VectorValPtr& v);
+extern VectorValPtr vec_op_neg__CPP(const VectorValPtr& v, const TypePtr& t);
-extern VectorValPtr vec_op_not__CPP(const VectorValPtr& v);
+extern VectorValPtr vec_op_not__CPP(const VectorValPtr& v, const TypePtr& t);
-extern VectorValPtr vec_op_comp__CPP(const VectorValPtr& v);
+extern VectorValPtr vec_op_comp__CPP(const VectorValPtr& v, const TypePtr& t);
 
 // Binary vector operations.
 extern VectorValPtr vec_op_add__CPP(const VectorValPtr& v1, const VectorValPtr& v2);
@@ -81,8 +81,7 @@ extern VectorValPtr vec_coerce_to_bro_uint_t__CPP(const VectorValPtr& v, TypePtr
 extern VectorValPtr vec_coerce_to_double__CPP(const VectorValPtr& v, TypePtr targ);
 
 // A dummy function used during code generation for unsupported operations
-// that mix vector and scalar arguments.  We don't define it in RuntimeVec.cc
+// that mix vector and scalar arguments.
-// so that it'll generate a linking error.
 extern VectorValPtr vec_scalar_mixed_with_vector();
 
 	} // namespace zeek::detail

src/script_opt/ProfileFunc.cc

@@ -23,7 +23,13 @@ p_hash_type p_hash(const Obj* o)
 	return p_hash(d.Description());
 	}
 
-std::string script_specific_filename(const StmtPtr& body)
+// Returns a filename associated with the given function body.  Used to
+// provide distinctness to identical function bodies seen in separate,
+// potentially conflicting incremental compilations.  An example of this
+// is a function named foo() that calls bar(), for which in two different
+// compilation contexts bar() has differing semantics, even though foo()'s
+// (shallow) semantics are the same.
+static std::string script_specific_filename(const Stmt* body)
 	{
 	// The specific filename is taken from the location filename, making
 	// it absolute if necessary.
@@ -52,7 +58,9 @@ std::string script_specific_filename(const StmtPtr& body)
 	return bl_f;
 	}
 
-p_hash_type script_specific_hash(const StmtPtr& body, p_hash_type generic_hash)
+// Returns a incremental-compilation-specific hash for the given function
+// body, given its non-specific hash is "generic_hash".
+static p_hash_type script_specific_hash(const Stmt* body, p_hash_type generic_hash)
 	{
 	auto bl_f = script_specific_filename(body);
 	return merge_p_hashes(generic_hash, p_hash(bl_f));
@@ -470,16 +478,22 @@ ProfileFuncs::ProfileFuncs(std::vector<FuncInfo>& funcs, is_compilable_pred pred
 	// functions.  Recursively compute their hashes.
 	ComputeTypeHashes(main_types);
 
-	// Computing the hashes can have marked expressions (seen in
+	do
-	// record attributes) for further analysis.  Likewise, when
+		{
-	// doing the profile merges above we may have noted lambda
+		// Computing the hashes can have marked expressions (seen in
-	// expressions.  Analyze these, and iteratively any further
+		// record attributes) for further analysis.  Likewise, when
-	// expressions that that analysis uncovers.
+		// doing the profile merges above we may have noted lambda
-	DrainPendingExprs();
+		// expressions.  Analyze these, and iteratively any further
+		// expressions that that analysis uncovers.
+		DrainPendingExprs();
 
-	// We now have all the information we need to form definitive,
+		// We now have all the information we need to form definitive,
-	// deterministic hashes.
+		// deterministic hashes.
-	ComputeBodyHashes(funcs);
+		ComputeBodyHashes(funcs);
+
+		// Computing those hashes could have led to traversals that
+		// create more pending expressions to analyze.
+		} while ( ! pending_exprs.empty() );
 	}
 
 void ProfileFuncs::MergeInProfile(ProfileFunc* pf)
@@ -692,6 +706,9 @@ void ProfileFuncs::ComputeProfileHash(std::shared_ptr<ProfileFunc> pf)
 	for ( auto i : pf->AdditionalHashes() )
 		h = merge_p_hashes(h, i);
 
+	if ( ! pf->Stmts().empty() )
+		h = script_specific_hash(pf->Stmts()[0], h);
+
 	pf->SetHashVal(h);
 	}
 

src/script_opt/ProfileFunc.h

@@ -74,17 +74,6 @@ inline p_hash_type merge_p_hashes(p_hash_type h1, p_hash_type h2)
 	return h1 ^ (h2 + 0x9e3779b9 + (h1 << 6) + (h1 >> 2));
 	}
 
-// Returns a filename associated with the given function body.  Used to
-// provide distinctness to identical function bodies seen in separate,
-// potentially conflicting incremental compilations.  This is only germane
-// for allowing incremental compilation of subsets of the test suite, so
-// if we decide to forgo that capability, we can remove this.
-extern std::string script_specific_filename(const StmtPtr& body);
-
-// Returns a incremental-compilation-specific hash for the given function
-// body, given it's non-specific hash is "generic_hash".
-extern p_hash_type script_specific_hash(const StmtPtr& body, p_hash_type generic_hash);
-
 // Class for profiling the components of a single function (or expression).
 class ProfileFunc : public TraversalCallback
 	{

src/script_opt/ScriptOpt.cc

@@ -99,16 +99,16 @@ bool should_analyze(const ScriptFuncPtr& f, const StmtPtr& body)
 	if ( ofiles.empty() && ofuncs.empty() )
 		return true;
 
-	auto fn = f->Name();
+	auto fun = f->Name();
 
 	for ( auto& o : ofuncs )
-		if ( std::regex_match(fn, o) )
+		if ( std::regex_match(fun, o) )
 			return true;
 
-	fn = body->GetLocationInfo()->filename;
+	auto fin = util::detail::normalize_path(body->GetLocationInfo()->filename);
 
 	for ( auto& o : ofiles )
-		if ( std::regex_match(fn, o) )
+		if ( std::regex_match(fin, o) )
 			return true;
 
 	return false;
@@ -296,7 +296,14 @@ static void init_options()
 
 	if ( analysis_options.only_funcs.empty() )
 		{
-		auto zo = getenv("ZEEK_ONLY");
+		auto zo = getenv("ZEEK_FUNC_ONLY");
+		if ( zo )
+			add_func_analysis_pattern(analysis_options, zo);
+		}
+
+	if ( analysis_options.only_files.empty() )
+		{
+		auto zo = getenv("ZEEK_FILE_ONLY");
 		if ( zo )
 			add_file_analysis_pattern(analysis_options, zo);
 		}
@@ -343,17 +350,8 @@ static void report_CPP()
 		auto name = f.Func()->Name();
 		auto hash = f.Profile()->HashVal();
 		bool have = compiled_scripts.count(hash) > 0;
-		auto specific = "";
 
-		if ( ! have )
+		printf("script function %s (hash %llu): %s\n", name, hash, have ? "yes" : "no");
-			{
-			hash = script_specific_hash(f.Body(), hash);
-			have = compiled_scripts.count(hash) > 0;
-			if ( have )
-				specific = " - specific";
-			}
-
-		printf("script function %s (hash %llu%s): %s\n", name, hash, specific, have ? "yes" : "no");
 
 		if ( have )
 			already_reported.insert(hash);
@@ -380,12 +378,6 @@ static void use_CPP()
 		auto hash = f.Profile()->HashVal();
 		auto s = compiled_scripts.find(hash);
 
-		if ( s == compiled_scripts.end() )
-			{ // Look for script-specific body.
-			hash = script_specific_hash(f.Body(), hash);
-			s = compiled_scripts.find(hash);
-			}
-
 		if ( s != compiled_scripts.end() )
 			{
 			auto b = s->second.body;
@@ -411,6 +403,10 @@ static void use_CPP()
 				auto h = event_registry->Register(e);
 				h->SetUsed();
 				}
+
+			auto finish = s->second.finish_init_func;
+			if ( finish )
+				(*finish)();
 			}
 		}
 

src/script_opt/ZAM/Ops.in

@@ -1064,7 +1064,7 @@ macro EvalConstructRecord(map_init, map_accessor)
 			const auto& t_ind = rt->GetFieldType(ind);
 			v_i->AsVectorVal()->Concretize(t_ind->Yield());
 			}
-		new_r->Assign(ind, aux->ToVal(frame, i));
+		new_r->Assign(ind, v_i);
 		}
 	auto& r = frame[z.v1].record_val;
 	Unref(r);