Script optimization maintenance and updates:

maintenance fixes for variadic run-time checks, '_' placeholder identifier
  "-O allow-cond" permits compiling scripts to C++ when influenced by @if conditionals
  more robust standalone compile-to-C++ properties
  fix for nested "when" statements
  test suite updates

src/script_opt/CPP/DeclFunc.cc

@@ -101,13 +101,9 @@ void CPPCompile::CreateFunction(const FuncTypePtr& ft, const ProfileFunc* pf, co
 		compiled_funcs.emplace(fname);
 		}
 
-	auto h = pf->HashVal();
-
-	body_hashes[fname] = h;
+	body_hashes[fname] = pf->HashVal();
 	body_priorities[fname] = priority;
 	body_names.emplace(body.get(), fname);
-
-	total_hash = merge_p_hashes(total_hash, h);
 	}
 
 void CPPCompile::DeclareSubclass(const FuncTypePtr& ft, const ProfileFunc* pf, const string& fname,

src/script_opt/CPP/Driver.cc

@@ -73,9 +73,8 @@ void CPPCompile::Compile(bool report_uncompilable)
 
 	working_dir = buf;
 
-	GenProlog();
-
 	unordered_set<string> filenames_reported_as_skipped;
+	bool had_to_skip = false;
 
 	// Determine which functions we can call directly, and reuse
 	// previously compiled instances of those if present.
@@ -84,11 +83,19 @@ void CPPCompile::Compile(bool report_uncompilable)
 		const auto& f = func.Func();
 
 		auto& ofiles = analysis_options.only_files;
+		auto allow_cond = analysis_options.allow_cond;
+
 		string fn = func.Body()->GetLocationInfo()->filename;
 
-		if ( ! func.ShouldSkip() && ! ofiles.empty() && files_with_conditionals.count(fn) > 0 )
+		if ( ! allow_cond && ! func.ShouldSkip() && ! ofiles.empty() &&
+		     files_with_conditionals.count(fn) > 0 )
 			{
-			if ( filenames_reported_as_skipped.count(fn) == 0 )
+			if ( report_uncompilable )
+				reporter->Warning(
+					"%s cannot be compiled to C++ due to source file %s having conditional code",
+					f->Name(), fn.c_str());
+
+			else if ( filenames_reported_as_skipped.count(fn) == 0 )
 				{
 				reporter->Warning(
 					"skipping compilation of files in %s due to presence of conditional code",
@@ -96,6 +103,7 @@ void CPPCompile::Compile(bool report_uncompilable)
 				filenames_reported_as_skipped.insert(fn);
 				}
 
+			had_to_skip = true;
 			func.SetSkip(true);
 			}
 
@@ -114,17 +122,30 @@ void CPPCompile::Compile(bool report_uncompilable)
 			}
 		else
 			{
-			if ( reason && standalone )
-				reporter->Error("%s cannot be compiled to standalone C++ due to %s", f->Name(),
-				                reason);
-
-			else if ( reason && report_uncompilable )
-				fprintf(stderr, "%s cannot be compiled to C++ due to %s\n", f->Name(), reason);
+			if ( reason && report_uncompilable )
+				{
+				had_to_skip = true;
+				reporter->Warning("%s cannot be compiled to C++ due to %s", f->Name(), reason);
+				}
 
 			not_fully_compilable.insert(f->Name());
 			}
 		}
 
+	if ( standalone && had_to_skip )
+		reporter->FatalError(
+			"aborting standalone compilation to C++ due to having to skip some functions");
+
+	// Generate a hash unique for this compilation.
+	for ( const auto& func : funcs )
+		if ( ! func.ShouldSkip() )
+			total_hash = merge_p_hashes(total_hash, func.Profile()->HashVal());
+
+	auto t = util::current_time();
+	total_hash = merge_p_hashes(total_hash, hash<double>{}(t));
+
+	GenProlog();
+
 	// Track all of the types we'll be using.
 	for ( const auto& t : pfs.RepTypes() )
 		{
@@ -213,7 +234,7 @@ void CPPCompile::GenProlog()
 		Emit("#include \"zeek/script_opt/CPP/Runtime.h\"\n");
 
 	Emit("namespace zeek::detail { //\n");
-	Emit("namespace CPP_%s { // %s\n", Fmt(addl_tag), working_dir);
+	Emit("namespace CPP_%s { // %s\n", Fmt(total_hash), working_dir);
 
 	// The following might-or-might-not wind up being populated/used.
 	Emit("std::vector<int> field_mapping;");

src/script_opt/CPP/Exprs.cc

@@ -305,11 +305,13 @@ string CPPCompile::GenCallExpr(const CallExpr* c, GenType gt)
 		auto f_id = f->AsNameExpr()->Id();
 		const auto& params = f_id->GetType()->AsFuncType()->Params();
 		auto id_name = f_id->Name();
+		auto nargs = args_l->Exprs().length();
 
 		bool is_compiled = compiled_simple_funcs.count(id_name) > 0;
 		bool was_compiled = hashed_funcs.count(id_name) > 0;
+		bool is_variadic = params->NumFields() == 1 && nargs != 1;
 
-		if ( ! is_async && (is_compiled || was_compiled) )
+		if ( ! is_async && ! is_variadic && (is_compiled || was_compiled) )
 			{ // Can call directly.
 			string fname;
 
@@ -318,7 +320,7 @@ string CPPCompile::GenCallExpr(const CallExpr* c, GenType gt)
 			else
 				fname = compiled_simple_funcs[id_name];
 
-			if ( args_l->Exprs().length() > 0 )
+			if ( nargs > 0 )
 				gen = fname + "(" + GenArgs(params, args_l) + ", f__CPP)";
 			else
 				gen = fname + "(f__CPP)";
@@ -1097,6 +1099,10 @@ string CPPCompile::GenDirectAssign(const ExprPtr& lhs, const string& rhs_native,
                                    const string& rhs_val_ptr, GenType gt, bool top_level)
 	{
 	auto n = lhs->AsNameExpr()->Id();
+
+	if ( n->IsBlank() )
+		return rhs_native;
+
 	auto name = IDNameStr(n);
 
 	string gen;

src/script_opt/CPP/Inits.cc

@@ -338,14 +338,7 @@ void CPPCompile::GenStandaloneActivation()
 
 void CPPCompile::GenLoad()
 	{
-	// First, generate a hash unique to this compilation.
-	auto t = util::current_time();
-	auto th = hash<double>{}(t);
-
-	total_hash = merge_p_hashes(total_hash, th);
-
 	Emit("register_scripts__CPP(%s, standalone_init__CPP);", Fmt(total_hash));
-
 	printf("global init_CPP_%llu = load_CPP(%llu);\n", total_hash, total_hash);
 	}
 

src/script_opt/CPP/README.md

@@ -162,7 +162,7 @@ about associated expressions/statements, making them hard to puzzle out.
 This could be fixed, but would add execution overhead in passing around
 the necessary strings / `Location` objects.
 
-* To avoid subtle bugs, the compiler will refrain from compiling script elements (functions, hooks, event handlers) that include conditional code.  In addition, when using `--optimize-files` it will not compile any functions appearing in a source file that includes conditional code (even if it's not in a function body).
+* To avoid subtle bugs, the compiler will refrain from compiling script elements (functions, hooks, event handlers) that include conditional code.  In addition, when using `--optimize-files` it will not compile any functions appearing in a source file that includes conditional code (even if it's not in a function body).  You can override this refusal with `-O allow-cond`.
 
 * Code compiled with `-O gen-standalone-C++` will not execute any global
 statements when invoked using the "stand-in" script.  The right fix for
@@ -170,7 +170,7 @@ this is to shift from encapsulating global statements in a pseudo-function,
 as currently done, to instead be in a pseudo-event handler.
 
 * Code compiled with `-O gen-standalone-C++` likely has bugs if that
-code requires initializing a global variable that specifies extend fields in
+code requires initializing a global variable that specifies extending fields in
 an extensible record (i.e., fields added using `redef`).
 
 * If a lambda generates an event that is not otherwise referred to, that

src/script_opt/CPP/RuntimeOps.h

@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include "zeek/Frame.h"
 #include "zeek/Val.h"
 #include "zeek/script_opt/CPP/Func.h"
 
@@ -46,6 +47,7 @@ extern ValPtr when_index_slice__CPP(VectorVal* vec, const ListVal* lv);
 // but (2) needing to have the address of that vector.
 inline ValPtr invoke__CPP(Func* f, std::vector<ValPtr> args, Frame* frame)
 	{
+	frame->SetOnlyCall(nullptr);
 	return f->Invoke(&args, frame);
 	}
 

src/script_opt/CPP/Stmts.cc

@@ -499,13 +499,16 @@ void CPPCompile::GenForOverTable(const ExprPtr& tbl, const IDPtr& value_var,
 	Emit("auto* current_tev__CPP = lve__CPP.value;");
 	Emit("auto ind_lv__CPP = tv__CPP->RecreateIndex(*k__CPP);");
 
-	if ( value_var )
+	if ( value_var && ! value_var->IsBlank() )
 		Emit("%s = %s;", IDName(value_var),
 		     GenericValPtrToGT("current_tev__CPP->GetVal()", value_var->GetType(), GEN_NATIVE));
 
 	for ( int i = 0; i < loop_vars->length(); ++i )
 		{
 		auto var = (*loop_vars)[i];
+		if ( var->IsBlank() )
+			continue;
+
 		const auto& v_t = var->GetType();
 		auto acc = NativeAccessor(v_t);
 
@@ -526,9 +529,12 @@ void CPPCompile::GenForOverVector(const ExprPtr& vec, const IDPtr& value_var,
 
 	Emit("if ( ! vv__CPP->Has(i__CPP) ) continue;");
 
-	Emit("%s = i__CPP;", IDName((*loop_vars)[0]));
+	auto lv0 = (*loop_vars)[0];
 
-	if ( value_var )
+	if ( ! lv0->IsBlank() )
+		Emit("%s = i__CPP;", IDName(lv0));
+
+	if ( value_var && ! value_var->IsBlank() )
 		{
 		auto vv = IDName(value_var);
 		auto access = "vv__CPP->ValAt(i__CPP)";
@@ -545,7 +551,10 @@ void CPPCompile::GenForOverString(const ExprPtr& str, const IDPList* loop_vars)
 	StartBlock();
 
 	Emit("auto sv__CPP = make_intrusive<StringVal>(1, (const char*) sval__CPP->Bytes() + i__CPP);");
-	Emit("%s = std::move(sv__CPP);", IDName((*loop_vars)[0]));
+
+	auto lv0 = (*loop_vars)[0];
+	if ( ! lv0->IsBlank() )
+		Emit("%s = std::move(sv__CPP);", IDName(lv0));
 	}
 
 	} // zeek::detail

src/script_opt/CPP/Util.cc

@@ -28,7 +28,7 @@ string Fmt(double d)
 
 string scope_prefix(const string& scope)
 	{
-	return string("zeek::detail::CPP_") + scope + "::";
+	return "zeek::detail::CPP_" + scope;
 	}
 
 string scope_prefix(int scope)
@@ -38,6 +38,9 @@ string scope_prefix(int scope)
 
 bool is_CPP_compilable(const ProfileFunc* pf, const char** reason)
 	{
+	if ( analysis_options.allow_cond )
+		return true;
+
 	auto body = pf->ProfiledBody();
 	if ( body && ! body->GetOptInfo()->is_free_of_conditionals )
 		{

src/script_opt/CPP/maint/README

@@ -15,7 +15,7 @@ The maintenance workflow:
     to check in updates to the list of how the compiler currently fares
     on various btests (see end of this doc):
 
-	Wed Oct 12 11:36:46 PDT 2022
+	Mon Nov  7 14:30:51 PST 2022
 
 2.  Run "find-test-files.sh" to generate a list (to stdout) of all of the
     possible Zeek source files found in the test suite.
@@ -62,23 +62,19 @@ These BTests won't successfully run due to the indicated issue:
 	bad-constructor - uses a complex old-style constructor that
 		should be updated
 	bad-when - deliberately has old-style "when" without captures
+	cond - not compilable due to the presence of conditional code
 	deprecated - uses deprecated features not support for -O gen-C++
 	test-glitch - fails because of how we do testing: the first -O gen-C++
 		pass leaves httpd running, which causes the second -O use-C++
 		pass to fail when it tries to start up a new httpd
-	opaque - needs a global whose value is (or contains) an opaque value,
-		not currently supported
 	skipped - test can be skipped due to environmental reasons (e.g.,
 		whether we have a certain Kerberos setup)
 
 Database Of Known Issues (keep sorted)
 
 ../testing/btest/bifs/table_values.zeek bad-constructor
-../testing/btest/language/alternate-event-hook-prototypes.zeek deprecated
-../testing/btest/language/redef-same-prefixtable-idx.zeek deprecated
-../testing/btest/language/table-redef.zeek deprecated
-../testing/btest/language/when-aggregates.zeek bad-when
 ../testing/btest/scripts/base/protocols/krb/smb2_krb.test skipped
 ../testing/btest/scripts/base/protocols/krb/smb2_krb_nokeytab.test skipped
 ../testing/btest/scripts/base/utils/active-http.test test-glitch
+../testing/btest/scripts/policy/frameworks/dpd/packet-segment-logging.zeek cond
 ../testing/btest/scripts/policy/misc/dump-events.zeek skipped