Merge remote-tracking branch 'origin/topic/vern/CPP-global-init'

* origin/topic/vern/CPP-global-init:
  updates to -O C++ user and maintenance documentation
  use dynamic rather than static initialization of globals for scripts compiled to C++
  safety checking for initializing scripts compiled to C++
  fixes for initializing scripts compiled to C++
  restructured tracking of initializations of globals for script compilation

src/script_opt/CPP/Attrs.cc

@@ -18,7 +18,7 @@ shared_ptr<CPP_InitInfo> CPPCompile::RegisterAttributes(const AttributesPtr& att
 	if ( pa != processed_attrs.end() )
 		return pa->second;
 
-	attributes.AddKey(attrs);
+	attributes.AddKey(attrs, pfs.HashAttrs(attrs));
 
 	// The cast is just so we can make an IntrusivePtr.
 	auto a_rep = const_cast<Attributes*>(attributes.GetRep(attrs));
@@ -49,7 +49,7 @@ shared_ptr<CPP_InitInfo> CPPCompile::RegisterAttr(const AttrPtr& attr)
 
 	const auto& e = a->GetExpr();
 	if ( e && ! IsSimpleInitExpr(e) )
-		init_exprs.AddKey(e);
+		init_exprs.AddKey(e, p_hash(e));
 
 	auto gi = make_shared<AttrInfo>(this, attr);
 	attr_info->AddInstance(gi);

src/script_opt/CPP/Compile.h

@@ -1015,6 +1015,10 @@ private:
 	// Generate code to initialize indirect references to constants.
 	void InitializeConsts();
 
+	// Generate code to initialize globals (using dynamic statements
+	// rather than constants).
+	void InitializeGlobals();
+
 	// Generate the initialization hook for this set of compiled code.
 	void GenInitHook();
 

src/script_opt/CPP/Driver.cc

@@ -5,6 +5,7 @@
 #include <cerrno>
 
 #include "zeek/script_opt/CPP/Compile.h"
+#include "zeek/script_opt/IDOptInfo.h"
 
 extern std::unordered_set<std::string> files_with_conditionals;
 
@@ -310,6 +311,9 @@ void CPPCompile::RegisterCompiledBody(const string& f)
 
 void CPPCompile::GenEpilog()
 	{
+	NL();
+	InitializeGlobals();
+
 	NL();
 	for ( const auto& ii : init_infos )
 		GenInitExpr(ii.second);
@@ -468,6 +472,9 @@ void CPPCompile::GenFinishInit()
 	NL();
 	Emit("load_BiFs__CPP();");
 
+	NL();
+	Emit("init_globals__CPP();");
+
 	EndBlock();
 	}
 

src/script_opt/CPP/Inits.cc

@@ -188,6 +188,50 @@ void CPPCompile::InitializeConsts()
 	EndBlock(true);
 	}
 
+void CPPCompile::InitializeGlobals()
+	{
+	Emit("static void init_globals__CPP()");
+	StartBlock();
+
+	Emit("Frame* f__CPP = nullptr;");
+	NL();
+
+	for ( auto ginit : IDOptInfo::GetGlobalInitExprs() )
+		{
+		auto g = ginit.Id();
+		if ( pfs.Globals().count(g) == 0 )
+			continue;
+
+		auto ic = ginit.IC();
+		auto& init = ginit.Init();
+
+		if ( ic == INIT_NONE )
+			{
+			IDPtr gid = {NewRef{}, const_cast<ID*>(g)};
+			auto gn = make_intrusive<RefExpr>(make_intrusive<NameExpr>(gid));
+			auto ae = make_intrusive<AssignExpr>(gn, init, true);
+			Emit(GenExpr(ae.get(), GEN_NATIVE, true) + ";");
+			}
+
+		else
+			{
+			// This branch occurs for += or -= initializations that
+			// use associated functions.
+			string ics;
+			if ( ic == INIT_EXTRA )
+				ics = "INIT_EXTRA";
+			else if ( ic == INIT_REMOVE )
+				ics = "INIT_REMOVE";
+			else
+				reporter->FatalError("bad initialization class in CPPCompile::InitializeGlobals()");
+
+			Emit("%s->SetValue(%s, %s);", globals[g->Name()], GenExpr(init, GEN_NATIVE, true), ics);
+			}
+		}
+
+	EndBlock();
+	}
+
 void CPPCompile::GenInitHook()
 	{
 	NL();

src/script_opt/CPP/InitsInfo.cc

@@ -339,16 +339,7 @@ GlobalInitInfo::GlobalInitInfo(CPPCompile* c, const ID* g, string _CPP_name)
 		attrs = -1;
 
 	exported = g->IsExport();
-
-	auto v = g->GetVal();
-	if ( v && gt->Tag() == TYPE_OPAQUE )
-		{
-		reporter->Error("cannot compile to C++ global \"%s\" initialized to opaque value",
-		                g->Name());
-		v = nullptr;
-		}
-
-	val = ValElem(c, v);
+	val = ValElem(c, nullptr); // empty because we initialize dynamically
 	}
 
 void GlobalInitInfo::InitializerVals(std::vector<std::string>& ivs) const
@@ -439,6 +430,9 @@ ListTypeInfo::ListTypeInfo(CPPCompile* _c, TypePtr _t)
 		if ( gi )
 			init_cohort = max(init_cohort, gi->InitCohort());
 		}
+
+	if ( ! types.empty() )
+		++init_cohort;
 	}
 
 void ListTypeInfo::AddInitializerVals(std::vector<std::string>& ivs) const

src/script_opt/CPP/README.md

@@ -173,9 +173,6 @@ as currently done, to instead be in a pseudo-event handler.
 code requires initializing a global variable that specifies extend fields in
 an extensible record (i.e., fields added using `redef`).
 
-* The compiler will not compile bodies that include "when" statements
-This is fairly involved to fix.
-
 * If a lambda generates an event that is not otherwise referred to, that
 event will not be registered upon instantiating the lambda.  This is not
 particularly difficult to fix.

src/script_opt/CPP/RuntimeInits.cc

@@ -487,11 +487,10 @@ void CPP_GlobalInit::Generate(InitsManager* im, std::vector<void*>& /* inits_vec
 	global = lookup_global__CPP(name, im->Types(type), exported);
 
 	if ( ! global->HasVal() && val >= 0 )
-		{
 		global->SetVal(im->ConstVals(val));
-		if ( attrs >= 0 )
-			global->SetAttrs(im->Attributes(attrs));
-		}
+
+	if ( attrs >= 0 )
+		global->SetAttrs(im->Attributes(attrs));
 	}
 
 void generate_indices_set(int* inits, std::vector<std::vector<int>>& indices_set)

src/script_opt/CPP/RuntimeInits.h

@@ -66,12 +66,41 @@ public:
 	// Accessors for the sundry initialization vectors, each retrieving
 	// a specific element identified by an index/offset.
 	const std::vector<int>& Indices(int offset) const { return indices[offset]; }
-	const char* Strings(int offset) const { return strings[offset]; }
-	const p_hash_type Hashes(int offset) const { return hashes[offset]; }
-	const TypePtr& Types(int offset) const { return types[offset]; }
-	const AttributesPtr& Attributes(int offset) const { return attributes[offset]; }
-	const AttrPtr& Attrs(int offset) const { return attrs[offset]; }
-	const CallExprPtr& CallExprs(int offset) const { return call_exprs[offset]; }
+	const char* Strings(int offset) const
+		{
+		ASSERT(offset >= 0 && offset < strings.size());
+		ASSERT(strings[offset]);
+		return strings[offset];
+		}
+	const p_hash_type Hashes(int offset) const
+		{
+		ASSERT(offset >= 0 && offset < hashes.size());
+		return hashes[offset];
+		}
+	const TypePtr& Types(int offset) const
+		{
+		ASSERT(offset >= 0 && offset < types.size());
+		ASSERT(types[offset]);
+		return types[offset];
+		}
+	const AttributesPtr& Attributes(int offset) const
+		{
+		ASSERT(offset >= 0 && offset < attributes.size());
+		ASSERT(attributes[offset]);
+		return attributes[offset];
+		}
+	const AttrPtr& Attrs(int offset) const
+		{
+		ASSERT(offset >= 0 && offset < attrs.size());
+		ASSERT(attrs[offset]);
+		return attrs[offset];
+		}
+	const CallExprPtr& CallExprs(int offset) const
+		{
+		ASSERT(offset >= 0 && offset < call_exprs.size());
+		ASSERT(call_exprs[offset]);
+		return call_exprs[offset];
+		}
 
 private:
 	std::vector<CPP_ValElem>& const_vals;

src/script_opt/CPP/maint/README

@@ -15,7 +15,7 @@ The maintenance workflow:
     to check in updates to the list of how the compiler currently fares
     on various btests (see end of this doc):
 
-	Fri Sep 16 16:13:49 PDT 2022
+	Thu Sep 29 14:49:49 PDT 2022
 
 2.  Run "find-test-files.sh" to generate a list (to stdout) of all of the
     possible Zeek source files found in the test suite.
@@ -74,18 +74,11 @@ These BTests won't successfully run due to the indicated issue:
 Database Of Known Issues (keep sorted)
 
 ../testing/btest/bifs/table_values.zeek bad-constructor
-../testing/btest/core/global_opaque_val.zeek opaque
 ../testing/btest/language/alternate-event-hook-prototypes.zeek deprecated
-../testing/btest/language/global-init-calls-bif.zeek opaque
 ../testing/btest/language/redef-same-prefixtable-idx.zeek deprecated
 ../testing/btest/language/table-redef.zeek deprecated
 ../testing/btest/language/when-aggregates.zeek bad-when
 ../testing/btest/scripts/base/protocols/krb/smb2_krb.test skipped
 ../testing/btest/scripts/base/protocols/krb/smb2_krb_nokeytab.test skipped
 ../testing/btest/scripts/base/utils/active-http.test test-glitch
-../testing/btest/scripts/policy/frameworks/telemetry/log-prefixes.zeek opaque
-../testing/btest/scripts/policy/frameworks/telemetry/log.zeek opaque
 ../testing/btest/scripts/policy/misc/dump-events.zeek skipped
-../testing/btest/telemetry/counter.zeek opaque
-../testing/btest/telemetry/gauge.zeek opaque
-../testing/btest/telemetry/histogram.zeek opaque