Merge remote-tracking branch 'origin/topic/vern/CPP-standalone-fixes'

* origin/topic/vern/CPP-standalone-fixes:
  fixes for initializing globals when using -O gen-standalone-C++

CHANGES

@@ -1,3 +1,7 @@
+7.2.0-dev.52 | 2025-01-09 12:02:45 -0700
+
+  * fixes for initializing globals when using -O gen-standalone-C++ (Vern Paxson, Corelight)
+
 7.2.0-dev.50 | 2025-01-09 16:26:00 +0100
 
   * Update COPYING date to now and fix some [skip CI] (Evan Typanski, Corelight)

VERSION

@@ -1 +1 @@
-7.2.0-dev.50
+7.2.0-dev.52

src/script_opt/CPP/Driver.cc

@@ -44,7 +44,16 @@ void CPPCompile::Compile(bool report_uncompilable) {
             reporter->FatalError("aborting standalone compilation to C++ due to having to skip some functions");
 
         for ( auto& g : global_scope()->OrderedVars() ) {
-            if ( ! obj_matches_opt_files(g) )
+            bool compiled_global = obj_matches_opt_files(g);
+
+            if ( ! compiled_global )
+                for ( const auto& i_e : g->GetOptInfo()->GetInitExprs() )
+                    if ( obj_matches_opt_files(i_e) ) {
+                        compiled_global = true;
+                        break;
+                    }
+
+            if ( ! compiled_global )
                 continue;
 
             // We will need to generate this global's definition, including
@@ -63,6 +72,10 @@ void CPPCompile::Compile(bool report_uncompilable) {
                     (void)pfs->HashType(t);
                     rep_types.insert(TypeRep(t));
                 }
+                for ( auto& g : pf->Globals() )
+                    accessed_globals.insert(g);
+                for ( auto& ag : pf->AllGlobals() )
+                    all_accessed_globals.insert(ag);
             }
         }
 

src/script_opt/CPP/InitsInfo.cc

@@ -355,14 +355,16 @@ AttrsInfo::AttrsInfo(CPPCompile* _c, const AttributesPtr& _attrs) : CompoundItem
     }
 }
 
-GlobalLookupInitInfo::GlobalLookupInitInfo(CPPCompile* c, const ID* g, string _CPP_name)
+GlobalLookupInitInfo::GlobalLookupInitInfo(CPPCompile* c, const ID* g, string _CPP_name, bool do_init)
     : CPP_InitInfo(g), CPP_name(std::move(_CPP_name)) {
     Zeek_name = g->Name();
+    val = ValElem(c, do_init ? g->GetVal() : nullptr);
 }
 
 void GlobalLookupInitInfo::InitializerVals(std::vector<std::string>& ivs) const {
     ivs.push_back(CPP_name);
     ivs.push_back(string("\"") + Zeek_name + "\"");
+    ivs.push_back(val);
 }
 
 GlobalInitInfo::GlobalInitInfo(CPPCompile* c, const ID* g, string _CPP_name)
@@ -380,7 +382,8 @@ GlobalInitInfo::GlobalInitInfo(CPPCompile* c, const ID* g, string _CPP_name)
     else
         attrs = -1;
 
-    exported = g->IsExport();
+    is_exported = g->IsExport();
+    is_option = g->IsOption();
     val = ValElem(c, nullptr); // empty because we initialize dynamically
 
     if ( gt->Tag() == TYPE_FUNC && (! g->GetVal() || g->GetVal()->AsFunc()->GetKind() == Func::BUILTIN_FUNC) )
@@ -396,7 +399,8 @@ void GlobalInitInfo::InitializerVals(std::vector<std::string>& ivs) const {
     ivs.push_back(Fmt(type));
     ivs.push_back(Fmt(attrs));
     ivs.push_back(val);
-    ivs.push_back(Fmt(exported));
+    ivs.push_back(Fmt(is_exported));
+    ivs.push_back(Fmt(is_option));
     ivs.push_back(Fmt(func_with_no_val));
 }
 

src/script_opt/CPP/InitsInfo.h

@@ -479,10 +479,11 @@ public:
 };
 
 // A lightweight initializer for a Zeek global that will look it up at
-// initialization time but not create it if missing.
+// initialization time but not create it if missing. If do_init is true,
+// then the global will be (re-)initialized to its value during compilation.
 class GlobalLookupInitInfo : public CPP_InitInfo {
 public:
-    GlobalLookupInitInfo(CPPCompile* c, const ID* g, std::string CPP_name);
+    GlobalLookupInitInfo(CPPCompile* c, const ID* g, std::string CPP_name, bool do_init = false);
 
     std::string InitializerType() const override { return "CPP_GlobalLookupInit"; }
     void InitializerVals(std::vector<std::string>& ivs) const override;
@@ -490,6 +491,7 @@ public:
 protected:
     std::string Zeek_name;
     std::string CPP_name;
+    std::string val;
 };
 
 // Information for initializing a Zeek global.
@@ -504,7 +506,8 @@ protected:
     int type;
     int attrs;
     std::string val;
-    bool exported;
+    bool is_exported;
+    bool is_option;
     bool func_with_no_val = false; // needed to handle some error situations
 };
 

src/script_opt/CPP/RuntimeInits.cc

@@ -457,11 +457,17 @@ int CPP_EnumMapping::ComputeOffset(InitsManager* im) const {
 
 void CPP_GlobalLookupInit::Generate(InitsManager* im, std::vector<void*>& /* inits_vec */, int /* offset */) const {
     global = find_global__CPP(name);
+    if ( val >= 0 )
+        // Have explicit initialization value.
+        global->SetVal(im->ConstVals(val));
 }
 
 void CPP_GlobalInit::Generate(InitsManager* im, std::vector<void*>& /* inits_vec */, int /* offset */) const {
     auto& t = im->Types(type);
-    global = lookup_global__CPP(name, t, exported);
+    global = lookup_global__CPP(name, t, is_exported);
+
+    if ( is_option )
+        global->SetOption();
 
     if ( ! global->HasVal() ) {
         if ( val >= 0 )

src/script_opt/CPP/RuntimeInits.h

@@ -383,26 +383,29 @@ public:
 // vector, so we use void* as the underlying type.
 class CPP_GlobalLookupInit : public CPP_Init<void*> {
 public:
-    CPP_GlobalLookupInit(IDPtr& _global, const char* _name) : CPP_Init<void*>(), global(_global), name(_name) {}
+    CPP_GlobalLookupInit(IDPtr& _global, const char* _name, int _val)
+        : CPP_Init<void*>(), global(_global), name(_name), val(_val) {}
 
     void Generate(InitsManager* im, std::vector<void*>& /* inits_vec */, int /* offset */) const override;
 
 protected:
     IDPtr& global;
     const char* name;
+    int val;
 };
 
 class CPP_GlobalInit : public CPP_Init<void*> {
 public:
-    CPP_GlobalInit(IDPtr& _global, const char* _name, int _type, int _attrs, int _val, bool _exported,
-                   bool _func_with_no_val)
+    CPP_GlobalInit(IDPtr& _global, const char* _name, int _type, int _attrs, int _val, bool _is_exported,
+                   bool _is_option, bool _func_with_no_val)
         : CPP_Init<void*>(),
           global(_global),
           name(_name),
           type(_type),
           attrs(_attrs),
           val(_val),
-          exported(_exported),
+          is_exported(_is_exported),
+          is_option(_is_option),
           func_with_no_val(_func_with_no_val) {}
 
     void Generate(InitsManager* im, std::vector<void*>& /* inits_vec */, int /* offset */) const override;
@@ -413,7 +416,8 @@ protected:
     int type;
     int attrs;
     int val;
-    bool exported;
+    bool is_exported;
+    bool is_option;
     bool func_with_no_val;
 };
 

src/script_opt/CPP/Vars.cc

@@ -1,6 +1,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "zeek/script_opt/CPP/Compile.h"
+#include "zeek/script_opt/IDOptInfo.h"
 #include "zeek/script_opt/ProfileFunc.h"
 
 namespace zeek::detail {
@@ -32,12 +33,7 @@ void CPPCompile::CreateGlobal(const ID* g) {
             // This is an event that's also used as a variable.
             Emit("EventHandlerPtr %s_ev;", globals[gn]);
 
-        shared_ptr<CPP_InitInfo> gi;
-        if ( standalone )
-            gi = make_shared<GlobalInitInfo>(this, g, globals[gn]);
-        else
-            gi = make_shared<GlobalLookupInitInfo>(this, g, globals[gn]);
-
+        auto gi = GenerateGlobalInit(g);
         global_id_info->AddInstance(gi);
         global_gis[g] = gi;
     }
@@ -69,18 +65,36 @@ std::shared_ptr<CPP_InitInfo> CPPCompile::RegisterGlobal(const ID* g) {
             return gg->second;
     }
 
-    shared_ptr<CPP_InitInfo> gi;
-    if ( standalone )
-        gi = make_shared<GlobalInitInfo>(this, g, globals[gn]);
-    else
-        gi = make_shared<GlobalLookupInitInfo>(this, g, globals[gn]);
-
+    auto gi = GenerateGlobalInit(g);
     global_id_info->AddInstance(gi);
     global_gis[g] = gi;
 
     return gi;
 }
 
+std::shared_ptr<CPP_InitInfo> CPPCompile::GenerateGlobalInit(const ID* g) {
+    auto gn = string(g->Name());
+    if ( ! standalone )
+        return make_shared<GlobalLookupInitInfo>(this, g, globals[gn]);
+
+    if ( obj_matches_opt_files(g) )
+        return make_shared<GlobalInitInfo>(this, g, globals[gn]);
+
+    // It's not a global that's created by the scripts we're compiling,
+    // but it might have a redef in those scripts, in which case we need
+    // to generate an initializer that will both look it up and then assign
+    // it to that value.
+    bool needs_redef = false;
+
+    for ( const auto& i_e : g->GetOptInfo()->GetInitExprs() )
+        if ( obj_matches_opt_files(i_e) ) {
+            needs_redef = true;
+            break;
+        }
+
+    return make_shared<GlobalLookupInitInfo>(this, g, globals[gn], needs_redef);
+}
+
 void CPPCompile::AddBiF(const ID* b, bool is_var) {
     auto bn = b->Name();
     auto n = string(bn);

src/script_opt/CPP/Vars.h

@@ -14,6 +14,10 @@ private:
 // as a variable (not just as a function being called), track it as such.
 void CreateGlobal(const ID* g);
 
+// Low-level function for generating an initializer for a global. Takes
+// into account differences for standalone-compilation.
+std::shared_ptr<CPP_InitInfo> GenerateGlobalInit(const ID* g);
+
 // Register the given identifier as a BiF.  If is_var is true then the BiF
 // is also used in a non-call context.
 void AddBiF(const ID* b, bool is_var);

src/script_opt/CPP/maint/README

@@ -21,7 +21,11 @@ The maintenance workflow:
 
 	rm CPP-gen.cc
 	ninja
-	src/zeek -b -O gen-standalone-C++ --optimize-files=base/protocols/conn base/protocols/conn >my-test.zeek
+	src/zeek -b -O gen-standalone-C++ \
+		--optimize-files=base/protocols/conn \
+		--optimize-files=base/utils/files \
+		--optimize-files=base/utils/addr \
+		base/protocols/conn >my-test.zeek
 	ninja
 	src/zeek -b -r some.pcap my-test.zeek
 	# Confirm that it generates conn.log