Merge remote-tracking branch 'origin/topic/vern/script-opt-maint.Jan24'

* origin/topic/vern/script-opt-maint.Jan24:
  ZAM speedup for constructing empty vectors
  fixes for ZAM optimization of "switch" statements
  BTests to catch regressions for recent ZAM fixes
  "-a zam" BTest baseline update for recent changes
  fix for needing to always flush optimization information for identifiers
  fix for logic bug in ldap base script
  better name for key variable in script optimization
  ZAM fix for tracking variable usage
  ZAM fixes for "for" loops that are only used to choose an element from a table/set
  ZAM fixes for loops indexed with variables not used in the loop body
  fix for ZAM location tracking - more extensive changes are pending
  fixes for ZAM's special-casing of that "cat" BiF
  some fixes for ZAM memory management
  streamlining of some script optimization APIs
  fixes for initializations of "-O gen-C++" script compilations
  script optimization fixes for "concretizing" vector-of-any's

CHANGES

@@ -1,3 +1,37 @@
+6.2.0-dev.396 | 2024-01-15 15:17:45 +0100
+
+  * ZAM speedup for constructing empty vectors (Vern Paxson, Corelight)
+
+  * fixes for ZAM optimization of "switch" statements (Vern Paxson, Corelight)
+
+  * BTests to catch regressions for recent ZAM fixes (Vern Paxson, Corelight)
+
+  * "-a zam" BTest baseline update for recent changes (Vern Paxson, Corelight)
+
+  * fix for needing to always flush optimization information for identifiers (Vern Paxson, Corelight)
+
+  * fix for logic bug in ldap base script (Vern Paxson, Corelight)
+
+  * better name for key variable in script optimization (Vern Paxson, Corelight)
+
+  * ZAM fix for tracking variable usage (Vern Paxson, Corelight)
+
+  * ZAM fixes for "for" loops that are only used to choose an element from a table/set (Vern Paxson, Corelight)
+
+  * ZAM fixes for loops indexed with variables not used in the loop body (Vern Paxson, Corelight)
+
+  * fix for ZAM location tracking - more extensive changes are pending (Vern Paxson, Corelight)
+
+  * fixes for ZAM's special-casing of that "cat" BiF (Vern Paxson, Corelight)
+
+  * some fixes for ZAM memory management (Vern Paxson, Corelight)
+
+  * streamlining of some script optimization APIs (Vern Paxson, Corelight)
+
+  * fixes for initializations of "-O gen-C++" script compilations (Vern Paxson, Corelight)
+
+  * script optimization fixes for "concretizing" vector-of-any's (Vern Paxson, Corelight)
+
 6.2.0-dev.378 | 2024-01-12 10:48:33 +0100
 
   * btest/smtp: Test with smtp-bdat-pipeline-8bitmime.pcap (Arne Welzel, Corelight)

VERSION

@@ -1 +1 @@
-6.2.0-dev.378
+6.2.0-dev.396

scripts/base/protocols/ldap/main.zeek

@@ -203,7 +203,7 @@ event LDAP::message(c: connection,
     }
 
     if ( diagnostic_message != "" ) {
-      if ( ! sm?$diagnostic_message && sm$diagnostic_message != diagnostic_message ) {
+      if ( sm?$diagnostic_message && sm$diagnostic_message != diagnostic_message ) {
         Reporter::conn_weird("LDAP_search_diagnostic_message_change", c,
                              fmt("%s: %s -> %s", message_id, sm$diagnostic_message, diagnostic_message), "LDAP");
       }

src/Expr.h

@@ -1310,6 +1310,10 @@ public:
     // Optimization-related:
     ExprPtr Duplicate() override;
 
+    bool IsReduced(Reducer* c) const override;
+    bool WillTransform(Reducer* c) const override;
+    ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
+
 protected:
     ValPtr Fold(Val* v) const override;
 };

src/Type.cc

@@ -902,6 +902,9 @@ public:
         if ( coerce_type )
             v = v->AsRecordVal()->CoerceTo(coerce_type);
 
+        else if ( init_type->Tag() == TYPE_VECTOR )
+            concretize_if_unspecified(cast_intrusive<VectorVal>(v), init_type->Yield());
+
         return ZVal(v, init_type);
     }
 

src/Type.h

@@ -366,6 +366,15 @@ public:
 protected:
     IndexType(TypeTag t, TypeListPtr arg_indices, TypePtr arg_yield_type)
         : Type(t), indices(std::move(arg_indices)), yield_type(std::move(arg_yield_type)) {
+        // "indices" might be nil if we're deferring construction of the type
+        // for "-O use-C++" initialization.
+        if ( indices )
+            SetSpecialIndices();
+        else
+            is_subnet_index = is_pattern_index = false; // placeholders
+    }
+
+    void SetSpecialIndices() {
         const auto& types = indices->GetTypes();
         is_subnet_index = types.size() == 1 && types[0]->Tag() == TYPE_SUBNET;
         is_pattern_index = types.size() == 1 && types[0]->Tag() == TYPE_PATTERN;

src/Val.cc

@@ -1889,45 +1889,56 @@ ValPtr TableVal::Default(const ValPtr& index) {
         return nullptr;
     }
 
+    ValPtr result;
+
     if ( def_val->GetType()->Tag() != TYPE_FUNC || same_type(def_val->GetType(), GetType()->Yield()) ) {
         if ( def_attr->GetExpr()->IsConst() )
             return def_val;
 
         try {
-            return def_val->Clone();
+            result = def_val->Clone();
         } catch ( InterpreterException& e ) { /* Already reported. */
         }
 
-        Error("&default value for table is not clone-able");
-        return nullptr;
+        if ( ! result ) {
+            Error("&default value for table is not clone-able");
+            return nullptr;
+        }
+    }
+    else {
+        const Func* f = def_val->AsFunc();
+        Args vl;
+
+        if ( index->GetType()->Tag() == TYPE_LIST ) {
+            auto lv = index->AsListVal();
+            vl.reserve(lv->Length());
+
+            for ( const auto& v : lv->Vals() )
+                vl.emplace_back(v);
+        }
+        else
+            vl.emplace_back(index);
+
+        try {
+            result = f->Invoke(&vl);
+        }
+
+        catch ( InterpreterException& e ) { /* Already reported. */
+        }
+
+        if ( ! result ) {
+            Error("no value returned from &default function");
+            return nullptr;
+        }
     }
 
-    const Func* f = def_val->AsFunc();
-    Args vl;
-
-    if ( index->GetType()->Tag() == TYPE_LIST ) {
-        auto lv = index->AsListVal();
-        vl.reserve(lv->Length());
-
-        for ( const auto& v : lv->Vals() )
-            vl.emplace_back(v);
-    }
-    else
-        vl.emplace_back(index);
-
-    ValPtr result;
-
-    try {
-        result = f->Invoke(&vl);
-    }
-
-    catch ( InterpreterException& e ) { /* Already reported. */
-    }
-
-    if ( ! result ) {
-        Error("no value returned from &default function");
-        return nullptr;
-    }
+    auto rt = result->GetType();
+    if ( rt->Tag() == TYPE_VECTOR )
+        // The double-Yield() here is because this is a "table of vector of X"
+        // and we want X. If this is instead a "table of any", that'll be
+        // okay because concretize_if_unspecified() correctly deals with
+        // nil target types.
+        detail::concretize_if_unspecified(cast_intrusive<VectorVal>(result), GetType()->Yield()->Yield());
 
     return result;
 }
@@ -3474,6 +3485,26 @@ bool VectorVal::Concretize(const TypePtr& t) {
     return true;
 }
 
+void detail::concretize_if_unspecified(VectorValPtr v, TypePtr t) {
+    if ( v->Size() != 0 )
+        // Concretization only applies to empty vectors.
+        return;
+
+    if ( v->GetType()->Yield()->Tag() != TYPE_ANY )
+        // It's not an unspecified vector.
+        return;
+
+    if ( ! t )
+        // "t" can be nil if the vector is being assigned to an "any" value.
+        return;
+
+    if ( t->Tag() == TYPE_ANY )
+        // No need to concretize.
+        return;
+
+    v->Concretize(t);
+}
+
 unsigned int VectorVal::ComputeFootprint(std::unordered_set<const Val*>* analyzed_vals) const {
     auto n = vector_val.size();
     unsigned int fp = n;

src/Val.h

@@ -1744,6 +1744,13 @@ namespace detail {
 // for normalization. If Func::nil is passed, no normalization happens.
 extern std::variant<ValPtr, std::string> ValFromJSON(std::string_view json_str, const TypePtr& t,
                                                      const FuncPtr& key_func);
+
+// If the given vector is an empty vector-of-any ("unspecified"),
+// concretizes it to the given type. *v* gives the vector and *t* the
+// type to concretize it to if appropriate. *t* can be nil, in which
+// case nothing is done.
+extern void concretize_if_unspecified(VectorValPtr v, TypePtr t);
+
 } // namespace detail
 
 } // namespace zeek

src/script_opt/CPP/InitsInfo.cc

@@ -441,7 +441,7 @@ TableTypeInfo::TableTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c,
     auto gi = c->RegisterType(tbl->GetIndices());
     ASSERT(gi);
     indices = gi->Offset();
-    final_init_cohort = gi->InitCohort();
+    final_init_cohort = gi->InitCohort() + 1;
 
     yield = tbl->Yield();
 

src/script_opt/CPP/RuntimeInitSupport.h

@@ -22,8 +22,10 @@ public:
     CPPTableType() : TableType(nullptr, nullptr){};
 
     void SetIndexAndYield(TypeListPtr ind, TypePtr yield) {
-        ind = std::move(indices);
+        indices = std::move(ind);
         yield_type = std::move(yield);
+        SetSpecialIndices();
+        RegenerateHash();
     }
 };
 

src/script_opt/CPP/RuntimeOps.cc

@@ -221,7 +221,7 @@ TableValPtr set_constructor__CPP(vector<ValPtr> elements, TableTypePtr t, vector
     auto aggr = make_intrusive<TableVal>(std::move(t), std::move(attrs));
 
     for ( auto& elem : elements )
-        aggr->Assign(std::move(elem), nullptr);
+        aggr->Assign(elem, nullptr);
 
     return aggr;
 }

src/script_opt/Expr.cc

@@ -1443,8 +1443,6 @@ ExprPtr AssignExpr::Reduce(Reducer* c, StmtPtr& red_stmt) {
     StmtPtr rhs_reduce;
 
     if ( lhs_is_any != rhs_is_any ) {
-        auto op2_loc = op2->GetLocationInfo();
-
         ExprPtr red_rhs = op2->ReduceToSingleton(c, rhs_reduce);
 
         if ( lhs_is_any ) {
@@ -1455,15 +1453,11 @@ ExprPtr AssignExpr::Reduce(Reducer* c, StmtPtr& red_stmt) {
         }
         else
             op2 = make_intrusive<CoerceFromAnyExpr>(red_rhs, t1);
-
-        op2->SetLocationInfo(op2_loc);
     }
 
     if ( t1->Tag() == TYPE_VECTOR && t1->Yield()->Tag() != TYPE_ANY && t2->Yield() && t2->Yield()->Tag() == TYPE_ANY ) {
-        auto op2_loc = op2->GetLocationInfo();
         ExprPtr red_rhs = op2->ReduceToSingleton(c, rhs_reduce);
         op2 = make_intrusive<CoerceFromAnyVecExpr>(red_rhs, t1);
-        op2->SetLocationInfo(op2_loc);
     }
 
     auto lhs_ref = op1->AsRefExprPtr();
@@ -1950,6 +1944,30 @@ ExprPtr VectorCoerceExpr::Duplicate() {
     return SetSucc(new VectorCoerceExpr(op_dup, GetType<VectorType>()));
 }
 
+bool VectorCoerceExpr::IsReduced(Reducer* c) const {
+    if ( WillTransform(c) )
+        return NonReduced(this);
+
+    return UnaryExpr::IsReduced(c);
+}
+
+bool VectorCoerceExpr::WillTransform(Reducer* c) const {
+    return op->Tag() == EXPR_VECTOR_CONSTRUCTOR && op->GetType<VectorType>()->IsUnspecifiedVector();
+}
+
+ExprPtr VectorCoerceExpr::Reduce(Reducer* c, StmtPtr& red_stmt) {
+    if ( WillTransform(c) ) {
+        auto op1_list = op->GetOp1();
+        ASSERT(op1_list->Tag() == EXPR_LIST);
+        auto empty_list = cast_intrusive<ListExpr>(op1_list);
+        auto new_me = make_intrusive<VectorConstructorExpr>(empty_list, type);
+        auto red_e = new_me->Reduce(c, red_stmt);
+        return TransformMe(std::move(red_e), c, red_stmt);
+    }
+
+    return UnaryExpr::Reduce(c, red_stmt);
+}
+
 ExprPtr ScheduleExpr::Duplicate() {
     auto when_d = when->Duplicate();
     auto event_d = event->Duplicate()->AsEventExprPtr();

src/script_opt/FuncInfo.cc

@@ -550,22 +550,23 @@ static std::unordered_map<std::string, unsigned int> func_attrs = {
 // Some of these have side effects that could be checked for in a specific
 // context, but the gains from doing so likely aren't worth the complexity.
 
-bool is_special_script_func(std::string func_name) {
+bool is_special_script_func(const std::string& func_name) {
     auto f_attr = func_attrs.find(func_name);
     return f_attr != func_attrs.end() && (f_attr->second & ATTR_SPECIAL_SCRIPT_FUNC) != 0;
 }
 
-bool is_idempotent(std::string func_name) {
+bool is_idempotent(const std::string& func_name) {
     auto f_attr = func_attrs.find(func_name);
     return f_attr != func_attrs.end() && (f_attr->second & ATTR_IDEMPOTENT) != 0;
 }
 
-bool has_no_script_side_effects(std::string func_name) {
+bool has_script_side_effects(const std::string& func_name) {
     auto f_attr = func_attrs.find(func_name);
     if ( f_attr == func_attrs.end() )
-        return false;
+        // We don't know about it, so be conservative.
+        return true;
 
-    return (f_attr->second & (ATTR_NO_SCRIPT_SIDE_EFFECTS | ATTR_NO_ZEEK_SIDE_EFFECTS | ATTR_IDEMPOTENT)) != 0;
+    return (f_attr->second & (ATTR_NO_SCRIPT_SIDE_EFFECTS | ATTR_NO_ZEEK_SIDE_EFFECTS | ATTR_IDEMPOTENT)) == 0;
 }
 
 } // namespace zeek::detail

src/script_opt/FuncInfo.h

@@ -10,15 +10,15 @@ namespace zeek::detail {
 
 // A "special script function" is one that the event engine explicitly
 // knows about.
-extern bool is_special_script_func(std::string func_name);
+extern bool is_special_script_func(const std::string& func_name);
 
 // An idempotent function returns the same value when called with the
 // same arguments (and has no meaningful side effects in terms of script-level
 // or Zeek-internal state).
-extern bool is_idempotent(std::string func_name);
+extern bool is_idempotent(const std::string& func_name);
 
-// Whether the given function (currently, just BiFs) has no Zeek-script-level
+// Whether the given function (currently, just BiFs) has Zeek-script-level
 // side effects.
-extern bool has_no_script_side_effects(std::string func_name);
+extern bool has_script_side_effects(const std::string& func_name);
 
 } // namespace zeek::detail

src/script_opt/GenIDDefs.cc

@@ -50,7 +50,7 @@ void GenIDDefs::TraverseFunction(const FuncPtr& f, ScopePtr scope, StmtPtr body)
 }
 
 TraversalCode GenIDDefs::PreStmt(const Stmt* s) {
-    curr_stmt = s;
+    last_stmt_traversed = s;
 
     auto si = s->GetOptInfo();
     si->stmt_num = ++stmt_num;
@@ -122,11 +122,11 @@ TraversalCode GenIDDefs::PreStmt(const Stmt* s) {
 
             t_branch->Traverse(this);
             if ( ! t_branch->NoFlowAfter(false) )
-                BranchBeyond(curr_stmt, s, true);
+                BranchBeyond(last_stmt_traversed, s, true);
 
             f_branch->Traverse(this);
             if ( ! f_branch->NoFlowAfter(false) )
-                BranchBeyond(curr_stmt, s, true);
+                BranchBeyond(last_stmt_traversed, s, true);
 
             EndConfluenceBlock(true);
 
@@ -155,7 +155,7 @@ TraversalCode GenIDDefs::PreStmt(const Stmt* s) {
             body->Traverse(this);
 
             if ( ! body->NoFlowAfter(false) )
-                BranchBackTo(curr_stmt, s, true);
+                BranchBackTo(last_stmt_traversed, s, true);
 
             EndConfluenceBlock();
 
@@ -183,7 +183,7 @@ TraversalCode GenIDDefs::PreStmt(const Stmt* s) {
             body->Traverse(this);
 
             if ( ! body->NoFlowAfter(false) )
-                BranchBackTo(curr_stmt, s, true);
+                BranchBackTo(last_stmt_traversed, s, true);
 
             EndConfluenceBlock();
 
@@ -242,7 +242,7 @@ TraversalCode GenIDDefs::PostStmt(const Stmt* s) {
 
         case STMT_RETURN: ReturnAt(s); break;
 
-        case STMT_NEXT: BranchBackTo(curr_stmt, FindLoop(), false); break;
+        case STMT_NEXT: BranchBackTo(last_stmt_traversed, FindLoop(), false); break;
 
         case STMT_BREAK: {
             auto target = FindBreakTarget();
@@ -403,8 +403,9 @@ void GenIDDefs::CheckVarUsage(const Expr* e, const ID* id) {
 
     auto oi = id->GetOptInfo();
 
-    if ( ! oi->DidUndefinedWarning() && ! oi->IsDefinedBefore(curr_stmt) && ! id->GetAttr(ATTR_IS_ASSIGNED) ) {
-        if ( ! oi->IsPossiblyDefinedBefore(curr_stmt) ) {
+    if ( ! oi->DidUndefinedWarning() && ! oi->IsDefinedBefore(last_stmt_traversed) &&
+         ! id->GetAttr(ATTR_IS_ASSIGNED) ) {
+        if ( ! oi->IsPossiblyDefinedBefore(last_stmt_traversed) ) {
             e->Warn("used without definition");
             oi->SetDidUndefinedWarning();
         }
@@ -423,7 +424,7 @@ void GenIDDefs::StartConfluenceBlock(const Stmt* s) {
 
 void GenIDDefs::EndConfluenceBlock(bool no_orig) {
     for ( auto id : modified_IDs.back() )
-        id->GetOptInfo()->ConfluenceBlockEndsAfter(curr_stmt, no_orig);
+        id->GetOptInfo()->ConfluenceBlockEndsAfter(last_stmt_traversed, no_orig);
 
     confluence_blocks.pop_back();
     modified_IDs.pop_back();
@@ -489,7 +490,7 @@ void GenIDDefs::TrackID(const ID* id, const ExprPtr& e) {
     // here to set the lowest limit for definitions. For now we leave
     // DefinedAfter as capable of supporting that distinction in case we
     // find need to revive it in the future.
-    oi->DefinedAfter(curr_stmt, e, confluence_blocks, 0);
+    oi->DefinedAfter(last_stmt_traversed, e, confluence_blocks, 0);
 
     // Ensure we track this identifier across all relevant
     // confluence regions.

src/script_opt/GenIDDefs.h

@@ -82,8 +82,8 @@ private:
     // outer "break" in that context.
     FunctionFlavor func_flavor;
 
-    // The statement we are currently traversing.
-    const Stmt* curr_stmt = nullptr;
+    // The most recently traversed statement.
+    const Stmt* last_stmt_traversed = nullptr;
 
     // Used to number Stmt objects found during AST traversal.
     int stmt_num;

src/script_opt/IDOptInfo.cc

@@ -185,7 +185,6 @@ void IDOptInfo::BranchBackTo(const Stmt* from, const Stmt* to, bool close_all) {
             auto& ur = usage_regions[i];
 
             if ( ur.DefinedAfter() < new_def ) {
-                ASSERT(ur.DefinedAfter() != NO_DEF);
                 ur.UpdateDefinedAfter(new_def);
                 ur.SetDefExpr(nullptr);
             }

src/script_opt/ProfileFunc.cc

@@ -635,7 +635,7 @@ bool ProfileFuncs::GetCallSideEffects(const NameExpr* n, IDSet& non_local_ids, T
 
     auto func = fv->AsFunc();
     if ( func->GetKind() == Func::BUILTIN_FUNC ) {
-        if ( ! has_no_script_side_effects(func->Name()) )
+        if ( has_script_side_effects(func->Name()) )
             is_unknown = true;
         return true;
     }
@@ -1168,7 +1168,7 @@ bool ProfileFuncs::DefinitelyHasNoSideEffects(const ExprPtr& e) const {
         return false;
 
     for ( auto& b : pf->BiFGlobals() )
-        if ( ! has_no_script_side_effects(b->Name()) )
+        if ( has_script_side_effects(b->Name()) )
             return false;
 
     return true;
@@ -1259,7 +1259,7 @@ bool ProfileFuncs::AssessSideEffects(const ProfileFunc* pf, IDSet& non_local_ids
     }
 
     for ( auto& b : pf->BiFGlobals() )
-        if ( ! has_no_script_side_effects(b->Name()) ) {
+        if ( has_script_side_effects(b->Name()) ) {
             is_unknown = true;
             return true;
         }

src/script_opt/Reduce.cc

@@ -124,6 +124,14 @@ StmtPtr Reducer::GenParam(const IDPtr& id, ExprPtr rhs, bool is_modified) {
         // the inline block's execution.
         is_modified = true;
 
+    auto& id_t = id->GetType();
+    if ( id_t->Tag() == TYPE_VECTOR && rhs->GetType()->Yield() != id_t->Yield() )
+        // Presumably either the identifier or the RHS is a vector-of-any.
+        // This means there will essentially be a modification of the RHS
+        // due to the need to use (or omit) operations coercing from such
+        // vectors.
+        is_modified = true;
+
     if ( ! is_modified ) {
         // Can use a temporary variable, which then supports
         // optimization via alias propagation.

src/script_opt/ScriptOpt.cc

@@ -507,9 +507,6 @@ static void analyze_scripts_for_ZAM() {
 void clear_script_analysis() {
     IDOptInfo::ClearGlobalInitExprs();
 
-    // Keep the functions around if we're debugging, so we can
-    // generate profiles.
-#ifndef DEBUG
     // We need to explicitly clear out the optimization information
     // associated with identifiers.  They have reference loops with
     // the parent identifier that will prevent reclamation of the
@@ -519,8 +516,10 @@ void clear_script_analysis() {
         for ( auto& id : f.Scope()->OrderedVars() )
             id->ClearOptInfo();
 
-    funcs.clear();
-#endif
+    // Keep the functions around if we're profiling, so we can loop
+    // over them to generate the profiles.
+    if ( ! analysis_options.profile_ZAM )
+        funcs.clear();
 
     non_recursive_funcs.clear();
     lambdas.clear();

src/script_opt/ZAM/AM-Opt.cc

@@ -158,15 +158,15 @@ bool ZAMCompiler::RemoveDeadCode() {
         }
 
         if ( t && t->inst_num > i0->inst_num && (! i1 || t->inst_num <= i1->inst_num) ) {
-            // This is effectively a branch to the next
-            // instruction.  Even if i0 is conditional, there's
-            // no point executing it because regardless of the
-            // outcome of the conditional, we go to the next
-            // successive live instruction (and we don't have
-            // conditionals with side effects).
-            KillInst(i0);
-            did_removal = true;
-            continue;
+            // This is effectively a branch to the next instruction.
+            // We can remove it *unless* the instruction has side effects.
+            // Conditionals don't, but loop-iteration-advancement
+            // instructions do.
+            if ( ! i0->IsLoopIterationAdvancement() ) {
+                KillInst(i0);
+                did_removal = true;
+                continue;
+            }
         }
 
         if ( i0->DoesNotContinue() && i1 && i1->num_labels == 0 ) {

src/script_opt/ZAM/BuiltInSupport.cc

@@ -7,7 +7,7 @@
 
 namespace zeek::detail {
 
-FixedCatArg::FixedCatArg(const TypePtr& _t) : t(_t) {
+FixedCatArg::FixedCatArg(TypePtr _t) : t(std::move(_t)) {
     switch ( t->Tag() ) {
         case TYPE_BOOL: max_size = 1; break;
 
@@ -27,13 +27,13 @@ FixedCatArg::FixedCatArg(const TypePtr& _t) : t(_t) {
         case TYPE_ENUM: {
             size_t n = 0;
             for ( const auto& e : t->AsEnumType()->Names() )
-                n += e.first.size();
+                n = std::max(n, e.first.size());
             max_size = n;
             break;
         }
 
         case TYPE_PORT:
-            max_size = 5 + 1 + 7; // <number> + / + "unknown
+            max_size = 5 + 1 + 7; // <number> + / + "unknown"
             break;
 
         case TYPE_ADDR:

src/script_opt/ZAM/BuiltInSupport.h

@@ -36,12 +36,12 @@ protected:
 
 class FixedCatArg : public CatArg {
 public:
-    FixedCatArg(const TypePtr& t);
+    FixedCatArg(TypePtr t);
 
     void RenderInto(ZVal* zframe, int slot, char*& res) override;
 
 protected:
-    const TypePtr& t;
+    TypePtr t;
     char tmp[256];
 };
 
@@ -80,7 +80,7 @@ protected:
 
 class DescCatArg : public CatArg {
 public:
-    DescCatArg(const TypePtr& _t) : CatArg(), t(_t) { d.SetStyle(RAW_STYLE); }
+    DescCatArg(TypePtr _t) : CatArg(), t(std::move(_t)) { d.SetStyle(RAW_STYLE); }
 
     void RenderInto(ZVal* zframe, int slot, char*& res) override {
         auto n = d.Len();

src/script_opt/ZAM/Compile.h

@@ -110,11 +110,14 @@ private:
     template<typename T>
     using CaseMapsI = std::vector<CaseMapI<T>>;
 
+    template<typename T>
+    void AdjustSwitchTables(CaseMapsI<T>& abstract_cases);
+
     template<typename T>
     void ConcretizeSwitchTables(const CaseMapsI<T>& abstract_cases, CaseMaps<T>& concrete_cases);
 
     template<typename T>
-    void DumpCases(const T& cases, const char* type_name) const;
+    void DumpCases(const CaseMaps<T>& cases, const char* type_name) const;
     void DumpInsts1(const FrameReMap* remappings);
 
 #include "zeek/ZAM-MethodDecls.h"

src/script_opt/ZAM/Driver.cc

@@ -264,6 +264,20 @@ void ZAMCompiler::AdjustBranches() {
         if ( auto t = inst->target )
             inst->target = FindLiveTarget(t);
     }
+
+    // Fix up the implicit branches in switches, too.
+    AdjustSwitchTables(int_casesI);
+    AdjustSwitchTables(uint_casesI);
+    AdjustSwitchTables(double_casesI);
+    AdjustSwitchTables(str_casesI);
+}
+
+template<typename T>
+void ZAMCompiler::AdjustSwitchTables(CaseMapsI<T>& abstract_cases) {
+    for ( auto& targs : abstract_cases ) {
+        for ( auto& targ : targs )
+            targ.second = FindLiveTarget(targ.second);
+    }
 }
 
 void ZAMCompiler::RetargetBranches() {
@@ -386,14 +400,14 @@ void ZAMCompiler::Dump() {
         inst->Dump(&frame_denizens, remappings);
     }
 
-    DumpCases(int_casesI, "int");
-    DumpCases(uint_casesI, "uint");
-    DumpCases(double_casesI, "double");
-    DumpCases(str_casesI, "str");
+    DumpCases(int_cases, "int");
+    DumpCases(uint_cases, "uint");
+    DumpCases(double_cases, "double");
+    DumpCases(str_cases, "str");
 }
 
 template<typename T>
-void ZAMCompiler::DumpCases(const T& cases, const char* type_name) const {
+void ZAMCompiler::DumpCases(const CaseMaps<T>& cases, const char* type_name) const {
     for ( auto i = 0U; i < cases.size(); ++i ) {
         printf("%s switch table #%d:", type_name, i);
         for ( auto& m : cases[i] ) {
@@ -404,7 +418,7 @@ void ZAMCompiler::DumpCases(const T& cases, const char* type_name) const {
                                 std::is_same_v<T, double> )
                 case_val = std::to_string(m.first);
 
-            printf(" %s->%d", case_val.c_str(), m.second->inst_num);
+            printf(" %s->%d", case_val.c_str(), m.second);
         }
         printf("\n");
     }

src/script_opt/ZAM/IterInfo.h

@@ -27,7 +27,7 @@ public:
     // Start looping over the elements of the given table.  "_aux"
     // provides information about the index variables, their types,
     // and the type of the value variable (if any).
-    void BeginLoop(const TableVal* _tv, ZInstAux* _aux) {
+    void BeginLoop(TableValPtr _tv, ZInstAux* _aux) {
         tv = _tv;
         aux = _aux;
         auto tvd = tv->AsTable();
@@ -76,10 +76,7 @@ public:
     }
 
 private:
-    // The table we're looping over.  If we want to allow for the table
-    // going away before we're able to clear our iterators then we
-    // could change this to non-const and use Ref/Unref.
-    const TableVal* tv = nullptr;
+    TableValPtr tv = nullptr;
 
     // Associated auxiliary information.
     ZInstAux* aux = nullptr;

src/script_opt/ZAM/Ops.in

@@ -1859,7 +1859,7 @@ internal-op Init-Table-Loop
 type VV
 op1-read
 eval	auto& ti = (*tiv_ptr)[z.v2];
-	ti.BeginLoop(frame[z.v1].table_val, z.aux);
+	ti.BeginLoop({NewRef{}, frame[z.v1].table_val}, z.aux);
 
 internal-op Next-Table-Iter
 op1-read
@@ -2446,7 +2446,8 @@ type VV
 eval	Cat1OpRef(frame[z.v2])
 
 macro Cat1FullVal(val)
-	Cat1Op(ZVal(ZAM_val_cat(val.ToVal(z.t))))
+	auto formatted_val = ZVal(ZAM_val_cat(val.ToVal(z.t)));
+	Cat1Op(formatted_val)
 
 internal-op Cat1Full
 type VC

src/script_opt/ZAM/Stmt.cc

@@ -681,13 +681,10 @@ const ZAMStmt ZAMCompiler::LoopOverTable(const ForStmt* f, const NameExpr* val)
     auto value_var = f->ValueVar();
     auto body = f->LoopBody();
 
-    // Check whether the loop variables are actually used in the body.
-    // This is motivated by an idiom where there's both loop_vars and
-    // a value_var, but the script only actually needs the value_var;
-    // and also some weird cases where the script is managing a
-    // separate iteration process manually.
-    ProfileFunc body_pf(body);
-
+    // We used to have more involved logic here to check whether the loop
+    // variables are actually used in the body. Now that we have '_'
+    // loop placeholder variables, this is no longer worth trying to
+    // optimize for, though we still optimize for those placeholders.
     int num_unused = 0;
 
     auto aux = new ZInstAux(0);
@@ -695,7 +692,7 @@ const ZAMStmt ZAMCompiler::LoopOverTable(const ForStmt* f, const NameExpr* val)
     for ( auto i = 0; i < loop_vars->length(); ++i ) {
         auto id = (*loop_vars)[i];
 
-        if ( body_pf.Locals().count(id) == 0 || id->IsBlank() )
+        if ( id->IsBlank() )
             ++num_unused;
 
         int slot = id->IsBlank() ? -1 : FrameSlot(id);
@@ -707,12 +704,6 @@ const ZAMStmt ZAMCompiler::LoopOverTable(const ForStmt* f, const NameExpr* val)
 
     bool no_loop_vars = (num_unused == loop_vars->length());
 
-    if ( value_var && body_pf.Locals().count(value_var.get()) == 0 )
-        // This is more clearly a coding botch - someone left in
-        // an unnecessary value_var variable.  But might as
-        // well not do the work.
-        value_var = nullptr;
-
     if ( value_var )
         aux->value_var_type = value_var->GetType();
 

src/script_opt/ZAM/ZInst.cc

@@ -214,6 +214,23 @@ ValPtr ZInst::ConstVal() const {
     return nullptr;
 }
 
+bool ZInst::IsLoopIterationAdvancement() const {
+    switch ( op ) {
+        case OP_NEXT_TABLE_ITER_VV:
+        case OP_NEXT_TABLE_ITER_NO_VARS_VV:
+        case OP_NEXT_TABLE_ITER_VAL_VAR_VVV:
+        case OP_NEXT_TABLE_ITER_VAL_VAR_NO_VARS_VVV:
+        case OP_NEXT_VECTOR_ITER_VVV:
+        case OP_NEXT_VECTOR_BLANK_ITER_VV:
+        case OP_NEXT_VECTOR_ITER_VAL_VAR_VVVV:
+        case OP_NEXT_VECTOR_BLANK_ITER_VAL_VAR_VVV:
+        case OP_NEXT_STRING_ITER_VVV:
+        case OP_NEXT_STRING_BLANK_ITER_VV: return true;
+
+        default: return false;
+    }
+}
+
 string ZInst::ConstDump() const {
     auto v = ConstVal();
 

src/script_opt/ZAM/ZInst.h

@@ -88,6 +88,10 @@ public:
     // Returns nil if this instruction doesn't have an associated constant.
     ValPtr ConstVal() const;
 
+    // Returns true if this instruction represents a form of advancing
+    // a loop iteration, false otherwise.
+    bool IsLoopIterationAdvancement() const;
+
     // Returns a string describing the constant.
     std::string ConstDump() const;
 

testing/btest/Baseline.zam/scripts.policy.misc.dump-events/really-all-events.log

@@ -2,7 +2,6 @@
          0.000000 zeek_init
 XXXXXXXXXX.XXXXXX network_time_init
 XXXXXXXXXX.XXXXXX Broker::log_flush
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_connection
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=56166/udp, resp_h=10.10.1.1, resp_p=53/udp], orig=[size=34, state=1, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=0 secs, service={\x0a\x0a}, history=D, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>]
 
@@ -21,9 +20,6 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=76, cap_len=76, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=62, id=9482, DF=F, MF=F, offset=0, ttl=128, p=17, sum=65420, src=10.10.1.4, dst=10.10.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=56166/udp, dport=53/udp, ulen=42], icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=56166/udp, resp_h=10.10.1.1, resp_p=53/udp], orig=[size=34, state=1, num_pkts=1, num_bytes_ip=62, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=100, state=1, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=34.0 msecs 24.953842 usecs, service={\x0a\x0a}, history=Dd, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>]
@@ -268,7 +264,6 @@ XXXXXXXXXX.XXXXXX tcp_packet
 XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=60, cap_len=60, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=40, id=8675, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17585, src=74.53.140.153, dst=10.10.1.4], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727269, ack=2126795706, hl=20, dl=0, reserved=0, flags=16, win=5840], udp=<uninitialized>, icmp=<uninitialized>]
 
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
@@ -340,9 +335,6 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=191, cap_len=191, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=177, id=8676, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17447, src=74.53.140.153, dst=10.10.1.4], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727269, ack=2126795706, hl=20, dl=137, reserved=0, flags=24, win=5840], udp=<uninitialized>, icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=21, state=4, num_pkts=3, num_bytes_ip=137, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=318, state=4, num_pkts=4, num_bytes_ip=486, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=1.0 sec 39.0 msecs 682.865143 usecs, service={\x0aSMTP\x0a}, history=ShAdDa, uid=ClEkJM2Vm5giqnMf4h, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, cc=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 HELP, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, tls=F, process_received_from=T, has_client_activity=F, process_smtp_headers=T, entity_count=0, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, trans_mail_from_seen=F, trans_rcpt_to_seen=F, invalid_transactions=0, analyzer_id=6, mime_depth=0]]
@@ -428,7 +420,6 @@ XXXXXXXXXX.XXXXXX smtp_request
 XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=84, cap_len=84, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=70, id=9513, DF=T, MF=F, offset=0, ttl=128, p=6, sum=62380, src=10.10.1.4, dst=74.53.140.153], ip6=<uninitialized>, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126795718, ack=2934727424, hl=20, dl=30, reserved=0, flags=24, win=65199], udp=<uninitialized>, icmp=<uninitialized>]
 
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
@@ -459,10 +450,6 @@ XXXXXXXXXX.XXXXXX smtp_reply
 XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=72, cap_len=72, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=58, id=8678, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17564, src=74.53.140.153, dst=10.10.1.4], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727424, ack=2126795748, hl=20, dl=18, reserved=0, flags=24, win=5840], udp=<uninitialized>, icmp=<uninitialized>]
 
-XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
@@ -492,9 +479,6 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=72, cap_len=72, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=58, id=9518, DF=T, MF=F, offset=0, ttl=128, p=6, sum=62387, src=10.10.1.4, dst=74.53.140.153], ip6=<uninitialized>, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126795748, ack=2934727442, hl=20, dl=18, reserved=0, flags=24, win=65181], udp=<uninitialized>, icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX Broker::log_flush
 XXXXXXXXXX.XXXXXX new_packet
@@ -589,7 +573,6 @@ XXXXXXXXXX.XXXXXX smtp_reply
 XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=62, cap_len=62, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=48, id=8680, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17572, src=74.53.140.153, dst=10.10.1.4], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727472, ack=2126795802, hl=20, dl=8, reserved=0, flags=24, win=5840], udp=<uninitialized>, icmp=<uninitialized>]
 
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
@@ -619,9 +602,6 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=93, cap_len=93, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=79, id=9528, DF=T, MF=F, offset=0, ttl=128, p=6, sum=62356, src=10.10.1.4, dst=74.53.140.153], ip6=<uninitialized>, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126795802, ack=2934727480, hl=20, dl=39, reserved=0, flags=24, win=65143], udp=<uninitialized>, icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=144, state=4, num_pkts=8, num_bytes_ip=472, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=406, state=4, num_pkts=8, num_bytes_ip=720, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=2.0 secs 790.0 msecs 662.050247 usecs, service={\x0aSMTP\x0a}, history=ShAdDa, uid=ClEkJM2Vm5giqnMf4h, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, cc=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 OK, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=T, entity_count=0, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=0]]
@@ -685,7 +665,6 @@ XXXXXXXXXX.XXXXXX raw_packet
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX Broker::log_flush
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=150, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=462, state=4, num_pkts=9, num_bytes_ip=774, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=3.0 secs 132.0 msecs 632.97081 usecs, service={\x0aSMTP\x0a}, history=ShAdDa, uid=ClEkJM2Vm5giqnMf4h, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, cc=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 Accepted, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=T, entity_count=1, entity=[filename=<uninitialized>], fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=1]]
                   [1] p: pkt_hdr         = [ip=[hl=20, tos=96, len=96, id=8682, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17522, src=74.53.140.153, dst=10.10.1.4], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727494, ack=2126795847, hl=20, dl=56, reserved=0, flags=24, win=5840], udp=<uninitialized>, icmp=<uninitialized>]
@@ -715,9 +694,6 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=110, cap_len=110, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=96, id=8682, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17522, src=74.53.140.153, dst=10.10.1.4], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727494, ack=2126795847, hl=20, dl=56, reserved=0, flags=24, win=5840], udp=<uninitialized>, icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=3.0 secs 163.0 msecs 697.004318 usecs, service={\x0aSMTP\x0a}, history=ShAdDa, uid=ClEkJM2Vm5giqnMf4h, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, cc=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=T, entity_count=1, entity=[filename=<uninitialized>], fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=1]]
@@ -2934,7 +2910,6 @@ XXXXXXXXXX.XXXXXX tcp_packet
 XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=60, cap_len=60, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=40, id=8684, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17576, src=74.53.140.153, dst=10.10.1.4], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727550, ack=2126798751, hl=20, dl=0, reserved=0, flags=16, win=11616], udp=<uninitialized>, icmp=<uninitialized>]
 
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
@@ -2958,9 +2933,6 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=1506, cap_len=1506, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=1492, id=9558, DF=T, MF=F, offset=0, ttl=128, p=6, sum=60913, src=10.10.1.4, dst=74.53.140.153], ip6=<uninitialized>, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126800203, ack=2934727550, hl=20, dl=1452, reserved=0, flags=16, win=65073], udp=<uninitialized>, icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=7410, state=4, num_pkts=17, num_bytes_ip=12486, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=462, state=4, num_pkts=12, num_bytes_ip=950, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=3.0 secs 940.0 msecs 768.003464 usecs, service={\x0aSMTP\x0a}, history=ShAdDaT, uid=ClEkJM2Vm5giqnMf4h, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={\x0a<raj_deol2002in@yahoo.co.in>\x0a}, cc=<uninitialized>, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=F, entity_count=5, entity=[filename=NEWS.txt], fuids=[FmFp351N5nhsMmAfQg, Fqrb1K5DWEfgy4WU2, FEFYSd1s8Onn9LynKj]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=5]]
@@ -3937,7 +3909,6 @@ XXXXXXXXXX.XXXXXX smtp_data
 XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=1506, cap_len=1506, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=1492, id=9560, DF=T, MF=F, offset=0, ttl=128, p=6, sum=60911, src=10.10.1.4, dst=74.53.140.153], ip6=<uninitialized>, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126803107, ack=2934727550, hl=20, dl=1452, reserved=0, flags=24, win=65073], udp=<uninitialized>, icmp=<uninitialized>]
 
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
@@ -4341,12 +4312,8 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=1506, cap_len=1506, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=1492, id=9561, DF=T, MF=F, offset=0, ttl=128, p=6, sum=60910, src=10.10.1.4, dst=74.53.140.153], ip6=<uninitialized>, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126804559, ack=2934727550, hl=20, dl=1452, reserved=0, flags=16, win=65073], udp=<uninitialized>, icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX Broker::log_flush
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=10314, state=4, num_pkts=20, num_bytes_ip=16962, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=462, state=4, num_pkts=13, num_bytes_ip=990, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=4.0 secs 305.0 msecs 548.906326 usecs, service={\x0aSMTP\x0a}, history=ShAdDaT, uid=ClEkJM2Vm5giqnMf4h, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={\x0a<raj_deol2002in@yahoo.co.in>\x0a}, cc=<uninitialized>, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=F, entity_count=5, entity=[filename=NEWS.txt], fuids=[FmFp351N5nhsMmAfQg, Fqrb1K5DWEfgy4WU2, FEFYSd1s8Onn9LynKj]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=5]]
                   [1] p: pkt_hdr         = [ip=[hl=20, tos=96, len=40, id=8686, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17574, src=74.53.140.153, dst=10.10.1.4], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727550, ack=2126801655, hl=20, dl=0, reserved=0, flags=16, win=17424], udp=<uninitialized>, icmp=<uninitialized>]
@@ -4364,9 +4331,6 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=60, cap_len=60, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=40, id=8686, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17574, src=74.53.140.153, dst=10.10.1.4], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727550, ack=2126801655, hl=20, dl=0, reserved=0, flags=16, win=17424], udp=<uninitialized>, icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=11766, state=4, num_pkts=20, num_bytes_ip=16962, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=462, state=4, num_pkts=14, num_bytes_ip=1030, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=4.0 secs 305.0 msecs 582.046509 usecs, service={\x0aSMTP\x0a}, history=ShAdDaT, uid=ClEkJM2Vm5giqnMf4h, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={\x0a<raj_deol2002in@yahoo.co.in>\x0a}, cc=<uninitialized>, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=F, entity_count=5, entity=[filename=NEWS.txt], fuids=[FmFp351N5nhsMmAfQg, Fqrb1K5DWEfgy4WU2, FEFYSd1s8Onn9LynKj]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=5]]
@@ -6044,7 +6008,6 @@ XXXXXXXXXX.XXXXXX connection_EOF
 XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=54, cap_len=54, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=40, id=9574, DF=T, MF=F, offset=0, ttl=128, p=6, sum=62349, src=10.10.1.4, dst=74.53.140.153], ip6=<uninitialized>, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126810402, ack=2934727578, hl=20, dl=0, reserved=0, flags=17, win=65045], udp=<uninitialized>, icmp=<uninitialized>]
 
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
@@ -6076,9 +6039,6 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=102, cap_len=102, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=88, id=8695, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17517, src=74.53.140.153, dst=10.10.1.4], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727578, ack=2126810402, hl=20, dl=48, reserved=0, flags=24, win=34848], udp=<uninitialized>, icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=14705, state=5, num_pkts=27, num_bytes_ip=21633, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=538, state=5, num_pkts=23, num_bytes_ip=1466, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=7.0 secs 576.0 msecs 952.934265 usecs, service={\x0aSMTP\x0a}, history=ShAdDaTFf, uid=ClEkJM2Vm5giqnMf4h, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=2, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, cc=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=221 xc90.websitewelcome.com closing connection, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, tls=F, process_received_from=T, has_client_activity=F, process_smtp_headers=T, entity_count=0, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=1, pending_messages=<uninitialized>, trans_mail_from_seen=F, trans_rcpt_to_seen=F, invalid_transactions=0, analyzer_id=6, mime_depth=5]]
@@ -6530,7 +6490,6 @@ XXXXXXXXXX.XXXXXX smtp_reply
 XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=101, cap_len=101, src=00:08:ca:cc:ad:4c, dst=58:b0:35:86:54:8d, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=87, id=51483, DF=T, MF=F, offset=0, ttl=64, p=6, sum=58729, src=192.168.133.102, dst=192.168.133.100], ip6=<uninitialized>, tcp=[sport=25/tcp, dport=49648/tcp, seq=2131788255, ack=3976465341, hl=32, dl=35, reserved=0, flags=24, win=227], udp=<uninitialized>, icmp=<uninitialized>]
 
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
@@ -6573,9 +6532,6 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=58:b0:35:86:54:8d, dst=00:08:ca:cc:ad:4c, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=57836, DF=T, MF=F, offset=0, ttl=64, p=6, sum=52411, src=192.168.133.100, dst=192.168.133.102], ip6=<uninitialized>, tcp=[sport=49648/tcp, dport=25/tcp, seq=3976465341, ack=2131788290, hl=32, dl=0, reserved=0, flags=16, win=4116], udp=<uninitialized>, icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
                   [0] c: connection      = [id=[orig_h=192.168.133.100, orig_p=49648/tcp, resp_h=192.168.133.102, resp_p=25/tcp], orig=[size=24, state=4, num_pkts=3, num_bytes_ip=168, flow_label=0, l2_addr=58:b0:35:86:54:8d], resp=[size=35, state=4, num_pkts=2, num_bytes_ip=147, flow_label=0, l2_addr=00:08:ca:cc:ad:4c], start_time=XXXXXXXXXX.XXXXXX, duration=26.0 msecs 411.056519 usecs, service={\x0a\x0a}, history=ShAdD, uid=CmES5u32sYpV7JYN, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=CmES5u32sYpV7JYN, id=[orig_h=192.168.133.100, orig_p=49648/tcp, resp_h=192.168.133.102, resp_p=25/tcp], trans_depth=1, helo=<uninitialized>, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, cc=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=220 uprise ESMTP SubEthaSMTP null, path=[192.168.133.102, 192.168.133.100], user_agent=<uninitialized>, tls=F, process_received_from=T, has_client_activity=F, process_smtp_headers=T, entity_count=0, entity=<uninitialized>, fuids=[]], smtp_state=[helo=<uninitialized>, messages_transferred=0, pending_messages=<uninitialized>, trans_mail_from_seen=F, trans_rcpt_to_seen=F, invalid_transactions=0, analyzer_id=<uninitialized>, mime_depth=0]]
@@ -8171,7 +8127,6 @@ XXXXXXXXXX.XXXXXX tcp_options
 XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=58:b0:35:86:54:8d, dst=cc:b2:55:f4:62:92, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=6886, DF=T, MF=F, offset=0, ttl=64, p=6, sum=18327, src=192.168.133.100, dst=74.125.71.189], ip6=<uninitialized>, tcp=[sport=49336/tcp, dport=443/tcp, seq=1584252430, ack=3980080343, hl=32, dl=0, reserved=0, flags=16, win=4093], udp=<uninitialized>, icmp=<uninitialized>]
 
-XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_packet
@@ -8214,9 +8169,6 @@ XXXXXXXXXX.XXXXXX raw_packet
                   [0] p: raw_pkt_hdr     = [l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=58:b0:35:86:54:8d, dst=cc:b2:55:f4:62:92, vlan=<uninitialized>, inner_vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=3407, DF=T, MF=F, offset=0, ttl=64, p=6, sum=21806, src=192.168.133.100, dst=74.125.71.189], ip6=<uninitialized>, tcp=[sport=49336/tcp, dport=443/tcp, seq=1584252430, ack=3980080343, hl=32, dl=0, reserved=0, flags=16, win=4093], udp=<uninitialized>, icmp=<uninitialized>]
 
 XXXXXXXXXX.XXXXXX event_queue_flush_point
-XXXXXXXXXX.XXXXXX load_sample
-                  [2] dmem: int          = 0
-
 XXXXXXXXXX.XXXXXX event_queue_flush_point
 XXXXXXXXXX.XXXXXX new_connection_contents
                   [0] c: connection      = [id=[orig_h=192.168.133.100, orig_p=49153/tcp, resp_h=17.172.238.21, resp_p=5223/tcp], orig=[size=714, state=3, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=58:b0:35:86:54:8d], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=cc:b2:55:f4:62:92], start_time=XXXXXXXXXX.XXXXXX, duration=0 secs, service={\x0a\x0a}, history=D, uid=C37jN32gN3y3AZzyf6, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, dpd_state=<uninitialized>, service_violation={\x0a\x0a}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, removal_hooks=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>]

testing/btest/Baseline/language.spurious-table-expires/out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expire, new, 42

testing/btest/Baseline/opt.confluence-test/output

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+0

testing/btest/Baseline/opt.table-choose/output

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+2, 12

testing/btest/Baseline/opt.vector-any-concretize1/output

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[5, 3, 9]

testing/btest/Baseline/opt.vector-any-concretize2/output

@@ -0,0 +1,9 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[0.0]
+[0.0, 1.0]
+[0.0, 1.0, 2.0]
+[0.0, 1.0, 2.0, 3.0]
+[1.0, 2.0, 3.0, 3.0]
+[2.0, 3.0, 3.0, 3.0]
+[3.0, 3.0, 3.0, 3.0]
+[3.0, 3.0, 3.0, 3.0]

testing/btest/language/spurious-table-expires.zeek

@@ -0,0 +1,28 @@
+# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+# Default timer expiration interval is very conservative (10sec) and never runs for short pcaps.
+redef table_expire_interval = 0.01sec;
+
+function f(t: table[string] of count, k: string): interval
+	{
+	print "expire", k, t[k];
+	return 0.0sec;
+	}
+
+global t: table[string] of count &create_expire=0.1sec &expire_func=f;
+
+# Populate the initial table with two entries.
+event zeek_init() &priority=5
+	{
+	t["a"] = 10;
+	t["b"] = 20;
+	}
+
+# Replace global t, deleting all entries. In a DEBUG build, table continued
+# to exist and its entries spuriously expired over time.
+event zeek_init()
+	{
+	t = table() &create_expire=0.1sec &expire_func=f;
+	t["new"] = 42;
+	}

testing/btest/opt/confluence-test.zeek

@@ -0,0 +1,28 @@
+# @TEST-DOC: Regression test of ZAM analysis of complex variable "confluence".
+# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
+#
+# @TEST-EXEC: zeek -b -O ZAM %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+global my_T: bool;
+
+event zeek_init()
+	{
+	local vi: vector of int;
+	local outer_var: int;
+	outer_var = 0;
+
+	# This used to throw an assertion regarding the usage regions
+	# associated with outer_var.
+	for ( i in vi )
+		for ( n in vi )
+			if ( my_T )
+				break;
+			else
+				{
+				outer_var = 1;
+				break;
+				}
+
+	print outer_var;
+	}

testing/btest/opt/table-choose.zeek

@@ -0,0 +1,14 @@
+# @TEST-DOC: Regression test for past ZAM issues with for-loop table "choose".
+# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
+#
+# @TEST-EXEC: zeek -b -O ZAM %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+event zeek_init()
+	{
+	local v = table([1] = 4, [2] = 12);
+	for ( i1, i2 in v )
+		break;
+
+	print i1, i2;
+	}

testing/btest/opt/vector-any-concretize1.zeek

@@ -0,0 +1,26 @@
+# @TEST-DOC: Regression test for past ZAM issues with vector-of-any.
+# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
+#
+# @TEST-EXEC: zeek -b -O ZAM %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+function vector_copy(v: vector of any): vector of any
+	{
+	# This seems like an unnecessary initialization given the ensuing
+	# copy, but we preserve it because it's from the original script
+	# that triggered the need for some fixes, hence it's the full
+	# regression.
+	local v2 = copy(v);
+
+	for ( i in v )
+		v2[i] = v[i];
+
+	return v2;
+	}
+
+event zeek_init()
+	{
+	local v = vector(5, 3, 9);
+	local v_copy = vector_copy(v);
+	print v_copy;
+	}

testing/btest/opt/vector-any-concretize2.zeek

@@ -0,0 +1,29 @@
+# @TEST-DOC: Regression test for past ZAM issues with vector-of-any.
+# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
+#
+# @TEST-EXEC: zeek -b -O ZAM %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+global d: table[string] of vector of double &default=vector();
+
+function crank_one(key: string)
+	{
+	local c = d[key];
+	c += |c|;
+	print c;
+	if ( |c| > 3 )
+		c = c[1:];
+	d[key] = c;
+	}
+
+event zeek_init()
+	{
+	crank_one("foo");
+	crank_one("foo");
+	crank_one("foo");
+	crank_one("foo");
+	crank_one("foo");
+	crank_one("foo");
+	crank_one("foo");
+	crank_one("foo");
+	}