diff --git a/CHANGES b/CHANGES index 4649be9832..ed8adb5e60 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,37 @@ +6.2.0-dev.396 | 2024-01-15 15:17:45 +0100 + + * ZAM speedup for constructing empty vectors (Vern Paxson, Corelight) + + * fixes for ZAM optimization of "switch" statements (Vern Paxson, Corelight) + + * BTests to catch regressions for recent ZAM fixes (Vern Paxson, Corelight) + + * "-a zam" BTest baseline update for recent changes (Vern Paxson, Corelight) + + * fix for needing to always flush optimization information for identifiers (Vern Paxson, Corelight) + + * fix for logic bug in ldap base script (Vern Paxson, Corelight) + + * better name for key variable in script optimization (Vern Paxson, Corelight) + + * ZAM fix for tracking variable usage (Vern Paxson, Corelight) + + * ZAM fixes for "for" loops that are only used to choose an element from a table/set (Vern Paxson, Corelight) + + * ZAM fixes for loops indexed with variables not used in the loop body (Vern Paxson, Corelight) + + * fix for ZAM location tracking - more extensive changes are pending (Vern Paxson, Corelight) + + * fixes for ZAM's special-casing of that "cat" BiF (Vern Paxson, Corelight) + + * some fixes for ZAM memory management (Vern Paxson, Corelight) + + * streamlining of some script optimization APIs (Vern Paxson, Corelight) + + * fixes for initializations of "-O gen-C++" script compilations (Vern Paxson, Corelight) + + * script optimization fixes for "concretizing" vector-of-any's (Vern Paxson, Corelight) + 6.2.0-dev.378 | 2024-01-12 10:48:33 +0100 * btest/smtp: Test with smtp-bdat-pipeline-8bitmime.pcap (Arne Welzel, Corelight) diff --git a/VERSION b/VERSION index b55b0a5a7b..a387f23a07 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.2.0-dev.378 +6.2.0-dev.396 diff --git a/scripts/base/protocols/ldap/main.zeek b/scripts/base/protocols/ldap/main.zeek index 417d6afad0..93c301a65a 100644 --- a/scripts/base/protocols/ldap/main.zeek +++ b/scripts/base/protocols/ldap/main.zeek @@ -203,7 +203,7 @@ event LDAP::message(c: connection, } if ( diagnostic_message != "" ) { - if ( ! sm?$diagnostic_message && sm$diagnostic_message != diagnostic_message ) { + if ( sm?$diagnostic_message && sm$diagnostic_message != diagnostic_message ) { Reporter::conn_weird("LDAP_search_diagnostic_message_change", c, fmt("%s: %s -> %s", message_id, sm$diagnostic_message, diagnostic_message), "LDAP"); } diff --git a/src/Expr.h b/src/Expr.h index a45a1528b8..1a56cddea3 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -1310,6 +1310,10 @@ public: // Optimization-related: ExprPtr Duplicate() override; + bool IsReduced(Reducer* c) const override; + bool WillTransform(Reducer* c) const override; + ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override; + protected: ValPtr Fold(Val* v) const override; }; diff --git a/src/Type.cc b/src/Type.cc index fec2544e2d..925f46191e 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -902,6 +902,9 @@ public: if ( coerce_type ) v = v->AsRecordVal()->CoerceTo(coerce_type); + else if ( init_type->Tag() == TYPE_VECTOR ) + concretize_if_unspecified(cast_intrusive(v), init_type->Yield()); + return ZVal(v, init_type); } diff --git a/src/Type.h b/src/Type.h index dcdbc09e0d..fafa79dee1 100644 --- a/src/Type.h +++ b/src/Type.h @@ -366,6 +366,15 @@ public: protected: IndexType(TypeTag t, TypeListPtr arg_indices, TypePtr arg_yield_type) : Type(t), indices(std::move(arg_indices)), yield_type(std::move(arg_yield_type)) { + // "indices" might be nil if we're deferring construction of the type + // for "-O use-C++" initialization. + if ( indices ) + SetSpecialIndices(); + else + is_subnet_index = is_pattern_index = false; // placeholders + } + + void SetSpecialIndices() { const auto& types = indices->GetTypes(); is_subnet_index = types.size() == 1 && types[0]->Tag() == TYPE_SUBNET; is_pattern_index = types.size() == 1 && types[0]->Tag() == TYPE_PATTERN; diff --git a/src/Val.cc b/src/Val.cc index cb86351ae2..bd0068daeb 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -1889,45 +1889,56 @@ ValPtr TableVal::Default(const ValPtr& index) { return nullptr; } + ValPtr result; + if ( def_val->GetType()->Tag() != TYPE_FUNC || same_type(def_val->GetType(), GetType()->Yield()) ) { if ( def_attr->GetExpr()->IsConst() ) return def_val; try { - return def_val->Clone(); + result = def_val->Clone(); } catch ( InterpreterException& e ) { /* Already reported. */ } - Error("&default value for table is not clone-able"); - return nullptr; + if ( ! result ) { + Error("&default value for table is not clone-able"); + return nullptr; + } + } + else { + const Func* f = def_val->AsFunc(); + Args vl; + + if ( index->GetType()->Tag() == TYPE_LIST ) { + auto lv = index->AsListVal(); + vl.reserve(lv->Length()); + + for ( const auto& v : lv->Vals() ) + vl.emplace_back(v); + } + else + vl.emplace_back(index); + + try { + result = f->Invoke(&vl); + } + + catch ( InterpreterException& e ) { /* Already reported. */ + } + + if ( ! result ) { + Error("no value returned from &default function"); + return nullptr; + } } - const Func* f = def_val->AsFunc(); - Args vl; - - if ( index->GetType()->Tag() == TYPE_LIST ) { - auto lv = index->AsListVal(); - vl.reserve(lv->Length()); - - for ( const auto& v : lv->Vals() ) - vl.emplace_back(v); - } - else - vl.emplace_back(index); - - ValPtr result; - - try { - result = f->Invoke(&vl); - } - - catch ( InterpreterException& e ) { /* Already reported. */ - } - - if ( ! result ) { - Error("no value returned from &default function"); - return nullptr; - } + auto rt = result->GetType(); + if ( rt->Tag() == TYPE_VECTOR ) + // The double-Yield() here is because this is a "table of vector of X" + // and we want X. If this is instead a "table of any", that'll be + // okay because concretize_if_unspecified() correctly deals with + // nil target types. + detail::concretize_if_unspecified(cast_intrusive(result), GetType()->Yield()->Yield()); return result; } @@ -3474,6 +3485,26 @@ bool VectorVal::Concretize(const TypePtr& t) { return true; } +void detail::concretize_if_unspecified(VectorValPtr v, TypePtr t) { + if ( v->Size() != 0 ) + // Concretization only applies to empty vectors. + return; + + if ( v->GetType()->Yield()->Tag() != TYPE_ANY ) + // It's not an unspecified vector. + return; + + if ( ! t ) + // "t" can be nil if the vector is being assigned to an "any" value. + return; + + if ( t->Tag() == TYPE_ANY ) + // No need to concretize. + return; + + v->Concretize(t); +} + unsigned int VectorVal::ComputeFootprint(std::unordered_set* analyzed_vals) const { auto n = vector_val.size(); unsigned int fp = n; diff --git a/src/Val.h b/src/Val.h index 6318642fd8..7a6bdf0379 100644 --- a/src/Val.h +++ b/src/Val.h @@ -1744,6 +1744,13 @@ namespace detail { // for normalization. If Func::nil is passed, no normalization happens. extern std::variant ValFromJSON(std::string_view json_str, const TypePtr& t, const FuncPtr& key_func); + +// If the given vector is an empty vector-of-any ("unspecified"), +// concretizes it to the given type. *v* gives the vector and *t* the +// type to concretize it to if appropriate. *t* can be nil, in which +// case nothing is done. +extern void concretize_if_unspecified(VectorValPtr v, TypePtr t); + } // namespace detail } // namespace zeek diff --git a/src/script_opt/CPP/InitsInfo.cc b/src/script_opt/CPP/InitsInfo.cc index c8150dbe96..b887eec1b4 100644 --- a/src/script_opt/CPP/InitsInfo.cc +++ b/src/script_opt/CPP/InitsInfo.cc @@ -441,7 +441,7 @@ TableTypeInfo::TableTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c, auto gi = c->RegisterType(tbl->GetIndices()); ASSERT(gi); indices = gi->Offset(); - final_init_cohort = gi->InitCohort(); + final_init_cohort = gi->InitCohort() + 1; yield = tbl->Yield(); diff --git a/src/script_opt/CPP/RuntimeInitSupport.h b/src/script_opt/CPP/RuntimeInitSupport.h index 80a25db7fb..6f400a8d98 100644 --- a/src/script_opt/CPP/RuntimeInitSupport.h +++ b/src/script_opt/CPP/RuntimeInitSupport.h @@ -22,8 +22,10 @@ public: CPPTableType() : TableType(nullptr, nullptr){}; void SetIndexAndYield(TypeListPtr ind, TypePtr yield) { - ind = std::move(indices); + indices = std::move(ind); yield_type = std::move(yield); + SetSpecialIndices(); + RegenerateHash(); } }; diff --git a/src/script_opt/CPP/RuntimeOps.cc b/src/script_opt/CPP/RuntimeOps.cc index ae2b9c7744..680f4648cc 100644 --- a/src/script_opt/CPP/RuntimeOps.cc +++ b/src/script_opt/CPP/RuntimeOps.cc @@ -221,7 +221,7 @@ TableValPtr set_constructor__CPP(vector elements, TableTypePtr t, vector auto aggr = make_intrusive(std::move(t), std::move(attrs)); for ( auto& elem : elements ) - aggr->Assign(std::move(elem), nullptr); + aggr->Assign(elem, nullptr); return aggr; } diff --git a/src/script_opt/Expr.cc b/src/script_opt/Expr.cc index 146c21800b..ec69f0fe0c 100644 --- a/src/script_opt/Expr.cc +++ b/src/script_opt/Expr.cc @@ -1443,8 +1443,6 @@ ExprPtr AssignExpr::Reduce(Reducer* c, StmtPtr& red_stmt) { StmtPtr rhs_reduce; if ( lhs_is_any != rhs_is_any ) { - auto op2_loc = op2->GetLocationInfo(); - ExprPtr red_rhs = op2->ReduceToSingleton(c, rhs_reduce); if ( lhs_is_any ) { @@ -1455,15 +1453,11 @@ ExprPtr AssignExpr::Reduce(Reducer* c, StmtPtr& red_stmt) { } else op2 = make_intrusive(red_rhs, t1); - - op2->SetLocationInfo(op2_loc); } if ( t1->Tag() == TYPE_VECTOR && t1->Yield()->Tag() != TYPE_ANY && t2->Yield() && t2->Yield()->Tag() == TYPE_ANY ) { - auto op2_loc = op2->GetLocationInfo(); ExprPtr red_rhs = op2->ReduceToSingleton(c, rhs_reduce); op2 = make_intrusive(red_rhs, t1); - op2->SetLocationInfo(op2_loc); } auto lhs_ref = op1->AsRefExprPtr(); @@ -1950,6 +1944,30 @@ ExprPtr VectorCoerceExpr::Duplicate() { return SetSucc(new VectorCoerceExpr(op_dup, GetType())); } +bool VectorCoerceExpr::IsReduced(Reducer* c) const { + if ( WillTransform(c) ) + return NonReduced(this); + + return UnaryExpr::IsReduced(c); +} + +bool VectorCoerceExpr::WillTransform(Reducer* c) const { + return op->Tag() == EXPR_VECTOR_CONSTRUCTOR && op->GetType()->IsUnspecifiedVector(); +} + +ExprPtr VectorCoerceExpr::Reduce(Reducer* c, StmtPtr& red_stmt) { + if ( WillTransform(c) ) { + auto op1_list = op->GetOp1(); + ASSERT(op1_list->Tag() == EXPR_LIST); + auto empty_list = cast_intrusive(op1_list); + auto new_me = make_intrusive(empty_list, type); + auto red_e = new_me->Reduce(c, red_stmt); + return TransformMe(std::move(red_e), c, red_stmt); + } + + return UnaryExpr::Reduce(c, red_stmt); +} + ExprPtr ScheduleExpr::Duplicate() { auto when_d = when->Duplicate(); auto event_d = event->Duplicate()->AsEventExprPtr(); diff --git a/src/script_opt/FuncInfo.cc b/src/script_opt/FuncInfo.cc index e27dc8282c..6b89e4c3a6 100644 --- a/src/script_opt/FuncInfo.cc +++ b/src/script_opt/FuncInfo.cc @@ -550,22 +550,23 @@ static std::unordered_map func_attrs = { // Some of these have side effects that could be checked for in a specific // context, but the gains from doing so likely aren't worth the complexity. -bool is_special_script_func(std::string func_name) { +bool is_special_script_func(const std::string& func_name) { auto f_attr = func_attrs.find(func_name); return f_attr != func_attrs.end() && (f_attr->second & ATTR_SPECIAL_SCRIPT_FUNC) != 0; } -bool is_idempotent(std::string func_name) { +bool is_idempotent(const std::string& func_name) { auto f_attr = func_attrs.find(func_name); return f_attr != func_attrs.end() && (f_attr->second & ATTR_IDEMPOTENT) != 0; } -bool has_no_script_side_effects(std::string func_name) { +bool has_script_side_effects(const std::string& func_name) { auto f_attr = func_attrs.find(func_name); if ( f_attr == func_attrs.end() ) - return false; + // We don't know about it, so be conservative. + return true; - return (f_attr->second & (ATTR_NO_SCRIPT_SIDE_EFFECTS | ATTR_NO_ZEEK_SIDE_EFFECTS | ATTR_IDEMPOTENT)) != 0; + return (f_attr->second & (ATTR_NO_SCRIPT_SIDE_EFFECTS | ATTR_NO_ZEEK_SIDE_EFFECTS | ATTR_IDEMPOTENT)) == 0; } } // namespace zeek::detail diff --git a/src/script_opt/FuncInfo.h b/src/script_opt/FuncInfo.h index f4bf333cc0..53095d227d 100644 --- a/src/script_opt/FuncInfo.h +++ b/src/script_opt/FuncInfo.h @@ -10,15 +10,15 @@ namespace zeek::detail { // A "special script function" is one that the event engine explicitly // knows about. -extern bool is_special_script_func(std::string func_name); +extern bool is_special_script_func(const std::string& func_name); // An idempotent function returns the same value when called with the // same arguments (and has no meaningful side effects in terms of script-level // or Zeek-internal state). -extern bool is_idempotent(std::string func_name); +extern bool is_idempotent(const std::string& func_name); -// Whether the given function (currently, just BiFs) has no Zeek-script-level +// Whether the given function (currently, just BiFs) has Zeek-script-level // side effects. -extern bool has_no_script_side_effects(std::string func_name); +extern bool has_script_side_effects(const std::string& func_name); } // namespace zeek::detail diff --git a/src/script_opt/GenIDDefs.cc b/src/script_opt/GenIDDefs.cc index b05f334ea6..9fb93e61fd 100644 --- a/src/script_opt/GenIDDefs.cc +++ b/src/script_opt/GenIDDefs.cc @@ -50,7 +50,7 @@ void GenIDDefs::TraverseFunction(const FuncPtr& f, ScopePtr scope, StmtPtr body) } TraversalCode GenIDDefs::PreStmt(const Stmt* s) { - curr_stmt = s; + last_stmt_traversed = s; auto si = s->GetOptInfo(); si->stmt_num = ++stmt_num; @@ -122,11 +122,11 @@ TraversalCode GenIDDefs::PreStmt(const Stmt* s) { t_branch->Traverse(this); if ( ! t_branch->NoFlowAfter(false) ) - BranchBeyond(curr_stmt, s, true); + BranchBeyond(last_stmt_traversed, s, true); f_branch->Traverse(this); if ( ! f_branch->NoFlowAfter(false) ) - BranchBeyond(curr_stmt, s, true); + BranchBeyond(last_stmt_traversed, s, true); EndConfluenceBlock(true); @@ -155,7 +155,7 @@ TraversalCode GenIDDefs::PreStmt(const Stmt* s) { body->Traverse(this); if ( ! body->NoFlowAfter(false) ) - BranchBackTo(curr_stmt, s, true); + BranchBackTo(last_stmt_traversed, s, true); EndConfluenceBlock(); @@ -183,7 +183,7 @@ TraversalCode GenIDDefs::PreStmt(const Stmt* s) { body->Traverse(this); if ( ! body->NoFlowAfter(false) ) - BranchBackTo(curr_stmt, s, true); + BranchBackTo(last_stmt_traversed, s, true); EndConfluenceBlock(); @@ -242,7 +242,7 @@ TraversalCode GenIDDefs::PostStmt(const Stmt* s) { case STMT_RETURN: ReturnAt(s); break; - case STMT_NEXT: BranchBackTo(curr_stmt, FindLoop(), false); break; + case STMT_NEXT: BranchBackTo(last_stmt_traversed, FindLoop(), false); break; case STMT_BREAK: { auto target = FindBreakTarget(); @@ -403,8 +403,9 @@ void GenIDDefs::CheckVarUsage(const Expr* e, const ID* id) { auto oi = id->GetOptInfo(); - if ( ! oi->DidUndefinedWarning() && ! oi->IsDefinedBefore(curr_stmt) && ! id->GetAttr(ATTR_IS_ASSIGNED) ) { - if ( ! oi->IsPossiblyDefinedBefore(curr_stmt) ) { + if ( ! oi->DidUndefinedWarning() && ! oi->IsDefinedBefore(last_stmt_traversed) && + ! id->GetAttr(ATTR_IS_ASSIGNED) ) { + if ( ! oi->IsPossiblyDefinedBefore(last_stmt_traversed) ) { e->Warn("used without definition"); oi->SetDidUndefinedWarning(); } @@ -423,7 +424,7 @@ void GenIDDefs::StartConfluenceBlock(const Stmt* s) { void GenIDDefs::EndConfluenceBlock(bool no_orig) { for ( auto id : modified_IDs.back() ) - id->GetOptInfo()->ConfluenceBlockEndsAfter(curr_stmt, no_orig); + id->GetOptInfo()->ConfluenceBlockEndsAfter(last_stmt_traversed, no_orig); confluence_blocks.pop_back(); modified_IDs.pop_back(); @@ -489,7 +490,7 @@ void GenIDDefs::TrackID(const ID* id, const ExprPtr& e) { // here to set the lowest limit for definitions. For now we leave // DefinedAfter as capable of supporting that distinction in case we // find need to revive it in the future. - oi->DefinedAfter(curr_stmt, e, confluence_blocks, 0); + oi->DefinedAfter(last_stmt_traversed, e, confluence_blocks, 0); // Ensure we track this identifier across all relevant // confluence regions. diff --git a/src/script_opt/GenIDDefs.h b/src/script_opt/GenIDDefs.h index 5074415f54..8d2218b5f6 100644 --- a/src/script_opt/GenIDDefs.h +++ b/src/script_opt/GenIDDefs.h @@ -82,8 +82,8 @@ private: // outer "break" in that context. FunctionFlavor func_flavor; - // The statement we are currently traversing. - const Stmt* curr_stmt = nullptr; + // The most recently traversed statement. + const Stmt* last_stmt_traversed = nullptr; // Used to number Stmt objects found during AST traversal. int stmt_num; diff --git a/src/script_opt/IDOptInfo.cc b/src/script_opt/IDOptInfo.cc index 744f27e2f3..9e170a2191 100644 --- a/src/script_opt/IDOptInfo.cc +++ b/src/script_opt/IDOptInfo.cc @@ -185,7 +185,6 @@ void IDOptInfo::BranchBackTo(const Stmt* from, const Stmt* to, bool close_all) { auto& ur = usage_regions[i]; if ( ur.DefinedAfter() < new_def ) { - ASSERT(ur.DefinedAfter() != NO_DEF); ur.UpdateDefinedAfter(new_def); ur.SetDefExpr(nullptr); } diff --git a/src/script_opt/ProfileFunc.cc b/src/script_opt/ProfileFunc.cc index 0457482703..df83fe1dd8 100644 --- a/src/script_opt/ProfileFunc.cc +++ b/src/script_opt/ProfileFunc.cc @@ -635,7 +635,7 @@ bool ProfileFuncs::GetCallSideEffects(const NameExpr* n, IDSet& non_local_ids, T auto func = fv->AsFunc(); if ( func->GetKind() == Func::BUILTIN_FUNC ) { - if ( ! has_no_script_side_effects(func->Name()) ) + if ( has_script_side_effects(func->Name()) ) is_unknown = true; return true; } @@ -1168,7 +1168,7 @@ bool ProfileFuncs::DefinitelyHasNoSideEffects(const ExprPtr& e) const { return false; for ( auto& b : pf->BiFGlobals() ) - if ( ! has_no_script_side_effects(b->Name()) ) + if ( has_script_side_effects(b->Name()) ) return false; return true; @@ -1259,7 +1259,7 @@ bool ProfileFuncs::AssessSideEffects(const ProfileFunc* pf, IDSet& non_local_ids } for ( auto& b : pf->BiFGlobals() ) - if ( ! has_no_script_side_effects(b->Name()) ) { + if ( has_script_side_effects(b->Name()) ) { is_unknown = true; return true; } diff --git a/src/script_opt/Reduce.cc b/src/script_opt/Reduce.cc index d45bf0fa1d..7f63a978b7 100644 --- a/src/script_opt/Reduce.cc +++ b/src/script_opt/Reduce.cc @@ -124,6 +124,14 @@ StmtPtr Reducer::GenParam(const IDPtr& id, ExprPtr rhs, bool is_modified) { // the inline block's execution. is_modified = true; + auto& id_t = id->GetType(); + if ( id_t->Tag() == TYPE_VECTOR && rhs->GetType()->Yield() != id_t->Yield() ) + // Presumably either the identifier or the RHS is a vector-of-any. + // This means there will essentially be a modification of the RHS + // due to the need to use (or omit) operations coercing from such + // vectors. + is_modified = true; + if ( ! is_modified ) { // Can use a temporary variable, which then supports // optimization via alias propagation. diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index 34ffbf4c72..d9051f8113 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -507,9 +507,6 @@ static void analyze_scripts_for_ZAM() { void clear_script_analysis() { IDOptInfo::ClearGlobalInitExprs(); - // Keep the functions around if we're debugging, so we can - // generate profiles. -#ifndef DEBUG // We need to explicitly clear out the optimization information // associated with identifiers. They have reference loops with // the parent identifier that will prevent reclamation of the @@ -519,8 +516,10 @@ void clear_script_analysis() { for ( auto& id : f.Scope()->OrderedVars() ) id->ClearOptInfo(); - funcs.clear(); -#endif + // Keep the functions around if we're profiling, so we can loop + // over them to generate the profiles. + if ( ! analysis_options.profile_ZAM ) + funcs.clear(); non_recursive_funcs.clear(); lambdas.clear(); diff --git a/src/script_opt/ZAM/AM-Opt.cc b/src/script_opt/ZAM/AM-Opt.cc index dbb5804a58..cd4a3e6b6f 100644 --- a/src/script_opt/ZAM/AM-Opt.cc +++ b/src/script_opt/ZAM/AM-Opt.cc @@ -158,15 +158,15 @@ bool ZAMCompiler::RemoveDeadCode() { } if ( t && t->inst_num > i0->inst_num && (! i1 || t->inst_num <= i1->inst_num) ) { - // This is effectively a branch to the next - // instruction. Even if i0 is conditional, there's - // no point executing it because regardless of the - // outcome of the conditional, we go to the next - // successive live instruction (and we don't have - // conditionals with side effects). - KillInst(i0); - did_removal = true; - continue; + // This is effectively a branch to the next instruction. + // We can remove it *unless* the instruction has side effects. + // Conditionals don't, but loop-iteration-advancement + // instructions do. + if ( ! i0->IsLoopIterationAdvancement() ) { + KillInst(i0); + did_removal = true; + continue; + } } if ( i0->DoesNotContinue() && i1 && i1->num_labels == 0 ) { diff --git a/src/script_opt/ZAM/BuiltInSupport.cc b/src/script_opt/ZAM/BuiltInSupport.cc index c85786561a..84903ac95a 100644 --- a/src/script_opt/ZAM/BuiltInSupport.cc +++ b/src/script_opt/ZAM/BuiltInSupport.cc @@ -7,7 +7,7 @@ namespace zeek::detail { -FixedCatArg::FixedCatArg(const TypePtr& _t) : t(_t) { +FixedCatArg::FixedCatArg(TypePtr _t) : t(std::move(_t)) { switch ( t->Tag() ) { case TYPE_BOOL: max_size = 1; break; @@ -27,13 +27,13 @@ FixedCatArg::FixedCatArg(const TypePtr& _t) : t(_t) { case TYPE_ENUM: { size_t n = 0; for ( const auto& e : t->AsEnumType()->Names() ) - n += e.first.size(); + n = std::max(n, e.first.size()); max_size = n; break; } case TYPE_PORT: - max_size = 5 + 1 + 7; // + / + "unknown + max_size = 5 + 1 + 7; // + / + "unknown" break; case TYPE_ADDR: diff --git a/src/script_opt/ZAM/BuiltInSupport.h b/src/script_opt/ZAM/BuiltInSupport.h index f216c068ef..7169ea94d1 100644 --- a/src/script_opt/ZAM/BuiltInSupport.h +++ b/src/script_opt/ZAM/BuiltInSupport.h @@ -36,12 +36,12 @@ protected: class FixedCatArg : public CatArg { public: - FixedCatArg(const TypePtr& t); + FixedCatArg(TypePtr t); void RenderInto(ZVal* zframe, int slot, char*& res) override; protected: - const TypePtr& t; + TypePtr t; char tmp[256]; }; @@ -80,7 +80,7 @@ protected: class DescCatArg : public CatArg { public: - DescCatArg(const TypePtr& _t) : CatArg(), t(_t) { d.SetStyle(RAW_STYLE); } + DescCatArg(TypePtr _t) : CatArg(), t(std::move(_t)) { d.SetStyle(RAW_STYLE); } void RenderInto(ZVal* zframe, int slot, char*& res) override { auto n = d.Len(); diff --git a/src/script_opt/ZAM/Compile.h b/src/script_opt/ZAM/Compile.h index 686c92d40d..9d3744bed7 100644 --- a/src/script_opt/ZAM/Compile.h +++ b/src/script_opt/ZAM/Compile.h @@ -110,11 +110,14 @@ private: template using CaseMapsI = std::vector>; + template + void AdjustSwitchTables(CaseMapsI& abstract_cases); + template void ConcretizeSwitchTables(const CaseMapsI& abstract_cases, CaseMaps& concrete_cases); template - void DumpCases(const T& cases, const char* type_name) const; + void DumpCases(const CaseMaps& cases, const char* type_name) const; void DumpInsts1(const FrameReMap* remappings); #include "zeek/ZAM-MethodDecls.h" diff --git a/src/script_opt/ZAM/Driver.cc b/src/script_opt/ZAM/Driver.cc index 5840d56a9f..e165ac2f54 100644 --- a/src/script_opt/ZAM/Driver.cc +++ b/src/script_opt/ZAM/Driver.cc @@ -264,6 +264,20 @@ void ZAMCompiler::AdjustBranches() { if ( auto t = inst->target ) inst->target = FindLiveTarget(t); } + + // Fix up the implicit branches in switches, too. + AdjustSwitchTables(int_casesI); + AdjustSwitchTables(uint_casesI); + AdjustSwitchTables(double_casesI); + AdjustSwitchTables(str_casesI); +} + +template +void ZAMCompiler::AdjustSwitchTables(CaseMapsI& abstract_cases) { + for ( auto& targs : abstract_cases ) { + for ( auto& targ : targs ) + targ.second = FindLiveTarget(targ.second); + } } void ZAMCompiler::RetargetBranches() { @@ -386,14 +400,14 @@ void ZAMCompiler::Dump() { inst->Dump(&frame_denizens, remappings); } - DumpCases(int_casesI, "int"); - DumpCases(uint_casesI, "uint"); - DumpCases(double_casesI, "double"); - DumpCases(str_casesI, "str"); + DumpCases(int_cases, "int"); + DumpCases(uint_cases, "uint"); + DumpCases(double_cases, "double"); + DumpCases(str_cases, "str"); } template -void ZAMCompiler::DumpCases(const T& cases, const char* type_name) const { +void ZAMCompiler::DumpCases(const CaseMaps& cases, const char* type_name) const { for ( auto i = 0U; i < cases.size(); ++i ) { printf("%s switch table #%d:", type_name, i); for ( auto& m : cases[i] ) { @@ -404,7 +418,7 @@ void ZAMCompiler::DumpCases(const T& cases, const char* type_name) const { std::is_same_v ) case_val = std::to_string(m.first); - printf(" %s->%d", case_val.c_str(), m.second->inst_num); + printf(" %s->%d", case_val.c_str(), m.second); } printf("\n"); } diff --git a/src/script_opt/ZAM/IterInfo.h b/src/script_opt/ZAM/IterInfo.h index 7f2a40d97b..438200df93 100644 --- a/src/script_opt/ZAM/IterInfo.h +++ b/src/script_opt/ZAM/IterInfo.h @@ -27,7 +27,7 @@ public: // Start looping over the elements of the given table. "_aux" // provides information about the index variables, their types, // and the type of the value variable (if any). - void BeginLoop(const TableVal* _tv, ZInstAux* _aux) { + void BeginLoop(TableValPtr _tv, ZInstAux* _aux) { tv = _tv; aux = _aux; auto tvd = tv->AsTable(); @@ -76,10 +76,7 @@ public: } private: - // The table we're looping over. If we want to allow for the table - // going away before we're able to clear our iterators then we - // could change this to non-const and use Ref/Unref. - const TableVal* tv = nullptr; + TableValPtr tv = nullptr; // Associated auxiliary information. ZInstAux* aux = nullptr; diff --git a/src/script_opt/ZAM/Ops.in b/src/script_opt/ZAM/Ops.in index 66bb15c8fb..38413bb092 100644 --- a/src/script_opt/ZAM/Ops.in +++ b/src/script_opt/ZAM/Ops.in @@ -1859,7 +1859,7 @@ internal-op Init-Table-Loop type VV op1-read eval auto& ti = (*tiv_ptr)[z.v2]; - ti.BeginLoop(frame[z.v1].table_val, z.aux); + ti.BeginLoop({NewRef{}, frame[z.v1].table_val}, z.aux); internal-op Next-Table-Iter op1-read @@ -2446,7 +2446,8 @@ type VV eval Cat1OpRef(frame[z.v2]) macro Cat1FullVal(val) - Cat1Op(ZVal(ZAM_val_cat(val.ToVal(z.t)))) + auto formatted_val = ZVal(ZAM_val_cat(val.ToVal(z.t))); + Cat1Op(formatted_val) internal-op Cat1Full type VC diff --git a/src/script_opt/ZAM/Stmt.cc b/src/script_opt/ZAM/Stmt.cc index d38b99f95f..44a816c87a 100644 --- a/src/script_opt/ZAM/Stmt.cc +++ b/src/script_opt/ZAM/Stmt.cc @@ -681,13 +681,10 @@ const ZAMStmt ZAMCompiler::LoopOverTable(const ForStmt* f, const NameExpr* val) auto value_var = f->ValueVar(); auto body = f->LoopBody(); - // Check whether the loop variables are actually used in the body. - // This is motivated by an idiom where there's both loop_vars and - // a value_var, but the script only actually needs the value_var; - // and also some weird cases where the script is managing a - // separate iteration process manually. - ProfileFunc body_pf(body); - + // We used to have more involved logic here to check whether the loop + // variables are actually used in the body. Now that we have '_' + // loop placeholder variables, this is no longer worth trying to + // optimize for, though we still optimize for those placeholders. int num_unused = 0; auto aux = new ZInstAux(0); @@ -695,7 +692,7 @@ const ZAMStmt ZAMCompiler::LoopOverTable(const ForStmt* f, const NameExpr* val) for ( auto i = 0; i < loop_vars->length(); ++i ) { auto id = (*loop_vars)[i]; - if ( body_pf.Locals().count(id) == 0 || id->IsBlank() ) + if ( id->IsBlank() ) ++num_unused; int slot = id->IsBlank() ? -1 : FrameSlot(id); @@ -707,12 +704,6 @@ const ZAMStmt ZAMCompiler::LoopOverTable(const ForStmt* f, const NameExpr* val) bool no_loop_vars = (num_unused == loop_vars->length()); - if ( value_var && body_pf.Locals().count(value_var.get()) == 0 ) - // This is more clearly a coding botch - someone left in - // an unnecessary value_var variable. But might as - // well not do the work. - value_var = nullptr; - if ( value_var ) aux->value_var_type = value_var->GetType(); diff --git a/src/script_opt/ZAM/ZInst.cc b/src/script_opt/ZAM/ZInst.cc index ae9cf7999b..4224576fe1 100644 --- a/src/script_opt/ZAM/ZInst.cc +++ b/src/script_opt/ZAM/ZInst.cc @@ -214,6 +214,23 @@ ValPtr ZInst::ConstVal() const { return nullptr; } +bool ZInst::IsLoopIterationAdvancement() const { + switch ( op ) { + case OP_NEXT_TABLE_ITER_VV: + case OP_NEXT_TABLE_ITER_NO_VARS_VV: + case OP_NEXT_TABLE_ITER_VAL_VAR_VVV: + case OP_NEXT_TABLE_ITER_VAL_VAR_NO_VARS_VVV: + case OP_NEXT_VECTOR_ITER_VVV: + case OP_NEXT_VECTOR_BLANK_ITER_VV: + case OP_NEXT_VECTOR_ITER_VAL_VAR_VVVV: + case OP_NEXT_VECTOR_BLANK_ITER_VAL_VAR_VVV: + case OP_NEXT_STRING_ITER_VVV: + case OP_NEXT_STRING_BLANK_ITER_VV: return true; + + default: return false; + } +} + string ZInst::ConstDump() const { auto v = ConstVal(); diff --git a/src/script_opt/ZAM/ZInst.h b/src/script_opt/ZAM/ZInst.h index 2d58b7ac2e..8e53d39b69 100644 --- a/src/script_opt/ZAM/ZInst.h +++ b/src/script_opt/ZAM/ZInst.h @@ -88,6 +88,10 @@ public: // Returns nil if this instruction doesn't have an associated constant. ValPtr ConstVal() const; + // Returns true if this instruction represents a form of advancing + // a loop iteration, false otherwise. + bool IsLoopIterationAdvancement() const; + // Returns a string describing the constant. std::string ConstDump() const; diff --git a/testing/btest/Baseline.zam/scripts.policy.misc.dump-events/really-all-events.log b/testing/btest/Baseline.zam/scripts.policy.misc.dump-events/really-all-events.log index 9329a4add7..3ddca7b285 100644 --- a/testing/btest/Baseline.zam/scripts.policy.misc.dump-events/really-all-events.log +++ b/testing/btest/Baseline.zam/scripts.policy.misc.dump-events/really-all-events.log @@ -2,7 +2,6 @@ 0.000000 zeek_init XXXXXXXXXX.XXXXXX network_time_init XXXXXXXXXX.XXXXXX Broker::log_flush -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_connection [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=56166/udp, resp_h=10.10.1.1, resp_p=53/udp], orig=[size=34, state=1, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=0 secs, service={\x0a\x0a}, history=D, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks=, smtp=, smtp_state=] @@ -21,9 +20,6 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=76, cap_len=76, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=62, id=9482, DF=F, MF=F, offset=0, ttl=128, p=17, sum=65420, src=10.10.1.4, dst=10.10.1.1], ip6=, tcp=, udp=[sport=56166/udp, dport=53/udp, ulen=42], icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=56166/udp, resp_h=10.10.1.1, resp_p=53/udp], orig=[size=34, state=1, num_pkts=1, num_bytes_ip=62, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=100, state=1, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=34.0 msecs 24.953842 usecs, service={\x0a\x0a}, history=Dd, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks=, smtp=, smtp_state=] @@ -268,7 +264,6 @@ XXXXXXXXXX.XXXXXX tcp_packet XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=60, cap_len=60, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=40, id=8675, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17585, src=74.53.140.153, dst=10.10.1.4], ip6=, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727269, ack=2126795706, hl=20, dl=0, reserved=0, flags=16, win=5840], udp=, icmp=] -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet @@ -340,9 +335,6 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=191, cap_len=191, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=177, id=8676, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17447, src=74.53.140.153, dst=10.10.1.4], ip6=, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727269, ack=2126795706, hl=20, dl=137, reserved=0, flags=24, win=5840], udp=, icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=21, state=4, num_pkts=3, num_bytes_ip=137, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=318, state=4, num_pkts=4, num_bytes_ip=486, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=1.0 sec 39.0 msecs 682.865143 usecs, service={\x0aSMTP\x0a}, history=ShAdDa, uid=ClEkJM2Vm5giqnMf4h, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto=, date=, from=, to=, cc=, reply_to=, msg_id=, in_reply_to=, subject=, x_originating_ip=, first_received=, second_received=, last_reply=250 HELP, path=[74.53.140.153, 10.10.1.4], user_agent=, tls=F, process_received_from=T, has_client_activity=F, process_smtp_headers=T, entity_count=0, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, trans_mail_from_seen=F, trans_rcpt_to_seen=F, invalid_transactions=0, analyzer_id=6, mime_depth=0]] @@ -428,7 +420,6 @@ XXXXXXXXXX.XXXXXX smtp_request XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=84, cap_len=84, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=70, id=9513, DF=T, MF=F, offset=0, ttl=128, p=6, sum=62380, src=10.10.1.4, dst=74.53.140.153], ip6=, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126795718, ack=2934727424, hl=20, dl=30, reserved=0, flags=24, win=65199], udp=, icmp=] -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet @@ -459,10 +450,6 @@ XXXXXXXXXX.XXXXXX smtp_reply XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=72, cap_len=72, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=58, id=8678, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17564, src=74.53.140.153, dst=10.10.1.4], ip6=, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727424, ack=2126795748, hl=20, dl=18, reserved=0, flags=24, win=5840], udp=, icmp=] -XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet @@ -492,9 +479,6 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=72, cap_len=72, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=58, id=9518, DF=T, MF=F, offset=0, ttl=128, p=6, sum=62387, src=10.10.1.4, dst=74.53.140.153], ip6=, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126795748, ack=2934727442, hl=20, dl=18, reserved=0, flags=24, win=65181], udp=, icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX Broker::log_flush XXXXXXXXXX.XXXXXX new_packet @@ -589,7 +573,6 @@ XXXXXXXXXX.XXXXXX smtp_reply XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=62, cap_len=62, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=48, id=8680, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17572, src=74.53.140.153, dst=10.10.1.4], ip6=, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727472, ack=2126795802, hl=20, dl=8, reserved=0, flags=24, win=5840], udp=, icmp=] -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet @@ -619,9 +602,6 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=93, cap_len=93, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=79, id=9528, DF=T, MF=F, offset=0, ttl=128, p=6, sum=62356, src=10.10.1.4, dst=74.53.140.153], ip6=, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126795802, ack=2934727480, hl=20, dl=39, reserved=0, flags=24, win=65143], udp=, icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=144, state=4, num_pkts=8, num_bytes_ip=472, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=406, state=4, num_pkts=8, num_bytes_ip=720, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=2.0 secs 790.0 msecs 662.050247 usecs, service={\x0aSMTP\x0a}, history=ShAdDa, uid=ClEkJM2Vm5giqnMf4h, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=, from=, to=, cc=, reply_to=, msg_id=, in_reply_to=, subject=, x_originating_ip=, first_received=, second_received=, last_reply=250 OK, path=[74.53.140.153, 10.10.1.4], user_agent=, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=T, entity_count=0, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=0]] @@ -685,7 +665,6 @@ XXXXXXXXXX.XXXXXX raw_packet XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX Broker::log_flush -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=150, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=462, state=4, num_pkts=9, num_bytes_ip=774, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=3.0 secs 132.0 msecs 632.97081 usecs, service={\x0aSMTP\x0a}, history=ShAdDa, uid=ClEkJM2Vm5giqnMf4h, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=, from=, to=, cc=, reply_to=, msg_id=, in_reply_to=, subject=, x_originating_ip=, first_received=, second_received=, last_reply=250 Accepted, path=[74.53.140.153, 10.10.1.4], user_agent=, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=T, entity_count=1, entity=[filename=], fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=1]] [1] p: pkt_hdr = [ip=[hl=20, tos=96, len=96, id=8682, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17522, src=74.53.140.153, dst=10.10.1.4], ip6=, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727494, ack=2126795847, hl=20, dl=56, reserved=0, flags=24, win=5840], udp=, icmp=] @@ -715,9 +694,6 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=110, cap_len=110, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=96, id=8682, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17522, src=74.53.140.153, dst=10.10.1.4], ip6=, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727494, ack=2126795847, hl=20, dl=56, reserved=0, flags=24, win=5840], udp=, icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=3.0 secs 163.0 msecs 697.004318 usecs, service={\x0aSMTP\x0a}, history=ShAdDa, uid=ClEkJM2Vm5giqnMf4h, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=, from=, to=, cc=, reply_to=, msg_id=, in_reply_to=, subject=, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=T, entity_count=1, entity=[filename=], fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=1]] @@ -2934,7 +2910,6 @@ XXXXXXXXXX.XXXXXX tcp_packet XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=60, cap_len=60, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=40, id=8684, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17576, src=74.53.140.153, dst=10.10.1.4], ip6=, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727550, ack=2126798751, hl=20, dl=0, reserved=0, flags=16, win=11616], udp=, icmp=] -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet @@ -2958,9 +2933,6 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=1506, cap_len=1506, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=1492, id=9558, DF=T, MF=F, offset=0, ttl=128, p=6, sum=60913, src=10.10.1.4, dst=74.53.140.153], ip6=, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126800203, ack=2934727550, hl=20, dl=1452, reserved=0, flags=16, win=65073], udp=, icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=7410, state=4, num_pkts=17, num_bytes_ip=12486, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=462, state=4, num_pkts=12, num_bytes_ip=950, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=3.0 secs 940.0 msecs 768.003464 usecs, service={\x0aSMTP\x0a}, history=ShAdDaT, uid=ClEkJM2Vm5giqnMf4h, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={\x0a\x0a}, cc=, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=F, entity_count=5, entity=[filename=NEWS.txt], fuids=[FmFp351N5nhsMmAfQg, Fqrb1K5DWEfgy4WU2, FEFYSd1s8Onn9LynKj]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=5]] @@ -3937,7 +3909,6 @@ XXXXXXXXXX.XXXXXX smtp_data XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=1506, cap_len=1506, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=1492, id=9560, DF=T, MF=F, offset=0, ttl=128, p=6, sum=60911, src=10.10.1.4, dst=74.53.140.153], ip6=, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126803107, ack=2934727550, hl=20, dl=1452, reserved=0, flags=24, win=65073], udp=, icmp=] -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet @@ -4341,12 +4312,8 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=1506, cap_len=1506, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=1492, id=9561, DF=T, MF=F, offset=0, ttl=128, p=6, sum=60910, src=10.10.1.4, dst=74.53.140.153], ip6=, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126804559, ack=2934727550, hl=20, dl=1452, reserved=0, flags=16, win=65073], udp=, icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX Broker::log_flush -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=10314, state=4, num_pkts=20, num_bytes_ip=16962, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=462, state=4, num_pkts=13, num_bytes_ip=990, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=4.0 secs 305.0 msecs 548.906326 usecs, service={\x0aSMTP\x0a}, history=ShAdDaT, uid=ClEkJM2Vm5giqnMf4h, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={\x0a\x0a}, cc=, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=F, entity_count=5, entity=[filename=NEWS.txt], fuids=[FmFp351N5nhsMmAfQg, Fqrb1K5DWEfgy4WU2, FEFYSd1s8Onn9LynKj]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=5]] [1] p: pkt_hdr = [ip=[hl=20, tos=96, len=40, id=8686, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17574, src=74.53.140.153, dst=10.10.1.4], ip6=, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727550, ack=2126801655, hl=20, dl=0, reserved=0, flags=16, win=17424], udp=, icmp=] @@ -4364,9 +4331,6 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=60, cap_len=60, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=40, id=8686, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17574, src=74.53.140.153, dst=10.10.1.4], ip6=, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727550, ack=2126801655, hl=20, dl=0, reserved=0, flags=16, win=17424], udp=, icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=11766, state=4, num_pkts=20, num_bytes_ip=16962, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=462, state=4, num_pkts=14, num_bytes_ip=1030, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=4.0 secs 305.0 msecs 582.046509 usecs, service={\x0aSMTP\x0a}, history=ShAdDaT, uid=ClEkJM2Vm5giqnMf4h, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=gurpartap@patriots.in, rcptto={\x0araj_deol2002in@yahoo.co.in\x0a}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={\x0a\x0a}, cc=, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, process_smtp_headers=F, entity_count=5, entity=[filename=NEWS.txt], fuids=[FmFp351N5nhsMmAfQg, Fqrb1K5DWEfgy4WU2, FEFYSd1s8Onn9LynKj]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, trans_mail_from_seen=T, trans_rcpt_to_seen=T, invalid_transactions=0, analyzer_id=6, mime_depth=5]] @@ -6044,7 +6008,6 @@ XXXXXXXXXX.XXXXXX connection_EOF XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=54, cap_len=54, src=00:e0:1c:3c:17:c2, dst=00:1f:33:d9:81:60, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=40, id=9574, DF=T, MF=F, offset=0, ttl=128, p=6, sum=62349, src=10.10.1.4, dst=74.53.140.153], ip6=, tcp=[sport=1470/tcp, dport=25/tcp, seq=2126810402, ack=2934727578, hl=20, dl=0, reserved=0, flags=17, win=65045], udp=, icmp=] -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet @@ -6076,9 +6039,6 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=102, cap_len=102, src=00:1f:33:d9:81:60, dst=00:e0:1c:3c:17:c2, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=96, len=88, id=8695, DF=T, MF=F, offset=0, ttl=50, p=6, sum=17517, src=74.53.140.153, dst=10.10.1.4], ip6=, tcp=[sport=25/tcp, dport=1470/tcp, seq=2934727578, ack=2126810402, hl=20, dl=48, reserved=0, flags=24, win=34848], udp=, icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=14705, state=5, num_pkts=27, num_bytes_ip=21633, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=538, state=5, num_pkts=23, num_bytes_ip=1466, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=XXXXXXXXXX.XXXXXX, duration=7.0 secs 576.0 msecs 952.934265 usecs, service={\x0aSMTP\x0a}, history=ShAdDaTFf, uid=ClEkJM2Vm5giqnMf4h, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=ClEkJM2Vm5giqnMf4h, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=2, helo=GP, mailfrom=, rcptto=, date=, from=, to=, cc=, reply_to=, msg_id=, in_reply_to=, subject=, x_originating_ip=, first_received=, second_received=, last_reply=221 xc90.websitewelcome.com closing connection, path=[74.53.140.153, 10.10.1.4], user_agent=, tls=F, process_received_from=T, has_client_activity=F, process_smtp_headers=T, entity_count=0, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=1, pending_messages=, trans_mail_from_seen=F, trans_rcpt_to_seen=F, invalid_transactions=0, analyzer_id=6, mime_depth=5]] @@ -6530,7 +6490,6 @@ XXXXXXXXXX.XXXXXX smtp_reply XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=101, cap_len=101, src=00:08:ca:cc:ad:4c, dst=58:b0:35:86:54:8d, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=87, id=51483, DF=T, MF=F, offset=0, ttl=64, p=6, sum=58729, src=192.168.133.102, dst=192.168.133.100], ip6=, tcp=[sport=25/tcp, dport=49648/tcp, seq=2131788255, ack=3976465341, hl=32, dl=35, reserved=0, flags=24, win=227], udp=, icmp=] -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet @@ -6573,9 +6532,6 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=58:b0:35:86:54:8d, dst=00:08:ca:cc:ad:4c, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=57836, DF=T, MF=F, offset=0, ttl=64, p=6, sum=52411, src=192.168.133.100, dst=192.168.133.102], ip6=, tcp=[sport=49648/tcp, dport=25/tcp, seq=3976465341, ack=2131788290, hl=32, dl=0, reserved=0, flags=16, win=4116], udp=, icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet [0] c: connection = [id=[orig_h=192.168.133.100, orig_p=49648/tcp, resp_h=192.168.133.102, resp_p=25/tcp], orig=[size=24, state=4, num_pkts=3, num_bytes_ip=168, flow_label=0, l2_addr=58:b0:35:86:54:8d], resp=[size=35, state=4, num_pkts=2, num_bytes_ip=147, flow_label=0, l2_addr=00:08:ca:cc:ad:4c], start_time=XXXXXXXXXX.XXXXXX, duration=26.0 msecs 411.056519 usecs, service={\x0a\x0a}, history=ShAdD, uid=CmES5u32sYpV7JYN, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks={\x0a\x09SMTP::finalize_smtp\x0a\x09ZAM-code SMTP::finalize_smtp \x0a}, smtp=[ts=XXXXXXXXXX.XXXXXX, uid=CmES5u32sYpV7JYN, id=[orig_h=192.168.133.100, orig_p=49648/tcp, resp_h=192.168.133.102, resp_p=25/tcp], trans_depth=1, helo=, mailfrom=, rcptto=, date=, from=, to=, cc=, reply_to=, msg_id=, in_reply_to=, subject=, x_originating_ip=, first_received=, second_received=, last_reply=220 uprise ESMTP SubEthaSMTP null, path=[192.168.133.102, 192.168.133.100], user_agent=, tls=F, process_received_from=T, has_client_activity=F, process_smtp_headers=T, entity_count=0, entity=, fuids=[]], smtp_state=[helo=, messages_transferred=0, pending_messages=, trans_mail_from_seen=F, trans_rcpt_to_seen=F, invalid_transactions=0, analyzer_id=, mime_depth=0]] @@ -8171,7 +8127,6 @@ XXXXXXXXXX.XXXXXX tcp_options XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=58:b0:35:86:54:8d, dst=cc:b2:55:f4:62:92, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=6886, DF=T, MF=F, offset=0, ttl=64, p=6, sum=18327, src=192.168.133.100, dst=74.125.71.189], ip6=, tcp=[sport=49336/tcp, dport=443/tcp, seq=1584252430, ack=3980080343, hl=32, dl=0, reserved=0, flags=16, win=4093], udp=, icmp=] -XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_packet @@ -8214,9 +8169,6 @@ XXXXXXXXXX.XXXXXX raw_packet [0] p: raw_pkt_hdr = [l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=58:b0:35:86:54:8d, dst=cc:b2:55:f4:62:92, vlan=, inner_vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=3407, DF=T, MF=F, offset=0, ttl=64, p=6, sum=21806, src=192.168.133.100, dst=74.125.71.189], ip6=, tcp=[sport=49336/tcp, dport=443/tcp, seq=1584252430, ack=3980080343, hl=32, dl=0, reserved=0, flags=16, win=4093], udp=, icmp=] XXXXXXXXXX.XXXXXX event_queue_flush_point -XXXXXXXXXX.XXXXXX load_sample - [2] dmem: int = 0 - XXXXXXXXXX.XXXXXX event_queue_flush_point XXXXXXXXXX.XXXXXX new_connection_contents [0] c: connection = [id=[orig_h=192.168.133.100, orig_p=49153/tcp, resp_h=17.172.238.21, resp_p=5223/tcp], orig=[size=714, state=3, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=58:b0:35:86:54:8d], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=cc:b2:55:f4:62:92], start_time=XXXXXXXXXX.XXXXXX, duration=0 secs, service={\x0a\x0a}, history=D, uid=C37jN32gN3y3AZzyf6, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, service_violation={\x0a\x0a}, conn=, extract_orig=F, extract_resp=F, thresholds=, removal_hooks=, smtp=, smtp_state=] diff --git a/testing/btest/Baseline/language.spurious-table-expires/out b/testing/btest/Baseline/language.spurious-table-expires/out new file mode 100644 index 0000000000..ba5bc037ba --- /dev/null +++ b/testing/btest/Baseline/language.spurious-table-expires/out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expire, new, 42 diff --git a/testing/btest/Baseline/opt.confluence-test/output b/testing/btest/Baseline/opt.confluence-test/output new file mode 100644 index 0000000000..6365f5d82a --- /dev/null +++ b/testing/btest/Baseline/opt.confluence-test/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0 diff --git a/testing/btest/Baseline/opt.table-choose/output b/testing/btest/Baseline/opt.table-choose/output new file mode 100644 index 0000000000..bba45d1455 --- /dev/null +++ b/testing/btest/Baseline/opt.table-choose/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +2, 12 diff --git a/testing/btest/Baseline/opt.vector-any-concretize1/output b/testing/btest/Baseline/opt.vector-any-concretize1/output new file mode 100644 index 0000000000..b81e30139e --- /dev/null +++ b/testing/btest/Baseline/opt.vector-any-concretize1/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[5, 3, 9] diff --git a/testing/btest/Baseline/opt.vector-any-concretize2/output b/testing/btest/Baseline/opt.vector-any-concretize2/output new file mode 100644 index 0000000000..71a8d3a7d3 --- /dev/null +++ b/testing/btest/Baseline/opt.vector-any-concretize2/output @@ -0,0 +1,9 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[0.0] +[0.0, 1.0] +[0.0, 1.0, 2.0] +[0.0, 1.0, 2.0, 3.0] +[1.0, 2.0, 3.0, 3.0] +[2.0, 3.0, 3.0, 3.0] +[3.0, 3.0, 3.0, 3.0] +[3.0, 3.0, 3.0, 3.0] diff --git a/testing/btest/language/spurious-table-expires.zeek b/testing/btest/language/spurious-table-expires.zeek new file mode 100644 index 0000000000..fa0e0120ab --- /dev/null +++ b/testing/btest/language/spurious-table-expires.zeek @@ -0,0 +1,28 @@ +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT >out +# @TEST-EXEC: btest-diff out + +# Default timer expiration interval is very conservative (10sec) and never runs for short pcaps. +redef table_expire_interval = 0.01sec; + +function f(t: table[string] of count, k: string): interval + { + print "expire", k, t[k]; + return 0.0sec; + } + +global t: table[string] of count &create_expire=0.1sec &expire_func=f; + +# Populate the initial table with two entries. +event zeek_init() &priority=5 + { + t["a"] = 10; + t["b"] = 20; + } + +# Replace global t, deleting all entries. In a DEBUG build, table continued +# to exist and its entries spuriously expired over time. +event zeek_init() + { + t = table() &create_expire=0.1sec &expire_func=f; + t["new"] = 42; + } diff --git a/testing/btest/opt/confluence-test.zeek b/testing/btest/opt/confluence-test.zeek new file mode 100644 index 0000000000..8c83bfa636 --- /dev/null +++ b/testing/btest/opt/confluence-test.zeek @@ -0,0 +1,28 @@ +# @TEST-DOC: Regression test of ZAM analysis of complex variable "confluence". +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" +# +# @TEST-EXEC: zeek -b -O ZAM %INPUT >output +# @TEST-EXEC: btest-diff output + +global my_T: bool; + +event zeek_init() + { + local vi: vector of int; + local outer_var: int; + outer_var = 0; + + # This used to throw an assertion regarding the usage regions + # associated with outer_var. + for ( i in vi ) + for ( n in vi ) + if ( my_T ) + break; + else + { + outer_var = 1; + break; + } + + print outer_var; + } diff --git a/testing/btest/opt/table-choose.zeek b/testing/btest/opt/table-choose.zeek new file mode 100644 index 0000000000..7c59fa24d3 --- /dev/null +++ b/testing/btest/opt/table-choose.zeek @@ -0,0 +1,14 @@ +# @TEST-DOC: Regression test for past ZAM issues with for-loop table "choose". +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" +# +# @TEST-EXEC: zeek -b -O ZAM %INPUT >output +# @TEST-EXEC: btest-diff output + +event zeek_init() + { + local v = table([1] = 4, [2] = 12); + for ( i1, i2 in v ) + break; + + print i1, i2; + } diff --git a/testing/btest/opt/vector-any-concretize1.zeek b/testing/btest/opt/vector-any-concretize1.zeek new file mode 100644 index 0000000000..cbc5525b2e --- /dev/null +++ b/testing/btest/opt/vector-any-concretize1.zeek @@ -0,0 +1,26 @@ +# @TEST-DOC: Regression test for past ZAM issues with vector-of-any. +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" +# +# @TEST-EXEC: zeek -b -O ZAM %INPUT >output +# @TEST-EXEC: btest-diff output + +function vector_copy(v: vector of any): vector of any + { + # This seems like an unnecessary initialization given the ensuing + # copy, but we preserve it because it's from the original script + # that triggered the need for some fixes, hence it's the full + # regression. + local v2 = copy(v); + + for ( i in v ) + v2[i] = v[i]; + + return v2; + } + +event zeek_init() + { + local v = vector(5, 3, 9); + local v_copy = vector_copy(v); + print v_copy; + } diff --git a/testing/btest/opt/vector-any-concretize2.zeek b/testing/btest/opt/vector-any-concretize2.zeek new file mode 100644 index 0000000000..813dc8955f --- /dev/null +++ b/testing/btest/opt/vector-any-concretize2.zeek @@ -0,0 +1,29 @@ +# @TEST-DOC: Regression test for past ZAM issues with vector-of-any. +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" +# +# @TEST-EXEC: zeek -b -O ZAM %INPUT >output +# @TEST-EXEC: btest-diff output + +global d: table[string] of vector of double &default=vector(); + +function crank_one(key: string) + { + local c = d[key]; + c += |c|; + print c; + if ( |c| > 3 ) + c = c[1:]; + d[key] = c; + } + +event zeek_init() + { + crank_one("foo"); + crank_one("foo"); + crank_one("foo"); + crank_one("foo"); + crank_one("foo"); + crank_one("foo"); + crank_one("foo"); + crank_one("foo"); + }