diff --git a/CHANGES b/CHANGES index 3d19f78638..cf191b3039 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,15 @@ +5.0.0-dev.204 | 2022-03-25 15:31:21 -0700 + + * --event-trace / -E option to generate event trace (Vern Paxson, Corelight) + + * hooks to support event tracing (Vern Paxson, Corelight) + + * classes providing event-tracing/dumping functionality (Vern Paxson, Corelight) + + * provide access to Val internals for event tracing purposes (Vern Paxson, Corelight) + + * set_network_time() BiF in support of event replaying (Vern Paxson, Corelight) + 5.0.0-dev.195 | 2022-03-24 11:01:28 -0700 * switch variable initialization over to being expression-based (Vern Paxson, Corelight) diff --git a/VERSION b/VERSION index 54123c2df6..bb66556ea5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.0-dev.195 +5.0.0-dev.204 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d8a82b92bb..c69137403f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -313,6 +313,7 @@ set(MAIN_SRCS EventHandler.cc EventLauncher.cc EventRegistry.cc + EventTrace.cc Expr.cc File.cc Flare.cc diff --git a/src/EventHandler.h b/src/EventHandler.h index ed981ae7e4..f47ff173d8 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -20,7 +20,7 @@ class EventHandler public: explicit EventHandler(std::string name); - const char* Name() { return name.data(); } + const char* Name() const { return name.data(); } const FuncPtr& GetFunc() { return local; } diff --git a/src/EventTrace.cc b/src/EventTrace.cc new file mode 100644 index 0000000000..e55e368e00 --- /dev/null +++ b/src/EventTrace.cc @@ -0,0 +1,1079 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/EventTrace.h" + +#include + +#include "zeek/Desc.h" +#include "zeek/EventHandler.h" +#include "zeek/Func.h" +#include "zeek/IPAddr.h" +#include "zeek/Reporter.h" +#include "zeek/ZeekString.h" + +namespace zeek::detail + { + +std::unique_ptr etm; + +// Helper function for generating a correct script-level representation +// of a string constant. +static std::string escape_string(const u_char* b, int len) + { + std::string res = "\""; + + for ( int i = 0; i < len; ++i ) + { + unsigned char c = b[i]; + + switch ( c ) + { + case '\a': + res += "\\a"; + break; + case '\b': + res += "\\b"; + break; + case '\f': + res += "\\f"; + break; + case '\n': + res += "\\n"; + break; + case '\r': + res += "\\r"; + break; + case '\t': + res += "\\t"; + break; + case '\v': + res += "\\v"; + break; + + case '\\': + res += "\\\\"; + break; + case '"': + res += "\\\""; + break; + + default: + if ( isprint(c) ) + res += c; + else + { + char buf[8192]; + snprintf(buf, sizeof buf, "%03o", c); + res += "\\"; + res += buf; + } + break; + } + } + + return res + "\""; + } + +ValTrace::ValTrace(const ValPtr& _v) : v(_v) + { + t = v->GetType(); + + switch ( t->Tag() ) + { + case TYPE_LIST: + TraceList(cast_intrusive(v)); + break; + + case TYPE_RECORD: + TraceRecord(cast_intrusive(v)); + break; + + case TYPE_TABLE: + TraceTable(cast_intrusive(v)); + break; + + case TYPE_VECTOR: + TraceVector(cast_intrusive(v)); + break; + + default: + break; + } + } + +ValTrace::~ValTrace() { } + +bool ValTrace::operator==(const ValTrace& vt) const + { + auto& vt_v = vt.GetVal(); + if ( vt_v == v ) + return true; + + auto tag = t->Tag(); + + if ( vt.GetType()->Tag() != tag ) + return false; + + switch ( tag ) + { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_ENUM: + return v->AsInt() == vt_v->AsInt(); + + case TYPE_COUNT: + case TYPE_PORT: + return v->AsCount() == vt_v->AsCount(); + + case TYPE_DOUBLE: + case TYPE_INTERVAL: + case TYPE_TIME: + return v->AsDouble() == vt_v->AsDouble(); + + case TYPE_STRING: + return (*v->AsString()) == (*vt_v->AsString()); + + case TYPE_ADDR: + return v->AsAddr() == vt_v->AsAddr(); + + case TYPE_SUBNET: + return v->AsSubNet() == vt_v->AsSubNet(); + + case TYPE_FUNC: + return v->AsFile() == vt_v->AsFile(); + + case TYPE_FILE: + return v->AsFile() == vt_v->AsFile(); + + case TYPE_PATTERN: + return v->AsPattern() == vt_v->AsPattern(); + + case TYPE_ANY: + return v->AsSubNet() == vt_v->AsSubNet(); + + case TYPE_TYPE: + return v->AsType() == vt_v->AsType(); + + case TYPE_OPAQUE: + return false; // needs pointer equivalence + + case TYPE_LIST: + return SameList(vt); + + case TYPE_RECORD: + return SameRecord(vt); + + case TYPE_TABLE: + return SameTable(vt); + + case TYPE_VECTOR: + return SameVector(vt); + + default: + reporter->InternalError("bad type in ValTrace::operator=="); + } + } + +void ValTrace::ComputeDelta(const ValTrace* prev, DeltaVector& deltas) const + { + auto tag = t->Tag(); + + if ( prev ) + { + ASSERT(prev->GetType()->Tag() == tag); + + auto& prev_v = prev->GetVal(); + + if ( prev_v != v ) + { + if ( *this != *prev ) + deltas.emplace_back(std::make_unique(this, v)); + return; + } + } + + switch ( tag ) + { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_ENUM: + case TYPE_COUNT: + case TYPE_PORT: + case TYPE_DOUBLE: + case TYPE_INTERVAL: + case TYPE_TIME: + case TYPE_STRING: + case TYPE_ADDR: + case TYPE_SUBNET: + case TYPE_FUNC: + case TYPE_PATTERN: + case TYPE_TYPE: + // These don't change in place. No need to create + // them as stand-alone variables, since we can just + // use the constant representation instead. + break; + + case TYPE_FILE: + case TYPE_OPAQUE: + case TYPE_ANY: + // These we have no way of creating as constants. + reporter->Error("cannot generate an event trace for an event of type %s", + type_name(tag)); + break; + + case TYPE_LIST: + // We shouldn't see these exposed directly, as they're + // not manipulable at script-level. An exception + // might be for "any" types that are then decomposed + // via compound assignment; for now, we don't support + // those. + reporter->InternalError("list type seen in ValTrace::ComputeDelta"); + break; + + case TYPE_RECORD: + if ( prev ) + ComputeRecordDelta(prev, deltas); + else + deltas.emplace_back(std::make_unique(this)); + break; + + case TYPE_TABLE: + if ( prev ) + ComputeTableDelta(prev, deltas); + + else if ( t->Yield() ) + deltas.emplace_back(std::make_unique(this)); + else + deltas.emplace_back(std::make_unique(this)); + break; + + case TYPE_VECTOR: + if ( prev ) + ComputeVectorDelta(prev, deltas); + else + deltas.emplace_back(std::make_unique(this)); + break; + + default: + reporter->InternalError("bad type in ValTrace::ComputeDelta"); + } + } + +void ValTrace::TraceList(const ListValPtr& lv) + { + auto vals = lv->Vals(); + for ( auto& v : vals ) + elems.emplace_back(std::make_shared(v)); + } + +void ValTrace::TraceRecord(const RecordValPtr& rv) + { + auto n = rv->NumFields(); + auto rt = rv->GetType(); + + for ( auto i = 0U; i < n; ++i ) + { + auto f = rv->RawOptField(i); + if ( f ) + { + auto val = f->ToVal(rt->GetFieldType(i)); + elems.emplace_back(std::make_shared(val)); + } + else + elems.emplace_back(nullptr); + } + } + +void ValTrace::TraceTable(const TableValPtr& tv) + { + for ( auto& elem : tv->ToMap() ) + { + auto& key = elem.first; + elems.emplace_back(std::make_shared(key)); + + auto& val = elem.second; + if ( val ) + elems2.emplace_back(std::make_shared(val)); + } + } + +void ValTrace::TraceVector(const VectorValPtr& vv) + { + auto& vec = vv->RawVec(); + auto n = vec->size(); + auto& yt = vv->RawYieldType(); + auto& yts = vv->RawYieldTypes(); + + for ( auto i = 0U; i < n; ++i ) + { + auto& elem = (*vec)[i]; + if ( elem ) + { + auto& t = yts ? (*yts)[i] : yt; + auto val = elem->ToVal(t); + elems.emplace_back(std::make_shared(val)); + } + else + elems.emplace_back(nullptr); + } + } + +bool ValTrace::SameList(const ValTrace& vt) const + { + return SameElems(vt); + } + +bool ValTrace::SameRecord(const ValTrace& vt) const + { + return SameElems(vt); + } + +bool ValTrace::SameTable(const ValTrace& vt) const + { + auto& vt_elems = vt.elems; + auto n = elems.size(); + if ( n != vt_elems.size() ) + return false; + + auto& vt_elems2 = vt.elems2; + auto n2 = elems2.size(); + if ( n2 != vt_elems2.size() ) + return false; + + ASSERT(n2 == 0 || n == n2); + + // We accommodate the possibility that keys are out-of-order + // between the two sets of elements. + + // The following is O(N^2), but presumably if tables are somehow + // involved (in fact we can only get here if they're used as + // indices into other tables), then they'll likely be small. + for ( auto i = 0U; i < n; ++i ) + { + auto& elem_i = elems[i]; + + // See if we can find a match for it. If we do, we don't + // have to worry that another entry matched it too, since + // all table/set indices will be distinct. + auto j = 0U; + for ( ; j < n; ++j ) + { + auto& vt_elem_j = vt_elems[j]; + if ( *elem_i == *vt_elem_j ) + break; + } + + if ( j == n ) + // No match for the index. + return false; + + if ( n2 > 0 ) + { + // Need a match for the corresponding yield values. + if ( *elems2[i] != *vt_elems2[j] ) + return false; + } + } + + return true; + } + +bool ValTrace::SameVector(const ValTrace& vt) const + { + return SameElems(vt); + } + +bool ValTrace::SameElems(const ValTrace& vt) const + { + auto& vt_elems = vt.elems; + auto n = elems.size(); + if ( n != vt_elems.size() ) + return false; + + for ( auto i = 0U; i < n; ++i ) + { + auto& trace_i = elems[i]; + auto& vt_trace_i = vt_elems[i]; + + if ( trace_i && vt_trace_i ) + { + if ( *trace_i != *vt_trace_i ) + return false; + } + + else if ( trace_i || vt_trace_i ) + return false; + } + + return true; + } + +bool ValTrace::SameSingleton(const ValTrace& vt) const + { + return ! IsAggr(t) && *this == vt; + } + +void ValTrace::ComputeRecordDelta(const ValTrace* prev, DeltaVector& deltas) const + { + auto& prev_elems = prev->elems; + auto n = elems.size(); + if ( n != prev_elems.size() ) + reporter->InternalError("size inconsistency in ValTrace::ComputeRecordDelta"); + + for ( auto i = 0U; i < n; ++i ) + { + const auto trace_i = elems[i].get(); + const auto prev_trace_i = prev_elems[i].get(); + + if ( trace_i ) + { + if ( prev_trace_i ) + { + auto& v = trace_i->GetVal(); + auto& prev_v = prev_trace_i->GetVal(); + + if ( v == prev_v ) + { + trace_i->ComputeDelta(prev_trace_i, deltas); + continue; + } + + if ( trace_i->SameSingleton(*prev_trace_i) ) + // No further work needed. + continue; + } + + deltas.emplace_back(std::make_unique(this, i, trace_i->GetVal())); + } + + else if ( prev_trace_i ) + deltas.emplace_back(std::make_unique(this, i)); + } + } + +void ValTrace::ComputeTableDelta(const ValTrace* prev, DeltaVector& deltas) const + { + auto& prev_elems = prev->elems; + auto& prev_elems2 = prev->elems2; + + auto n = elems.size(); + auto is_set = elems2.size() == 0; + auto prev_n = prev_elems.size(); + + // We can't compare pointers for the indices because they're + // new objects generated afresh by TableVal::ToMap. So we do + // explict full comparisons for equality, distinguishing values + // newly added, common to both, or (implicitly) removed. We'll + // then go through the common to check them further. + // + // Our approach is O(N^2), but presumably these tables aren't + // large, and in any case generating event traces is not something + // requiring high performance, so we opt for conceptual simplicity. + + // Track which index values are newly added: + std::set added_indices; + + // Track which entry traces are in common. Indexed by previous + // trace elem index, yielding current trace elem index. + std::map common_entries; + + for ( auto i = 0U; i < n; ++i ) + { + const auto trace_i = elems[i].get(); + + bool common = false; + + for ( auto j = 0U; j < prev_n; ++j ) + { + const auto prev_trace_j = prev_elems[j].get(); + + if ( *trace_i == *prev_trace_j ) + { + common_entries[j] = i; + common = true; + break; + } + } + + if ( ! common ) + { + auto v = trace_i->GetVal(); + + if ( is_set ) + deltas.emplace_back(std::make_unique(this, v)); + else + { + auto yield = elems2[i]->GetVal(); + deltas.emplace_back(std::make_unique(this, v, yield)); + } + + added_indices.insert(v.get()); + } + } + + for ( auto j = 0U; j < prev_n; ++j ) + { + auto common_pair = common_entries.find(j); + if ( common_pair == common_entries.end() ) + { + auto& prev_trace = prev_elems[j]; + auto& v = prev_trace->GetVal(); + deltas.emplace_back(std::make_unique(this, v)); + continue; + } + + if ( is_set ) + continue; + + // If we get here, we're analyzing a table for which there's + // a common index. The remaining question is whether the + // yield has changed. + auto i = common_pair->second; + auto& trace2 = elems2[i]; + const auto prev_trace2 = prev_elems2[j]; + + auto& yield = trace2->GetVal(); + auto& prev_yield = prev_trace2->GetVal(); + + if ( yield == prev_yield ) + // Same yield, look for differences in its sub-elements. + trace2->ComputeDelta(prev_trace2.get(), deltas); + + else if ( ! trace2->SameSingleton(*prev_trace2) ) + deltas.emplace_back( + std::make_unique(this, elems[i]->GetVal(), yield)); + } + } + +void ValTrace::ComputeVectorDelta(const ValTrace* prev, DeltaVector& deltas) const + { + auto& prev_elems = prev->elems; + auto n = elems.size(); + auto prev_n = prev_elems.size(); + + // TODO: The following hasn't been tested for robustness to vector holes. + + if ( n < prev_n ) + { + // The vector shrank in size. Easiest to just build it + // from scratch. + deltas.emplace_back(std::make_unique(this)); + return; + } + + // Look for existing entries that need reassigment. + auto i = 0U; + for ( ; i < prev_n; ++i ) + { + const auto trace_i = elems[i].get(); + const auto prev_trace_i = prev_elems[i].get(); + + auto& elem_i = trace_i->GetVal(); + auto& prev_elem_i = prev_trace_i->GetVal(); + + if ( elem_i == prev_elem_i ) + trace_i->ComputeDelta(prev_trace_i, deltas); + else if ( ! trace_i->SameSingleton(*prev_trace_i) ) + deltas.emplace_back(std::make_unique(this, i, elem_i)); + } + + // Now append any new entries. + for ( ; i < n; ++i ) + { + auto& trace_i = elems[i]; + auto& elem_i = trace_i->GetVal(); + deltas.emplace_back(std::make_unique(this, i, elem_i)); + } + } + +std::string ValDelta::Generate(ValTraceMgr* vtm) const + { + return ""; + } + +std::string DeltaReplaceValue::Generate(ValTraceMgr* vtm) const + { + return std::string(" = ") + vtm->ValName(new_val); + } + +std::string DeltaSetField::Generate(ValTraceMgr* vtm) const + { + auto rt = vt->GetType()->AsRecordType(); + auto f = rt->FieldName(field); + return std::string("$") + f + " = " + vtm->ValName(new_val); + } + +std::string DeltaRemoveField::Generate(ValTraceMgr* vtm) const + { + auto rt = vt->GetType()->AsRecordType(); + auto f = rt->FieldName(field); + return std::string("delete ") + vtm->ValName(vt) + "$" + f; + } + +std::string DeltaRecordCreate::Generate(ValTraceMgr* vtm) const + { + auto rv = cast_intrusive(vt->GetVal()); + auto rt = rv->GetType(); + auto n = rt->NumFields(); + + std::string args; + + for ( auto i = 0; i < n; ++i ) + { + auto v_i = rv->GetField(i); + if ( v_i ) + { + if ( ! args.empty() ) + args += ", "; + + args += std::string("$") + rt->FieldName(i) + "=" + vtm->ValName(v_i); + } + } + + auto name = rt->GetName(); + if ( name.empty() ) + name = "record"; + + return std::string(" = ") + name + "(" + args + ")"; + } + +std::string DeltaSetSetEntry::Generate(ValTraceMgr* vtm) const + { + return std::string("add ") + vtm->ValName(vt) + "[" + vtm->ValName(index) + "]"; + } + +std::string DeltaSetTableEntry::Generate(ValTraceMgr* vtm) const + { + return std::string("[") + vtm->ValName(index) + "] = " + vtm->ValName(new_val); + } + +std::string DeltaRemoveTableEntry::Generate(ValTraceMgr* vtm) const + { + return std::string("delete ") + vtm->ValName(vt) + "[" + vtm->ValName(index) + "]"; + } + +std::string DeltaSetCreate::Generate(ValTraceMgr* vtm) const + { + auto sv = cast_intrusive(vt->GetVal()); + auto members = sv->ToMap(); + + std::string args; + + for ( auto& m : members ) + { + if ( ! args.empty() ) + args += ", "; + + args += vtm->ValName(m.first); + } + + auto name = sv->GetType()->GetName(); + if ( name.empty() ) + name = "set"; + + return std::string(" = ") + name + "(" + args + ")"; + } + +std::string DeltaTableCreate::Generate(ValTraceMgr* vtm) const + { + auto tv = cast_intrusive(vt->GetVal()); + auto members = tv->ToMap(); + + std::string args; + + for ( auto& m : members ) + { + if ( ! args.empty() ) + args += ", "; + + args += std::string("[") + vtm->ValName(m.first) + "] = " + vtm->ValName(m.second); + } + + auto name = tv->GetType()->GetName(); + if ( name.empty() ) + name = "table"; + + return std::string(" = ") + name + "(" + args + ")"; + } + +std::string DeltaVectorSet::Generate(ValTraceMgr* vtm) const + { + return std::string("[") + std::to_string(index) + "] = " + vtm->ValName(elem); + } + +std::string DeltaVectorAppend::Generate(ValTraceMgr* vtm) const + { + return std::string("[") + std::to_string(index) + "] = " + vtm->ValName(elem); + } + +std::string DeltaVectorCreate::Generate(ValTraceMgr* vtm) const + { + auto& elems = vt->GetElems(); + std::string vec; + + for ( auto& e : elems ) + { + if ( vec.size() > 0 ) + vec += ", "; + + vec += vtm->ValName(e->GetVal()); + } + + return std::string(" = vector(") + vec + ")"; + } + +EventTrace::EventTrace(const ScriptFunc* _ev, double _nt, int event_num) : ev(_ev), nt(_nt) + { + auto ev_name = std::regex_replace(ev->Name(), std::regex(":"), "_"); + + name = ev_name + "_" + std::to_string(event_num) + "__et"; + } + +void EventTrace::Generate(FILE* f, ValTraceMgr& vtm, const DeltaGenVec& dvec, std::string successor, + int num_pre) const + { + int offset = 0; + for ( auto& d : dvec ) + { + auto& val = d.GetVal(); + + if ( d.IsFirstDef() && vtm.IsGlobal(val) ) + { + auto& val_name = vtm.ValName(val); + + std::string type_name; + auto& t = val->GetType(); + auto& tn = t->GetName(); + if ( tn.empty() ) + { + ODesc d; + t->Describe(&d); + type_name = d.Description(); + } + else + type_name = tn; + + auto anno = offset < num_pre ? " # from script" : ""; + + fprintf(f, "global %s: %s;%s\n", val_name.c_str(), type_name.c_str(), anno); + } + + ++offset; + } + + fprintf(f, "event %s()\n", name.c_str()); + fprintf(f, "\t{\n"); + + offset = 0; + for ( auto& d : dvec ) + { + fprintf(f, "\t"); + + auto& val = d.GetVal(); + + if ( d.IsFirstDef() && ! vtm.IsGlobal(val) ) + fprintf(f, "local "); + + if ( d.NeedsLHS() ) + fprintf(f, "%s", vtm.ValName(val).c_str()); + + auto anno = offset < num_pre ? " # from script" : ""; + + fprintf(f, "%s;%s\n", d.RHS().c_str(), anno); + + ++offset; + } + + if ( ! dvec.empty() ) + fprintf(f, "\n"); + + fprintf(f, "\tevent %s(%s);\n\n", ev->Name(), args.c_str()); + + if ( successor.empty() ) + { + // The following isn't necessary with our current approach + // to managing chains of events, which avoids having to set + // exit_only_after_terminate=T. + // fprintf(f, "\tterminate();\n"); + } + else + { + fprintf(f, "\tset_network_time(double_to_time(%.06f));\n", nt); + fprintf(f, "\tevent __EventTrace::%s();\n", successor.c_str()); + } + + fprintf(f, "\t}\n"); + } + +void EventTrace::Generate(FILE* f, ValTraceMgr& vtm, const EventTrace* predecessor, + std::string successor) const + { + if ( predecessor ) + { + auto& pre_deltas = predecessor->post_deltas; + int num_pre = pre_deltas.size(); + + if ( num_pre > 0 ) + { + auto total_deltas = pre_deltas; + total_deltas.insert(total_deltas.end(), deltas.begin(), deltas.end()); + Generate(f, vtm, total_deltas, successor, num_pre); + return; + } + } + + Generate(f, vtm, deltas, successor); + } + +void ValTraceMgr::TraceEventValues(std::shared_ptr et, const zeek::Args* args) + { + curr_ev = std::move(et); + + auto num_vals = vals.size(); + + std::string ev_args; + for ( auto& a : *args ) + { + AddVal(a); + + if ( ! ev_args.empty() ) + ev_args += ", "; + + ev_args += ValName(a); + } + + curr_ev->SetArgs(ev_args); + + // Now look for any values newly-processed with this event and + // remember them so we can catch uses of them in future events. + for ( auto i = num_vals; i < vals.size(); ++i ) + { + processed_vals.insert(vals[i].get()); + ASSERT(val_names.count(vals[i].get()) > 0); + } + } + +void ValTraceMgr::FinishCurrentEvent(const zeek::Args* args) + { + auto num_vals = vals.size(); + + curr_ev->SetDoingPost(); + + for ( auto& a : *args ) + AddVal(a); + + for ( auto i = num_vals; i < vals.size(); ++i ) + processed_vals.insert(vals[i].get()); + } + +const std::string& ValTraceMgr::ValName(const ValPtr& v) + { + auto find = val_names.find(v.get()); + if ( find == val_names.end() ) + { + if ( IsAggr(v->GetType()) ) + { // Aggregate shouldn't exist; create it + ASSERT(val_map.count(v.get()) == 0); + NewVal(v); + find = val_names.find(v.get()); + } + + else + { // Non-aggregate can be expressed using a constant + auto tag = v->GetType()->Tag(); + std::string rep; + + if ( tag == TYPE_STRING ) + { + auto s = v->AsStringVal(); + rep = escape_string(s->Bytes(), s->Len()); + } + + else if ( tag == TYPE_LIST ) + { + auto lv = cast_intrusive(v); + for ( auto& v_i : lv->Vals() ) + { + if ( ! rep.empty() ) + rep += ", "; + + rep += ValName(v_i); + } + } + + else if ( tag == TYPE_FUNC ) + rep = v->AsFunc()->Name(); + + else if ( tag == TYPE_TIME ) + rep = std::string("double_to_time(") + std::to_string(v->AsDouble()) + ")"; + + else if ( tag == TYPE_INTERVAL ) + rep = std::string("double_to_interval(") + std::to_string(v->AsDouble()) + ")"; + + else + { + ODesc d; + v->Describe(&d); + rep = d.Description(); + } + + val_names[v.get()] = rep; + vals.push_back(v); + find = val_names.find(v.get()); + } + + ASSERT(find != val_names.end()); + } + + ValUsed(v); + + return find->second; + } + +void ValTraceMgr::AddVal(ValPtr v) + { + auto mapping = val_map.find(v.get()); + + if ( mapping == val_map.end() ) + NewVal(v); + else + { + auto vt = std::make_shared(v); + AssessChange(vt.get(), mapping->second.get()); + val_map[v.get()] = vt; + } + } + +void ValTraceMgr::NewVal(ValPtr v) + { + // Make sure the Val sticks around into the future. + vals.push_back(v); + + auto vt = std::make_shared(v); + AssessChange(vt.get(), nullptr); + val_map[v.get()] = vt; + } + +void ValTraceMgr::ValUsed(const ValPtr& v) + { + ASSERT(val_names.count(v.get()) > 0); + if ( processed_vals.count(v.get()) > 0 ) + // We saw this value when processing a previous event. + globals.insert(v.get()); + } + +void ValTraceMgr::AssessChange(const ValTrace* vt, const ValTrace* prev_vt) + { + DeltaVector deltas; + + vt->ComputeDelta(prev_vt, deltas); + + // Used to track deltas across the batch, to suppress redundant ones + // (which can arise due to two aggregates both including the same + // sub-element). + std::unordered_set previous_deltas; + + for ( auto& d : deltas ) + { + auto vp = d->GetValTrace()->GetVal(); + auto v = vp.get(); + auto rhs = d->Generate(this); + + bool needs_lhs = d->NeedsLHS(); + bool is_first_def = false; + + if ( needs_lhs && val_names.count(v) == 0 ) + { + TrackVar(v); + is_first_def = true; + } + + ASSERT(val_names.count(v) > 0); + + // The "/" in the following is just to have a delimiter + // to make sure the string is unambiguous. + auto full_delta = val_names[v] + "/" + rhs; + if ( previous_deltas.count(full_delta) > 0 ) + continue; + + previous_deltas.insert(full_delta); + + ValUsed(vp); + curr_ev->AddDelta(vp, rhs, needs_lhs, is_first_def); + } + + auto& v = vt->GetVal(); + if ( IsAggr(v->GetType()) ) + ValUsed(vt->GetVal()); + } + +void ValTraceMgr::TrackVar(const Val* v) + { + auto val_name = std::string("__val") + std::to_string(num_vars++); + val_names[v] = val_name; + } + +EventTraceMgr::EventTraceMgr(const std::string& trace_file) + { + f = fopen(trace_file.c_str(), "w"); + if ( ! f ) + reporter->FatalError("can't open event trace file %s", trace_file.c_str()); + } + +EventTraceMgr::~EventTraceMgr() + { + if ( events.empty() ) + return; + + fprintf(f, "module __EventTrace;\n\n"); + + for ( auto& e : events ) + fprintf(f, "global %s: event();\n", e->GetName()); + + fprintf(f, "\nevent zeek_init() &priority=-999999\n"); + fprintf(f, "\t{\n"); + fprintf(f, "\tevent __EventTrace::%s();\n", events.front()->GetName()); + fprintf(f, "\t}\n"); + + for ( auto i = 0U; i < events.size(); ++i ) + { + fprintf(f, "\n"); + + auto predecessor = i > 0 ? events[i - 1] : nullptr; + auto successor = i + 1 < events.size() ? events[i + 1]->GetName() : ""; + events[i]->Generate(f, vtm, predecessor.get(), successor); + } + + fclose(f); + } + +void EventTraceMgr::StartEvent(const ScriptFunc* ev, const zeek::Args* args) + { + if ( script_events.count(ev->Name()) > 0 ) + return; + + auto nt = run_state::network_time; + if ( nt == 0.0 ) + return; + + auto et = std::make_shared(ev, nt, events.size()); + events.emplace_back(et); + + vtm.TraceEventValues(et, args); + } + +void EventTraceMgr::EndEvent(const ScriptFunc* ev, const zeek::Args* args) + { + if ( script_events.count(ev->Name()) > 0 ) + return; + + if ( run_state::network_time > 0.0 ) + vtm.FinishCurrentEvent(args); + } + +void EventTraceMgr::ScriptEventQueued(const EventHandlerPtr& h) + { + script_events.insert(h->Name()); + } + + } // namespace zeek::detail diff --git a/src/EventTrace.h b/src/EventTrace.h new file mode 100644 index 0000000000..ff28496f6a --- /dev/null +++ b/src/EventTrace.h @@ -0,0 +1,464 @@ +// Classes for tracing/dumping Zeek events. + +#pragma once + +#include "zeek/Val.h" + +namespace zeek::detail + { + +class ValTrace; +class ValTraceMgr; + +// Abstract class for capturing a single difference between two script-level +// values. Includes notions of inserting, changing, or deleting a value. +class ValDelta + { +public: + ValDelta(const ValTrace* _vt) : vt(_vt) { } + virtual ~ValDelta() { } + + // Return a string that performs the update operation, expressed + // as Zeek scripting. Does not include a terminating semicolon. + virtual std::string Generate(ValTraceMgr* vtm) const; + + // Whether the generated string needs the affected value to + // explicitly appear on the left-hand-side. Note that this + // might not be as a simple "LHS = RHS" assignment, but instead + // as "LHS$field = RHS" or "LHS[index] = RHS". + // + // Returns false for generated strings like "delete LHS[index]". + virtual bool NeedsLHS() const { return true; } + + const ValTrace* GetValTrace() const { return vt; } + +protected: + const ValTrace* vt; + }; + +using DeltaVector = std::vector>; + +// Tracks the elements of a value as seen at a given point in execution. +// For non-aggregates, this is simply the Val object, but for aggregates +// it is (recursively) each of the sub-elements, in a manner that can then +// be readily compared against future instances. +class ValTrace + { +public: + ValTrace(const ValPtr& v); + ~ValTrace(); + + const ValPtr& GetVal() const { return v; } + const TypePtr& GetType() const { return t; } + const auto& GetElems() const { return elems; } + + // Returns true if this trace and the given one represent the + // same underlying value. Can involve subelement-by-subelement + // (recursive) comparisons. + bool operator==(const ValTrace& vt) const; + bool operator!=(const ValTrace& vt) const { return ! ((*this) == vt); } + + // Computes the deltas between a previous ValTrace and this one. + // If "prev" is nil then we're creating this value from scratch + // (though if it's an aggregate, we may reuse existing values + // for some of its components). + // + // Returns the accumulated differences in "deltas". If on return + // nothing was added to "deltas" then the two ValTrace's are equivalent + // (no changes between them). + void ComputeDelta(const ValTrace* prev, DeltaVector& deltas) const; + +private: + // Methods for tracing different types of aggregate values. + void TraceList(const ListValPtr& lv); + void TraceRecord(const RecordValPtr& rv); + void TraceTable(const TableValPtr& tv); + void TraceVector(const VectorValPtr& vv); + + // Predicates for comparing different types of aggregates for equality. + bool SameList(const ValTrace& vt) const; + bool SameRecord(const ValTrace& vt) const; + bool SameTable(const ValTrace& vt) const; + bool SameVector(const ValTrace& vt) const; + + // Helper function that knows about the internal vector-of-subelements + // we use for aggregates. + bool SameElems(const ValTrace& vt) const; + + // True if this value is a singleton and it's the same value as + // represented in "vt". + bool SameSingleton(const ValTrace& vt) const; + + // Add to "deltas" the differences needed to turn a previous instance + // of the given type of aggregate to the current instance. + void ComputeRecordDelta(const ValTrace* prev, DeltaVector& deltas) const; + void ComputeTableDelta(const ValTrace* prev, DeltaVector& deltas) const; + void ComputeVectorDelta(const ValTrace* prev, DeltaVector& deltas) const; + + // Holds sub-elements for aggregates. + std::vector> elems; + + // A parallel vector used for the yield values of tables. + std::vector> elems2; + + ValPtr v; + TypePtr t; // v's type, for convenience + }; + +// Captures the basic notion of a new, non-equivalent value being assigned. +class DeltaReplaceValue : public ValDelta + { +public: + DeltaReplaceValue(const ValTrace* _vt, ValPtr _new_val) + : ValDelta(_vt), new_val(std::move(_new_val)) + { + } + + std::string Generate(ValTraceMgr* vtm) const override; + +private: + ValPtr new_val; + }; + +// Captures the notion of setting a record field. +class DeltaSetField : public ValDelta + { +public: + DeltaSetField(const ValTrace* _vt, int _field, ValPtr _new_val) + : ValDelta(_vt), field(_field), new_val(std::move(_new_val)) + { + } + + std::string Generate(ValTraceMgr* vtm) const override; + +private: + int field; + ValPtr new_val; + }; + +// Captures the notion of deleting a record field. +class DeltaRemoveField : public ValDelta + { +public: + DeltaRemoveField(const ValTrace* _vt, int _field) : ValDelta(_vt), field(_field) { } + + std::string Generate(ValTraceMgr* vtm) const override; + bool NeedsLHS() const override { return false; } + +private: + int field; + }; + +// Captures the notion of creating a record from scratch. +class DeltaRecordCreate : public ValDelta + { +public: + DeltaRecordCreate(const ValTrace* _vt) : ValDelta(_vt) { } + + std::string Generate(ValTraceMgr* vtm) const override; + }; + +// Captures the notion of adding an element to a set. Use DeltaRemoveTableEntry to +// delete values. +class DeltaSetSetEntry : public ValDelta + { +public: + DeltaSetSetEntry(const ValTrace* _vt, ValPtr _index) : ValDelta(_vt), index(_index) { } + + std::string Generate(ValTraceMgr* vtm) const override; + bool NeedsLHS() const override { return false; } + +private: + ValPtr index; + }; + +// Captures the notion of setting a table entry (which includes both changing +// an existing one and adding a new one). Use DeltaRemoveTableEntry to +// delete values. +class DeltaSetTableEntry : public ValDelta + { +public: + DeltaSetTableEntry(const ValTrace* _vt, ValPtr _index, ValPtr _new_val) + : ValDelta(_vt), index(_index), new_val(std::move(_new_val)) + { + } + + std::string Generate(ValTraceMgr* vtm) const override; + +private: + ValPtr index; + ValPtr new_val; + }; + +// Captures the notion of removing a table/set entry. +class DeltaRemoveTableEntry : public ValDelta + { +public: + DeltaRemoveTableEntry(const ValTrace* _vt, ValPtr _index) + : ValDelta(_vt), index(std::move(_index)) + { + } + + std::string Generate(ValTraceMgr* vtm) const override; + bool NeedsLHS() const override { return false; } + +private: + ValPtr index; + }; + +// Captures the notion of creating a set from scratch. +class DeltaSetCreate : public ValDelta + { +public: + DeltaSetCreate(const ValTrace* _vt) : ValDelta(_vt) { } + + std::string Generate(ValTraceMgr* vtm) const override; + }; + +// Captures the notion of creating a table from scratch. +class DeltaTableCreate : public ValDelta + { +public: + DeltaTableCreate(const ValTrace* _vt) : ValDelta(_vt) { } + + std::string Generate(ValTraceMgr* vtm) const override; + }; + +// Captures the notion of changing an element of a vector. +class DeltaVectorSet : public ValDelta + { +public: + DeltaVectorSet(const ValTrace* _vt, int _index, ValPtr _elem) + : ValDelta(_vt), index(_index), elem(std::move(_elem)) + { + } + + std::string Generate(ValTraceMgr* vtm) const override; + +private: + int index; + ValPtr elem; + }; + +// Captures the notion of adding an entry to the end of a vector. +class DeltaVectorAppend : public ValDelta + { +public: + DeltaVectorAppend(const ValTrace* _vt, int _index, ValPtr _elem) + : ValDelta(_vt), index(_index), elem(std::move(_elem)) + { + } + + std::string Generate(ValTraceMgr* vtm) const override; + +private: + int index; + ValPtr elem; + }; + +// Captures the notion of replacing a vector wholesale. +class DeltaVectorCreate : public ValDelta + { +public: + DeltaVectorCreate(const ValTrace* _vt) : ValDelta(_vt) { } + + std::string Generate(ValTraceMgr* vtm) const override; + +private: + }; + +// Manages the changes to (or creation of) a variable used to represent +// a value. +class DeltaGen + { +public: + DeltaGen(ValPtr _val, std::string _rhs, bool _needs_lhs, bool _is_first_def) + : val(std::move(_val)), rhs(std::move(_rhs)), needs_lhs(_needs_lhs), + is_first_def(_is_first_def) + { + } + + const ValPtr& GetVal() const { return val; } + const std::string& RHS() const { return rhs; } + bool NeedsLHS() const { return needs_lhs; } + bool IsFirstDef() const { return is_first_def; } + +private: + ValPtr val; + + // The expression to set the variable to. + std::string rhs; + + // Whether that expression needs the variable explicitly provides + // on the lefthand side. + bool needs_lhs; + + // Whether this is the first definition of the variable (in which + // case we also need to declare the variable). + bool is_first_def; + }; + +using DeltaGenVec = std::vector; + +// Tracks a single event. +class EventTrace + { +public: + // Constructed in terms of the associated script function, "network + // time" when the event occurred, and the position of this event + // within all of those being traced. + EventTrace(const ScriptFunc* _ev, double _nt, int event_num); + + // Sets a string representation of the arguments (values) being + // passed to the event. + void SetArgs(std::string _args) { args = std::move(_args); } + + // Adds to the trace an update for the given value. + void AddDelta(ValPtr val, std::string rhs, bool needs_lhs, bool is_first_def) + { + auto& d = is_post ? post_deltas : deltas; + d.emplace_back(DeltaGen(val, rhs, needs_lhs, is_first_def)); + } + + // Initially we analyze events pre-execution. When this flag + // is set, we switch to instead analyzing post-execution. The + // difference allows us to annotate the output with "# from script" + // comments that flag changes created by script execution rather + // than event engine activity. + void SetDoingPost() { is_post = true; } + + const char* GetName() const { return name.c_str(); } + + // Generates an internal event handler that sets up the values + // associated with the traced event, followed by queueing the traced + // event, and then queueing the successor internal event handler, + // if any. + // + // "predecessor", if non-nil, gives the event that came just before + // this one (used for "# from script" annotations"). "successor", + // if not empty, gives the name of the successor internal event. + void Generate(FILE* f, ValTraceMgr& vtm, const EventTrace* predecessor, + std::string successor) const; + +private: + // "dvec" is either just our deltas, or the "post_deltas" of our + // predecessor plus our deltas. + void Generate(FILE* f, ValTraceMgr& vtm, const DeltaGenVec& dvec, std::string successor, + int num_pre = 0) const; + + const ScriptFunc* ev; + double nt; + bool is_post = false; + + // The deltas needed to construct the values associated with this + // event prior to its execution. + DeltaGenVec deltas; + + // The deltas capturing any changes to the original values as induced + // by executing its event handlers. + DeltaGenVec post_deltas; + + // The event's name and a string representation of its arguments. + std::string name; + std::string args; + }; + +// Manages all of the events and associated values seen during the execution. +class ValTraceMgr + { +public: + // Invoked to trace a new event with the associated arguments. + void TraceEventValues(std::shared_ptr et, const zeek::Args* args); + + // Invoked when the current event finishes execution. The arguments + // are again provided, for convenience so we don't have to remember + // them from the previous method. + void FinishCurrentEvent(const zeek::Args* args); + + // Returns the name of the script variable associated with the + // given value. + const std::string& ValName(const ValPtr& v); + const std::string& ValName(const ValTrace* vt) { return ValName(vt->GetVal()); } + + // Returns true if the script variable associated with the given value + // needs to be global (because it's used across multiple events). + bool IsGlobal(const ValPtr& v) const { return globals.count(v.get()) > 0; } + +private: + // Traces the given value, which we may-or-may-not have seen before. + void AddVal(ValPtr v); + + // Creates a new value, associating a script variable with it. + void NewVal(ValPtr v); + + // Called when the given value is used in an expression that sets + // or updates another value. This lets us track which values are + // used across multiple events, and thus need to be global. + void ValUsed(const ValPtr& v); + + // Compares the two value traces to build up deltas capturing + // the difference between the previous one and the current one. + void AssessChange(const ValTrace* vt, const ValTrace* prev_vt); + + // Create and track a script variable associated with the given value. + void TrackVar(const Val* vt); + + // Maps values to their associated traces. + std::unordered_map> val_map; + + // Maps values to the "names" we associated with them. For simple + // values, the name is just a Zeek script constant. For aggregates, + // it's a dedicated script variable. + std::unordered_map val_names; + int num_vars = 0; // the number of dedicated script variables + + // Tracks which values we've processed up through the preceding event. + // Any re-use we then see for the current event (via a ValUsed() call) + // then tells us that the value is used across events, and thus its + // associated script variable needs to be global. + std::unordered_set processed_vals; + + // Tracks which values have associated script variables that need + // to be global. + std::unordered_set globals; + + // The event we're currently tracing. + std::shared_ptr curr_ev; + + // Hang on to values we're tracking to make sure the pointers don't + // get reused when the main use of the value ends. + std::vector vals; + }; + +// Manages tracing of all of the events seen during execution, including +// the final generation of the trace script. +class EventTraceMgr + { +public: + EventTraceMgr(const std::string& trace_file); + ~EventTraceMgr(); + + // Called at the beginning of invoking an event's handlers. + void StartEvent(const ScriptFunc* ev, const zeek::Args* args); + + // Called after finishing with invoking an event's handlers. + void EndEvent(const ScriptFunc* ev, const zeek::Args* args); + + // Used to track events generated at script-level. + void ScriptEventQueued(const EventHandlerPtr& h); + +private: + FILE* f = nullptr; + ValTraceMgr vtm; + + // All of the events we've traced so far. + std::vector> events; + + // The names of all of the script events that have been generated. + std::unordered_set script_events; + }; + +// If non-nil then we're doing event tracing. +extern std::unique_ptr etm; + + } // namespace zeek::detail diff --git a/src/Expr.cc b/src/Expr.cc index 4f79c8448a..572c615a1a 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -8,6 +8,7 @@ #include "zeek/Desc.h" #include "zeek/Event.h" #include "zeek/EventRegistry.h" +#include "zeek/EventTrace.h" #include "zeek/Frame.h" #include "zeek/Func.h" #include "zeek/Hash.h" @@ -4322,7 +4323,14 @@ ValPtr ScheduleExpr::Eval(Frame* f) const auto args = eval_list(f, event->Args()); if ( args ) - timer_mgr->Add(new ScheduleTimer(event->Handler(), std::move(*args), dt)); + { + auto handler = event->Handler(); + + if ( etm ) + etm->ScriptEventQueued(handler); + + timer_mgr->Add(new ScheduleTimer(handler, std::move(*args), dt)); + } return nullptr; } @@ -4861,7 +4869,12 @@ ValPtr EventExpr::Eval(Frame* f) const auto v = eval_list(f, args.get()); if ( handler ) + { + if ( etm ) + etm->ScriptEventQueued(handler); + event_mgr.Enqueue(handler, std::move(*v)); + } return nullptr; } diff --git a/src/Func.cc b/src/Func.cc index 282c65dde8..992acc9e0e 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -33,6 +33,7 @@ #include "zeek/Debug.h" #include "zeek/Desc.h" #include "zeek/Event.h" +#include "zeek/EventTrace.h" #include "zeek/Expr.h" #include "zeek/File.h" #include "zeek/Frame.h" @@ -401,6 +402,9 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const const CallExpr* call_expr = parent ? parent->GetCall() : nullptr; call_stack.emplace_back(CallInfo{call_expr, this, *args}); + if ( etm && Flavor() == FUNC_FLAVOR_EVENT ) + etm->StartEvent(this, args); + if ( g_trace_state.DoTrace() ) { ODesc d; @@ -481,6 +485,9 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const result = val_mgr->True(); } + else if ( etm && Flavor() == FUNC_FLAVOR_EVENT ) + etm->EndEvent(this, args); + // Warn if the function returns something, but we returned from // the function without an explicit return, or without a value. else if ( GetType()->Yield() && GetType()->Yield()->Tag() != TYPE_VOID && diff --git a/src/Options.cc b/src/Options.cc index ccc9ddae5e..0cf23a4fd6 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -114,6 +114,8 @@ void usage(const char* prog, int code) #endif fprintf(stderr, " -C|--no-checksums | ignore checksums\n"); fprintf(stderr, " -D|--deterministic | initialize random seeds to zero\n"); + fprintf(stderr, " -E|--event-trace | generate a replayable event trace to " + "the given file\n"); fprintf(stderr, " -F|--force-dns | force DNS\n"); fprintf(stderr, " -G|--load-seeds | load seeds from given file\n"); fprintf(stderr, " -H|--save-seeds | save seeds to given file\n"); @@ -380,6 +382,7 @@ Options parse_cmdline(int argc, char** argv) {"no-checksums", no_argument, nullptr, 'C'}, {"force-dns", no_argument, nullptr, 'F'}, {"deterministic", no_argument, nullptr, 'D'}, + {"event-trace", required_argument, nullptr, 'E'}, {"load-seeds", required_argument, nullptr, 'G'}, {"save-seeds", required_argument, nullptr, 'H'}, {"print-plugins", no_argument, nullptr, 'N'}, @@ -400,7 +403,7 @@ Options parse_cmdline(int argc, char** argv) {"mem-profile", no_argument, nullptr, 'M'}, #endif - {"pseudo-realtime", optional_argument, nullptr, 'E'}, + {"pseudo-realtime", optional_argument, nullptr, '~'}, {"jobs", optional_argument, nullptr, 'j'}, {"test", no_argument, nullptr, '#'}, @@ -408,7 +411,7 @@ Options parse_cmdline(int argc, char** argv) }; char opts[256]; - util::safe_strncpy(opts, "B:c:e:f:G:H:I:i:j::n:O:0:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv", + util::safe_strncpy(opts, "B:c:E:e:f:G:H:I:i:j::n:O:0:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv", sizeof(opts)); int op; @@ -523,9 +526,7 @@ Options parse_cmdline(int argc, char** argv) rval.deterministic_mode = true; break; case 'E': - rval.pseudo_realtime = 1.0; - if ( optarg ) - rval.pseudo_realtime = atof(optarg); + rval.event_trace_file = optarg; break; case 'F': if ( rval.dns_mode != detail::DNS_DEFAULT ) @@ -586,6 +587,12 @@ Options parse_cmdline(int argc, char** argv) break; #endif + case '~': + rval.pseudo_realtime = 1.0; + if ( optarg ) + rval.pseudo_realtime = atof(optarg); + break; + case '#': fprintf(stderr, "ERROR: --test only allowed as first argument.\n"); usage(zargs[0], 1); diff --git a/src/Options.h b/src/Options.h index b9bd1c5e9b..0065b527b9 100644 --- a/src/Options.h +++ b/src/Options.h @@ -73,6 +73,7 @@ struct Options std::optional process_status_file; std::optional zeekygen_config_file; std::optional unprocessed_output_file; + std::optional event_trace_file; std::set plugins_to_load; std::vector scripts_to_load; diff --git a/src/Stmt.cc b/src/Stmt.cc index 0467dad9ee..23b3faea4c 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -8,6 +8,7 @@ #include "zeek/Debug.h" #include "zeek/Desc.h" #include "zeek/Event.h" +#include "zeek/EventTrace.h" #include "zeek/Expr.h" #include "zeek/File.h" #include "zeek/Frame.h" @@ -1076,11 +1077,17 @@ EventStmt::EventStmt(EventExprPtr arg_e) : ExprStmt(STMT_EVENT, arg_e), event_ex ValPtr EventStmt::Exec(Frame* f, StmtFlowType& flow) { RegisterAccess(); + auto args = eval_list(f, event_expr->Args()); auto h = event_expr->Handler(); if ( args && h ) + { + if ( etm ) + etm->ScriptEventQueued(h); + event_mgr.Enqueue(h, std::move(*args)); + } flow = FLOW_NEXT; return nullptr; diff --git a/src/Val.h b/src/Val.h index 8f78a59f10..3e578a52d7 100644 --- a/src/Val.h +++ b/src/Val.h @@ -47,6 +47,7 @@ class PrefixTable; class CompositeHash; class HashKey; +class ValTrace; class ZBody; } // namespace detail @@ -1380,6 +1381,7 @@ public: static void DoneParsing(); protected: + friend class zeek::detail::ValTrace; friend class zeek::detail::ZBody; RecordValPtr DoCoerceTo(RecordTypePtr other, bool allow_orphaning) const; @@ -1401,9 +1403,9 @@ protected: record_val->emplace_back(std::nullopt); } - // For use by low-level ZAM instructions. Caller assumes - // responsibility for memory management. The first version - // allows manipulation of whether the field is present at all. + // For internal use by low-level ZAM instructions and event tracing. + // Caller assumes responsibility for memory management. The first + // version allows manipulation of whether the field is present at all. // The second version ensures that the optional value is present. std::optional& RawOptField(int field) { return (*record_val)[field]; } @@ -1614,10 +1616,13 @@ public: } const String* StringAt(unsigned int index) const { return StringValAt(index)->AsString(); } - // Only intended for low-level access by compiled code. + // Only intended for low-level access by internal or compiled code. const auto& RawVec() const { return vector_val; } auto& RawVec() { return vector_val; } + const auto& RawYieldType() const { return yield_type; } + const auto& RawYieldTypes() const { return yield_types; } + protected: /** * Returns the element at a given index or nullptr if it does not exist. diff --git a/src/analyzer/Analyzer.h b/src/analyzer/Analyzer.h index 6d56fa129a..4659f7af2b 100644 --- a/src/analyzer/Analyzer.h +++ b/src/analyzer/Analyzer.h @@ -605,6 +605,8 @@ public: * use this method to attach additional data to the connections. A * call to BuildConnVal() will in turn trigger a call to * UpdateConnVal(). + * TODO: The above comment needs updating, there's no BuildConnVal() + * anymore -VP * * @param conn_val The connenction value being updated. */ diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 2ae69ca061..561270c3c6 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -26,6 +26,7 @@ #include "zeek/Desc.h" #include "zeek/Event.h" #include "zeek/EventRegistry.h" +#include "zeek/EventTrace.h" #include "zeek/File.h" #include "zeek/Frag.h" #include "zeek/Frame.h" @@ -281,13 +282,13 @@ static void done_with_network() ZEEK_LSAN_DISABLE(); } -static void terminate_bro() +static void terminate_zeek() { - util::detail::set_processing_status("TERMINATING", "terminate_bro"); + util::detail::set_processing_status("TERMINATING", "terminate_zeek"); run_state::terminating = true; - iosource_mgr->Wakeup("terminate_bro"); + iosource_mgr->Wakeup("terminate_zeek"); // File analysis termination may produce events, so do it early on in // the termination process. @@ -299,8 +300,19 @@ static void terminate_bro() event_mgr.Enqueue(zeek_done, Args{}); timer_mgr->Expire(); + + // Drain() limits how many "generations" of newly created events + // it will process. When we're terminating, however, we're okay + // with long chains of events, and this makes the workings of + // event-tracing simpler. + // + // That said, we also need to ensure that it runs at least once, + // as it has side effects such as tickling triggers. event_mgr.Drain(); + while ( event_mgr.HasEvents() ) + event_mgr.Drain(); + if ( profiling_logger ) { // FIXME: There are some occasional crashes in the memory @@ -658,6 +670,9 @@ SetupResult setup(int argc, char** argv, Options* zopts) }; auto ipbb = make_intrusive(init_bifs, ipbid->Name(), false); + if ( options.event_trace_file ) + etm = make_unique(*options.event_trace_file); + run_state::is_parsing = true; yyparse(); run_state::is_parsing = false; @@ -943,7 +958,7 @@ int cleanup(bool did_run_loop) done_with_network(); run_state::detail::delete_run(); - terminate_bro(); + terminate_zeek(); sqlite3_shutdown(); @@ -974,7 +989,7 @@ void zeek_terminate_loop(const char* reason) zeek::detail::done_with_network(); delete_run(); - zeek::detail::terminate_bro(); + zeek::detail::terminate_zeek(); // Close files after net_delete(), because net_delete() // might write to connection content files. diff --git a/src/zeek.bif b/src/zeek.bif index 37a329f9d4..911a010761 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -321,7 +321,7 @@ static int next_fmt(const char*& fmt, const zeek::Args* args, zeek::ODesc* d, in ## ## Returns: The wall-clock time. ## -## .. zeek:see:: network_time +## .. zeek:see:: network_time set_network_time function current_time%(%): time %{ return zeek::make_intrusive(zeek::util::current_time()); @@ -333,12 +333,26 @@ function current_time%(%): time ## ## Returns: The timestamp of the packet processed. ## -## .. zeek:see:: current_time +## .. zeek:see:: current_time set_network_time function network_time%(%): time %{ return zeek::make_intrusive(zeek::run_state::network_time); %} +## Sets the timestamp associated with the last packet processed. Used for +## event replaying. +## +## nt: The time to which to set "network time". +## +## Returns: The timestamp of the packet processed. +## +## .. zeek:see:: current_time network_time +function set_network_time%(nt: time%): bool + %{ + zeek::run_state::network_time = nt; + return zeek::val_mgr->True(); + %} + ## Returns a system environment variable. ## ## var: The name of the variable whose value to request.