From 1419803dbdadb44f63451a21e0ed5dc2c93b84ab Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Mon, 8 May 2023 17:37:53 -0700 Subject: [PATCH] enhancements for event-tracing: - reporting of potentially sensitive constants - tracking of unsupported types enabling hand-editing to fix them - fixed generation of "unspecified" aggregates - fixed generation of IPv6 constants - fixed generation when running without a packet source --- src/EventTrace.cc | 269 +++++++++++++++++++++++++++++++++++----------- src/EventTrace.h | 43 +++++++- 2 files changed, 247 insertions(+), 65 deletions(-) diff --git a/src/EventTrace.cc b/src/EventTrace.cc index 3ca3f66941..ff52f079da 100644 --- a/src/EventTrace.cc +++ b/src/EventTrace.cc @@ -213,12 +213,16 @@ void ValTrace::ComputeDelta(const ValTrace* prev, DeltaVector& deltas) const // use the constant representation instead. break; + case TYPE_ANY: case TYPE_FILE: case TYPE_OPAQUE: - case TYPE_ANY: - // These we have no way of creating as constants. - reporter->Error("cannot generate an event trace for an event of type %s", - type_name(tag)); + // If we have a previous instance, we can ignore this + // one, because we know it's equivalent (due to the + // test at the beginning of this method), and it's + // not meaningful to recurse inside it looking for + // interior changes. + if ( ! prev ) + deltas.emplace_back(std::make_unique(this)); break; case TYPE_LIST: @@ -241,6 +245,13 @@ void ValTrace::ComputeDelta(const ValTrace* prev, DeltaVector& deltas) const if ( prev ) ComputeTableDelta(prev, deltas); + else if ( GetType()->AsTableType()->IsUnspecifiedTable() ) + // For unspecified values, we generate them + // as empty constructors, because we don't + // know their yield type and thus can't + // create variables corresponding to them. + break; + else if ( t->Yield() ) deltas.emplace_back(std::make_unique(this)); else @@ -250,6 +261,11 @@ void ValTrace::ComputeDelta(const ValTrace* prev, DeltaVector& deltas) const case TYPE_VECTOR: if ( prev ) ComputeVectorDelta(prev, deltas); + + else if ( GetType()->AsVectorType()->IsUnspecifiedVector() ) + // See above for empty tables/sets. + break; + else deltas.emplace_back(std::make_unique(this)); break; @@ -722,6 +738,11 @@ std::string DeltaVectorCreate::Generate(ValTraceMgr* vtm) const return std::string(" = vector(") + vec + ")"; } +std::string DeltaUnsupportedCreate::Generate(ValTraceMgr* vtm) const + { + return " = UNSUPPORTED " + obj_desc_short(vt->GetVal()->GetType().get()); + } + EventTrace::EventTrace(const ScriptFunc* _ev, double _nt, size_t event_num) : ev(_ev), nt(_nt) { auto ev_name = std::regex_replace(ev->Name(), std::regex(":"), "_"); @@ -770,13 +791,19 @@ void EventTrace::Generate(FILE* f, ValTraceMgr& vtm, const DeltaGenVec& dvec, st fprintf(f, "\t"); auto& val = d.GetVal(); + bool define_local = d.IsFirstDef() && ! vtm.IsGlobal(val); - if ( d.IsFirstDef() && ! vtm.IsGlobal(val) ) + if ( define_local ) fprintf(f, "local "); if ( d.NeedsLHS() ) + { fprintf(f, "%s", vtm.ValName(val).c_str()); + if ( define_local ) + fprintf(f, ": %s", obj_desc_short(val->GetType().get()).c_str()); + } + auto anno = offset < num_pre ? " # from script" : ""; fprintf(f, "%s;%s\n", d.RHS().c_str(), anno); @@ -798,7 +825,8 @@ void EventTrace::Generate(FILE* f, ValTraceMgr& vtm, const DeltaGenVec& dvec, st } else { - fprintf(f, "\tset_network_time(double_to_time(%.06f));\n", nt); + auto tm = vtm.TimeConstant(nt); + fprintf(f, "\tset_network_time(%s);\n", tm.c_str()); fprintf(f, "\tevent __EventTrace::%s();\n", successor.c_str()); } @@ -870,66 +898,28 @@ const std::string& ValTraceMgr::ValName(const ValPtr& v) { auto find = val_names.find(v.get()); if ( find == val_names.end() ) - { - if ( IsAggr(v->GetType()) ) - { // Aggregate shouldn't exist; create it - ASSERT(val_map.count(v.get()) == 0); - NewVal(v); - find = val_names.find(v.get()); - } - - else - { // Non-aggregate can be expressed using a constant - auto tag = v->GetType()->Tag(); - std::string rep; - - if ( tag == TYPE_STRING ) - { - auto s = v->AsStringVal(); - rep = escape_string(s->Bytes(), s->Len()); - } - - else if ( tag == TYPE_LIST ) - { - auto lv = cast_intrusive(v); - for ( auto& v_i : lv->Vals() ) - { - if ( ! rep.empty() ) - rep += ", "; - - rep += ValName(v_i); - } - } - - else if ( tag == TYPE_FUNC ) - rep = v->AsFunc()->Name(); - - else if ( tag == TYPE_TIME ) - rep = std::string("double_to_time(") + std::to_string(v->AsDouble()) + ")"; - - else if ( tag == TYPE_INTERVAL ) - rep = std::string("double_to_interval(") + std::to_string(v->AsDouble()) + ")"; - - else - { - ODesc d; - v->Describe(&d); - rep = d.Description(); - } - - val_names[v.get()] = rep; - vals.push_back(v); - find = val_names.find(v.get()); - } - - ASSERT(find != val_names.end()); - } + find = val_names.insert({v.get(), GenValName(v)}).first; ValUsed(v); return find->second; } +std::string ValTraceMgr::TimeConstant(double t) + { + if ( t < std::max(base_time, 1e6) ) + return "double_to_time(" + std::to_string(t) + ")"; + + if ( ! base_time ) + base_time = t; + + if ( t == base_time ) + return "double_to_time(__base_time)"; + + t -= base_time; + return "double_to_time(__base_time + " + std::to_string(t) + ")"; + } + void ValTraceMgr::AddVal(ValPtr v) { auto mapping = val_map.find(v.get()); @@ -1003,16 +993,143 @@ void ValTraceMgr::AssessChange(const ValTrace* vt, const ValTrace* prev_vt) } auto& v = vt->GetVal(); - if ( IsAggr(v->GetType()) ) + if ( IsAggr(v->GetType()) && (prev_vt || ! IsUnspecifiedAggregate(v)) ) ValUsed(vt->GetVal()); } void ValTraceMgr::TrackVar(const Val* v) { - auto val_name = std::string("__val") + std::to_string(num_vars++); + std::string base_name = IsUnsupported(v) ? "UNSUPPORTED" : "val"; + auto val_name = "__" + base_name + std::to_string(num_vars++); val_names[v] = val_name; } +std::string ValTraceMgr::GenValName(const ValPtr& v) + { + if ( IsAggr(v->GetType()) && ! IsUnspecifiedAggregate(v) ) + { // Aggregate shouldn't exist; create it + ASSERT(val_map.count(v.get()) == 0); + NewVal(v); + return val_names[v.get()]; + } + + // Non-aggregate (or unspecified aggregate) can be expressed using + // a constant. + auto t = v->GetType(); + auto tag = t->Tag(); + std::string rep; + bool track_constant = false; + + switch ( tag ) + { + case TYPE_STRING: + { + auto s = v->AsStringVal(); + rep = escape_string(s->Bytes(), s->Len()); + track_constant = s->Len() > 0; + break; + } + + case TYPE_LIST: + { + auto lv = cast_intrusive(v); + for ( auto& v_i : lv->Vals() ) + { + if ( ! rep.empty() ) + rep += ", "; + + rep += ValName(v_i); + } + break; + } + + case TYPE_FUNC: + rep = v->AsFunc()->Name(); + break; + + case TYPE_TIME: + { + auto tm = v->AsDouble(); + rep = TimeConstant(tm); + + if ( tm > 0.0 && rep.find("__base_time") == std::string::npos ) + // We're not representing it using base_time. + track_constant = true; + + break; + } + + case TYPE_INTERVAL: + rep = "double_to_interval(" + std::to_string(v->AsDouble()) + ")"; + break; + + case TYPE_TABLE: + rep = t->Yield() ? "table()" : "set()"; + break; + + case TYPE_VECTOR: + rep = "vector()"; + break; + + case TYPE_PATTERN: + case TYPE_PORT: + case TYPE_ADDR: + case TYPE_SUBNET: + { + ODesc d; + v->Describe(&d); + rep = d.Description(); + track_constant = true; + + if ( tag == TYPE_ADDR || tag == TYPE_SUBNET ) + { + // Fix up deficiency that IPv6 addresses are + // described without surrounding []'s. + const auto& addr = tag == TYPE_ADDR ? v->AsAddr() : v->AsSubNet().Prefix(); + if ( addr.GetFamily() == IPv6 ) + rep = "[" + rep + "]"; + } + } + break; + + default: + { + ODesc d; + v->Describe(&d); + rep = d.Description(); + } + } + + val_names[v.get()] = rep; + vals.push_back(v); + + if ( track_constant ) + constants[tag].insert(rep); + + std::array constants; + + return rep; + } + +bool ValTraceMgr::IsUnspecifiedAggregate(const ValPtr& v) const + { + auto t = v->GetType()->Tag(); + + if ( t == TYPE_TABLE && v->GetType()->IsUnspecifiedTable() ) + return true; + + if ( t == TYPE_VECTOR && v->GetType()->IsUnspecifiedVector() ) + return true; + + return false; + } + +bool ValTraceMgr::IsUnsupported(const Val* v) const + { + auto t = v->GetType()->Tag(); + return t == TYPE_ANY || t == TYPE_FILE || t == TYPE_OPAQUE; + } + EventTraceMgr::EventTraceMgr(const std::string& trace_file) { f = fopen(trace_file.c_str(), "w"); @@ -1027,6 +1144,11 @@ EventTraceMgr::~EventTraceMgr() fprintf(f, "module __EventTrace;\n\n"); + auto bt = vtm.GetBaseTime(); + + if ( bt ) + fprintf(f, "global __base_time = %.06f;\n\n", bt); + for ( auto& e : events ) fprintf(f, "global %s: event();\n", e->GetName()); @@ -1044,6 +1166,22 @@ EventTraceMgr::~EventTraceMgr() events[i]->Generate(f, vtm, predecessor.get(), successor); } + const auto& constants = vtm.GetConstants(); + + for ( auto tag = 0; tag < NUM_TYPES; ++tag ) + { + auto& c_t = constants[tag]; + if ( c_t.empty() && (tag != TYPE_TIME || ! bt) ) + continue; + + fprintf(f, "\n# constants of type %s:\n", type_name(TypeTag(tag))); + if ( tag == TYPE_TIME && bt ) + fprintf(f, "#\t__base_time = %.06f\n", bt); + + for ( auto& c : c_t ) + fprintf(f, "#\t%s\n", c.c_str()); + } + fclose(f); } @@ -1053,9 +1191,12 @@ void EventTraceMgr::StartEvent(const ScriptFunc* ev, const zeek::Args* args) return; auto nt = run_state::network_time; - if ( nt == 0.0 ) + if ( nt == 0.0 || util::streq(ev->Name(), "zeek_init") ) return; + if ( ! vtm.GetBaseTime() ) + vtm.SetBaseTime(nt); + auto et = std::make_shared(ev, nt, events.size()); events.emplace_back(et); @@ -1067,7 +1208,7 @@ void EventTraceMgr::EndEvent(const ScriptFunc* ev, const zeek::Args* args) if ( script_events.count(ev->Name()) > 0 ) return; - if ( run_state::network_time > 0.0 ) + if ( run_state::network_time > 0.0 && ! util::streq(ev->Name(), "zeek_init") ) vtm.FinishCurrentEvent(args); } diff --git a/src/EventTrace.h b/src/EventTrace.h index 9a0b95e5be..d958cd058d 100644 --- a/src/EventTrace.h +++ b/src/EventTrace.h @@ -264,8 +264,16 @@ public: DeltaVectorCreate(const ValTrace* _vt) : ValDelta(_vt) { } std::string Generate(ValTraceMgr* vtm) const override; + }; -private: +// Captures the notion of creating a value with an unsupported type +// (like "opaque"). +class DeltaUnsupportedCreate : public ValDelta + { +public: + DeltaUnsupportedCreate(const ValTrace* _vt) : ValDelta(_vt) { } + + std::string Generate(ValTraceMgr* vtm) const override; }; // Manages the changes to (or creation of) a variable used to represent @@ -385,6 +393,19 @@ public: // needs to be global (because it's used across multiple events). bool IsGlobal(const ValPtr& v) const { return globals.count(v.get()) > 0; } + // Returns or sets the "base time" from which eligible times are + // transformed into offsets rather than maintained as absolute + // values. + double GetBaseTime() const { return base_time; } + void SetBaseTime(double bt) { base_time = bt; } + + // Returns a Zeek script representation of the given "time" value. + // This might be relative to base_time or might be absolute. + std::string TimeConstant(double t); + + // Returns the array of per-type-tag constants. + const auto& GetConstants() const { return constants; } + private: // Traces the given value, which we may-or-may-not have seen before. void AddVal(ValPtr v); @@ -404,6 +425,17 @@ private: // Create and track a script variable associated with the given value. void TrackVar(const Val* vt); + // Generates a name for a value. + std::string GenValName(const ValPtr& v); + + // True if the given value is an unspecified (and empty set, + // table, or vector appearing as a constant rather than an + // already-typed value). + bool IsUnspecifiedAggregate(const ValPtr& v) const; + + // True if the given value has an unsupported type. + bool IsUnsupported(const Val* v) const; + // Maps values to their associated traces. std::unordered_map> val_map; @@ -423,6 +455,15 @@ private: // to be global. std::unordered_set globals; + // Indexed by type tag, stores an ordered set of all of the distinct + // representations of constants of that type. + std::array, NUM_TYPES> constants; + + // If non-zero, then we've established a "base time" and will report + // time constants as offsets from it (when reasonable, i.e., no + // negative offsets, and base_time can't be too close to 0.0). + double base_time = 0.0; + // The event we're currently tracing. std::shared_ptr curr_ev;