From 00f93411838fd08cbcbbbf2113ef858abdb9f72d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 5 Jun 2019 18:46:51 +0000 Subject: [PATCH 01/22] Couple of compile fixes. This is branched from topic/johanna/remove-serializer. --- src/analyzer/protocol/krb/KRB.cc | 2 ++ src/probabilistic/CounterVector.cc | 1 + 2 files changed, 3 insertions(+) diff --git a/src/analyzer/protocol/krb/KRB.cc b/src/analyzer/protocol/krb/KRB.cc index 4ee663dcf1..e6bd42b961 100644 --- a/src/analyzer/protocol/krb/KRB.cc +++ b/src/analyzer/protocol/krb/KRB.cc @@ -1,5 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include + #include "KRB.h" #include "types.bif.h" #include "events.bif.h" diff --git a/src/probabilistic/CounterVector.cc b/src/probabilistic/CounterVector.cc index 1a3c98c73f..e234d5c9d9 100644 --- a/src/probabilistic/CounterVector.cc +++ b/src/probabilistic/CounterVector.cc @@ -2,6 +2,7 @@ #include "CounterVector.h" +#include #include #include "BitVector.h" From c0c5dccd065f0568b7b9144f074c13d98045dfb1 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 7 Jun 2019 22:55:13 +0000 Subject: [PATCH 02/22] Add new test for when-statement watching global variables. --- aux/broker | 2 +- aux/btest | 2 +- aux/netcontrol-connectors | 2 +- aux/zeek-aux | 2 +- aux/zeekctl | 2 +- doc | 2 +- .../Baseline/language.when-on-globals/out | 4 ++ testing/btest/language/when-on-globals.zeek | 71 +++++++++++++++++++ 8 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 testing/btest/Baseline/language.when-on-globals/out create mode 100644 testing/btest/language/when-on-globals.zeek diff --git a/aux/broker b/aux/broker index 0c7a8816fd..e142a362b4 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 0c7a8816fd385af4f633cb7239e3c63e6c88c27e +Subproject commit e142a362b4c712699ac43494a245058948c085c8 diff --git a/aux/btest b/aux/btest index 6ece47ba64..539c2d8253 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 6ece47ba6438e7a6db5c7b85a68b3c16f0911871 +Subproject commit 539c2d82534345c62ba9a20c2e98ea5cbdea9c7e diff --git a/aux/netcontrol-connectors b/aux/netcontrol-connectors index e93235aa6e..43da5d80fd 160000 --- a/aux/netcontrol-connectors +++ b/aux/netcontrol-connectors @@ -1 +1 @@ -Subproject commit e93235aa6e45820af7e23e97627845a7b2b3d919 +Subproject commit 43da5d80fdf0923e790af3c21749f6e6241cda80 diff --git a/aux/zeek-aux b/aux/zeek-aux index 3ecc7b8c34..bac443d6ce 160000 --- a/aux/zeek-aux +++ b/aux/zeek-aux @@ -1 +1 @@ -Subproject commit 3ecc7b8c348a7b768092dad75e6cb54c6357b9d7 +Subproject commit bac443d6cebca567d3d0da52a25ff4e0bcdd1edd diff --git a/aux/zeekctl b/aux/zeekctl index a955e66c8b..37275b29f5 160000 --- a/aux/zeekctl +++ b/aux/zeekctl @@ -1 +1 @@ -Subproject commit a955e66c8b07fd6715c7ed379d0759acc592bb78 +Subproject commit 37275b29f5fe5fa68808229907a6f7bebcddafdf diff --git a/doc b/doc index e5422eafff..f8869ea5f7 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit e5422eafff850708f4d4ff590e54299ddc97ca42 +Subproject commit f8869ea5f72cda06d5acf0fbeefd5af102f7d77e diff --git a/testing/btest/Baseline/language.when-on-globals/out b/testing/btest/Baseline/language.when-on-globals/out new file mode 100644 index 0000000000..44dae2c89e --- /dev/null +++ b/testing/btest/Baseline/language.when-on-globals/out @@ -0,0 +1,4 @@ +"j" in x3[20]$x, expected timeout +15 in x2, T +x1 != 42, T +x2[10], T diff --git a/testing/btest/language/when-on-globals.zeek b/testing/btest/language/when-on-globals.zeek new file mode 100644 index 0000000000..087a88b4db --- /dev/null +++ b/testing/btest/language/when-on-globals.zeek @@ -0,0 +1,71 @@ +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT | sort >out +# @TEST-EXEC: btest-diff out + +redef exit_only_after_terminate = T; + +type X: record { + s: string; + x: set[string] &optional; +}; + +global x1 = 42; +global x2: table[count] of X; +global x3: table[count] of X; + +event quit() +{ + terminate(); +} + +event zeek_init() + { + x2[10] = [$s="foo"]; + x3[20] = [$s="bar", $x=set("i")]; + + when ( x1 != 42 ) + { + print "x1 != 42", x1 != 42; + } + timeout 1sec + { + print "unexpected timeout (1)"; + } + + when ( 15 in x2 ) + { + print "15 in x2", 10 in x2; + } + timeout 1sec + { + print "unexpected timeout (2)"; + } + + when ( x2[10]$s == "bar" ) + { + print "x2[10]", x2[10]$s == "bar"; + } + timeout 1sec + { + print "unexpected timeout (3)"; + } + + when ( "j" in x3[20]$x ) + { + print "unexpected trigger"; + } + timeout 1sec + { + print "\"j\" in x3[20]$x, expected timeout"; + } + + x1 = 100; + x2[15] = [$s="xyz"]; + x2[10]$s = "bar"; + + # This will *NOT* trigger then when-condition because we're modifying + # an inner value that's not directly tracked. + add x3[20]$x["j"]; + + schedule 2secs { quit() }; +} + From 02214dafc40cf3cf7996e879b99ec6cde1f187f1 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 6 Jun 2019 02:49:18 +0000 Subject: [PATCH 03/22] Redo NotfifierRegistry to no longer rely on StateAccess. We simplify the API to a simple Modified() operation. --- src/StateAccess.cc | 109 +++++++++++++++++++++++---------------------- src/StateAccess.h | 37 ++++++++------- src/Trigger.h | 4 +- 3 files changed, 80 insertions(+), 70 deletions(-) diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 997eb5a48d..cc867468c8 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -149,7 +149,7 @@ void StateAccess::Replay() Val* v = target.id->ID_Val(); TypeTag t = v ? v->Type()->Tag() : TYPE_VOID; - + if ( opcode != OP_ASSIGN && ! v ) { // FIXME: I think this warrants an internal error, @@ -514,22 +514,17 @@ void StateAccess::Describe(ODesc* d) const void StateAccess::Log(StateAccess* access) { - bool tracked = false; - if ( access->target_type == TYPE_ID ) { if ( access->target.id->FindAttr(ATTR_TRACKED) ) - tracked = true; + notifiers.Modified(access->target.id); } else { if ( access->target.val->GetProperties() & MutableVal::TRACKED ) - tracked = true; + notifiers.Modified(access->target.val); } - if ( tracked ) - notifiers.AccessPerformed(*access); - #ifdef DEBUG ODesc desc; access->Describe(&desc); @@ -543,6 +538,15 @@ void StateAccess::Log(StateAccess* access) NotifierRegistry notifiers; +NotifierRegistry::~NotifierRegistry() + { + for ( auto i : ids ) + Unref(i.first); + + for ( auto i : vals ) + Unref(i.first); + } + void NotifierRegistry::Register(ID* id, NotifierRegistry::Notifier* notifier) { DBG_LOG(DBG_NOTIFIERS, "registering ID %s for notifier %s", @@ -563,24 +567,20 @@ void NotifierRegistry::Register(ID* id, NotifierRegistry::Notifier* notifier) Unref(attr); - NotifierMap::iterator i = ids.find(id->Name()); - - if ( i != ids.end() ) - i->second->insert(notifier); - else - { - NotifierSet* s = new NotifierSet; - s->insert(notifier); - ids.insert(NotifierMap::value_type(id->Name(), s)); - } - + ids.insert({id, notifier}); Ref(id); } void NotifierRegistry::Register(Val* val, NotifierRegistry::Notifier* notifier) { - if ( val->IsMutableVal() ) - Register(val->AsMutableVal()->UniqueID(), notifier); + if ( ! val->IsMutableVal() ) + return; + + DBG_LOG(DBG_NOTIFIERS, "registering value %p for notifier %s", + val, notifier->Name()); + + vals.insert({val, notifier}); + Ref(val); } void NotifierRegistry::Unregister(ID* id, NotifierRegistry::Notifier* notifier) @@ -588,52 +588,55 @@ void NotifierRegistry::Unregister(ID* id, NotifierRegistry::Notifier* notifier) DBG_LOG(DBG_NOTIFIERS, "unregistering ID %s for notifier %s", id->Name(), notifier->Name()); - NotifierMap::iterator i = ids.find(id->Name()); - - if ( i == ids.end() ) - return; - - Attr* attr = id->Attrs()->FindAttr(ATTR_TRACKED); - id->Attrs()->RemoveAttr(ATTR_TRACKED); - Unref(attr); - - NotifierSet* s = i->second; - s->erase(notifier); - - if ( s->size() == 0 ) + auto x = ids.equal_range(id); + for ( auto i = x.first; i != x.second; i++ ) { - delete s; - ids.erase(i); + if ( i->second == notifier ) + { + ids.erase(i); + Unref(id); + break; + } } - Unref(id); + if ( ids.find(id) == ids.end() ) + // Last registered notifier for this ID + id->Attrs()->RemoveAttr(ATTR_TRACKED); } void NotifierRegistry::Unregister(Val* val, NotifierRegistry::Notifier* notifier) { - if ( val->IsMutableVal() ) - Unregister(val->AsMutableVal()->UniqueID(), notifier); + DBG_LOG(DBG_NOTIFIERS, "unregistering Val %p for notifier %s", + val, notifier->Name()); + + auto x = vals.equal_range(val); + for ( auto i = x.first; i != x.second; i++ ) + { + if ( i->second == notifier ) + { + vals.erase(i); + Unref(val); + break; + } + } } -void NotifierRegistry::AccessPerformed(const StateAccess& sa) +void NotifierRegistry::Modified(Val *val) { - ID* id = sa.Target(); + DBG_LOG(DBG_NOTIFIERS, "modification to tracked value %p", val); - NotifierMap::iterator i = ids.find(id->Name()); - - if ( i == ids.end() ) - return; + auto x = vals.equal_range(val); + for ( auto i = x.first; i != x.second; i++ ) + i->second->Modified(val); + } +void NotifierRegistry::Modified(ID *id) + { DBG_LOG(DBG_NOTIFIERS, "modification to tracked ID %s", id->Name()); - NotifierSet* s = i->second; - - if ( id->IsInternalGlobal() ) - for ( NotifierSet::iterator j = s->begin(); j != s->end(); j++ ) - (*j)->Access(id->ID_Val(), sa); - else - for ( NotifierSet::iterator j = s->begin(); j != s->end(); j++ ) - (*j)->Access(id, sa); + auto x = ids.equal_range(id); + for ( auto i = x.first; i != x.second; i++ ) + i->second->Modified(id); } const char* NotifierRegistry::Notifier::Name() const diff --git a/src/StateAccess.h b/src/StateAccess.h index 15e2ae4676..86f1dbe88c 100644 --- a/src/StateAccess.h +++ b/src/StateAccess.h @@ -4,7 +4,7 @@ #define STATEACESSS_H #include -#include +#include #include class Val; @@ -105,32 +105,39 @@ public: public: virtual ~Notifier() { } - // Called when a change is being performed. Note that when these - // methods are called, it is undefined whether the change has - // already been done or is just going to be performed soon. - virtual void Access(ID* id, const StateAccess& sa) = 0; - virtual void Access(Val* val, const StateAccess& sa) = 0; + // Called when a change is being performed. Note that when + // these methods are called, it is undefined whether the + // change has already been done or is just going to be + // performed soon. + virtual void Modified(ID* id) = 0; + virtual void Modified(Val* val) = 0; virtual const char* Name() const; // for debugging }; NotifierRegistry() { } - ~NotifierRegistry() { } + ~NotifierRegistry(); - // Inform the given notifier if ID/Val changes. + // Register a new notifier to be informed when ID/Val changes. Note + // that the registry will store a reference to the target, keeping + // the instance alive for as long as it's registered. void Register(ID* id, Notifier* notifier); void Register(Val* val, Notifier* notifier); - // Cancel notification for this ID/Val. + // Cancel a notifier's tracking for this ID/Val, also releasing the + // referencee being held. void Unregister(ID* id, Notifier* notifier); void Unregister(Val* val, Notifier* notifier); -private: - friend class StateAccess; - void AccessPerformed(const StateAccess& sa); + // Inform all registered notifiiers of a modification to a value/ID. + void Modified(ID *id); + void Modified(Val *val); - typedef std::set NotifierSet; - typedef std::map NotifierMap; - NotifierMap ids; +private: + typedef std::unordered_multimap ValMap; + typedef std::unordered_multimap IDMap; + + ValMap vals; + IDMap ids; }; extern NotifierRegistry notifiers; diff --git a/src/Trigger.h b/src/Trigger.h index 0f7889d19a..8dc478e049 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -61,9 +61,9 @@ public: { d->Add(""); } // Overidden from Notifier. We queue the trigger and evaluate it // later to avoid race conditions. - void Access(ID* id, const StateAccess& sa) override + void Modified(ID* id) override { QueueTrigger(this); } - void Access(Val* val, const StateAccess& sa) override + void Modified(Val* val) override { QueueTrigger(this); } const char* Name() const override; From 31ddca863cf352038914415cc1e64e056914c7b2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 6 Jun 2019 03:11:00 +0000 Subject: [PATCH 04/22] Remove StateAccess class. --- src/DebugLogger.cc | 2 +- src/DebugLogger.h | 3 +- src/ID.cc | 2 +- src/StateAccess.cc | 532 --------------------------------------------- src/StateAccess.h | 63 ------ src/Val.cc | 130 +---------- src/Val.h | 15 +- src/bro.bif | 11 - 8 files changed, 14 insertions(+), 744 deletions(-) diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 8f0425270d..ed8b3e1e95 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -12,7 +12,7 @@ DebugLogger debug_logger; // Same order here as in DebugStream. DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "serial", 0, false }, { "rules", 0, false }, - { "state", 0, false }, {"string", 0, false }, + { "string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, { "logging", 0, false }, {"input", 0, false }, diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 2e24c7064f..0e2862dc23 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -16,9 +16,8 @@ enum DebugStream { DBG_SERIAL, // Serialization DBG_RULES, // Signature matching - DBG_STATE, // StateAccess logging DBG_STRING, // String code - DBG_NOTIFIERS, // Notifiers (see StateAccess.h) + DBG_NOTIFIERS, // Notifiers DBG_MAINLOOP, // Main IOSource loop DBG_ANALYZER, // Analyzer framework DBG_TM, // Time-machine packet input via Brocolli diff --git a/src/ID.cc b/src/ID.cc index 12677bec75..915b5002b9 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -79,7 +79,7 @@ void ID::SetVal(Val* v, Opcode op, bool arg_weak_ref) #else if ( debug_logger.IsVerbose() || props ) #endif - StateAccess::Log(new StateAccess(op, this, v, val)); + notifiers.Modified(this); } if ( ! weak_ref ) diff --git a/src/StateAccess.cc b/src/StateAccess.cc index cc867468c8..853a8a7879 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -4,538 +4,6 @@ #include "NetVar.h" #include "DebugLogger.h" -int StateAccess::replaying = 0; - -StateAccess::StateAccess(Opcode arg_opcode, - const MutableVal* arg_target, const Val* arg_op1, - const Val* arg_op2, const Val* arg_op3) - { - opcode = arg_opcode; - target.val = const_cast(arg_target); - target_type = TYPE_MVAL; - op1.val = const_cast(arg_op1); - op1_type = TYPE_VAL; - op2 = const_cast(arg_op2); - op3 = const_cast(arg_op3); - delete_op1_key = false; - - RefThem(); - } - -StateAccess::StateAccess(Opcode arg_opcode, - const ID* arg_target, const Val* arg_op1, - const Val* arg_op2, const Val* arg_op3) - { - opcode = arg_opcode; - target.id = const_cast(arg_target); - target_type = TYPE_ID; - op1.val = const_cast(arg_op1); - op1_type = TYPE_VAL; - op2 = const_cast(arg_op2); - op3 = const_cast(arg_op3); - delete_op1_key = false; - - RefThem(); - } - -StateAccess::StateAccess(Opcode arg_opcode, - const ID* arg_target, const HashKey* arg_op1, - const Val* arg_op2, const Val* arg_op3) - { - opcode = arg_opcode; - target.id = const_cast(arg_target); - target_type = TYPE_ID; - op1.key = new HashKey(arg_op1->Key(), arg_op1->Size(), arg_op1->Hash()); - op1_type = TYPE_KEY; - op2 = const_cast(arg_op2); - op3 = const_cast(arg_op3); - delete_op1_key = true; - - RefThem(); - } - -StateAccess::StateAccess(Opcode arg_opcode, - const MutableVal* arg_target, const HashKey* arg_op1, - const Val* arg_op2, const Val* arg_op3) - { - opcode = arg_opcode; - target.val = const_cast(arg_target); - target_type = TYPE_MVAL; - op1.key = new HashKey(arg_op1->Key(), arg_op1->Size(), arg_op1->Hash()); - op1_type = TYPE_KEY; - op2 = const_cast(arg_op2); - op3 = const_cast(arg_op3); - delete_op1_key = true; - - RefThem(); - } - -StateAccess::StateAccess(const StateAccess& sa) - { - opcode = sa.opcode; - target_type = sa.target_type; - op1_type = sa.op1_type; - delete_op1_key = false; - - if ( target_type == TYPE_ID ) - target.id = sa.target.id; - else - target.val = sa.target.val; - - if ( op1_type == TYPE_VAL ) - op1.val = sa.op1.val; - else - { - // We need to copy the key as the pointer may not be - // valid anymore later. - op1.key = new HashKey(sa.op1.key->Key(), sa.op1.key->Size(), - sa.op1.key->Hash()); - delete_op1_key = true; - } - - op2 = sa.op2; - op3 = sa.op3; - - RefThem(); - } - -StateAccess::~StateAccess() - { - if ( target_type == TYPE_ID ) - Unref(target.id); - else - Unref(target.val); - - if ( op1_type == TYPE_VAL ) - Unref(op1.val); - else if ( delete_op1_key ) - delete op1.key; - - Unref(op2); - Unref(op3); - } - -void StateAccess::RefThem() - { - if ( target_type == TYPE_ID ) - Ref(target.id); - else - Ref(target.val); - - if ( op1_type == TYPE_VAL && op1.val ) - Ref(op1.val); - - if ( op2 ) - Ref(op2); - if ( op3 ) - Ref(op3); - } - -static Val* GetInteger(bro_int_t n, TypeTag t) - { - if ( t == TYPE_INT ) - return val_mgr->GetInt(n); - - return val_mgr->GetCount(n); - } - -void StateAccess::Replay() - { - // For simplicity we assume that we only replay unserialized accesses. - assert(target_type == TYPE_ID && op1_type == TYPE_VAL); - - if ( ! target.id ) - return; - - Val* v = target.id->ID_Val(); - TypeTag t = v ? v->Type()->Tag() : TYPE_VOID; - - if ( opcode != OP_ASSIGN && ! v ) - { - // FIXME: I think this warrants an internal error, - // but let's check that first ... - // reporter->InternalError("replay id lacking a value"); - reporter->Error("replay id lacks a value"); - return; - } - - ++replaying; - - switch ( opcode ) { - case OP_ASSIGN: - assert(op1.val); - // There mustn't be a direct assignment to a unique ID. - assert(target.id->Name()[0] != '#'); - - target.id->SetVal(op1.val->Ref()); - break; - - case OP_INCR: - if ( IsIntegral(t) ) - { - assert(op1.val && op2); - // We derive the amount as difference between old - // and new value. - bro_int_t amount = - op1.val->CoerceToInt() - op2->CoerceToInt(); - - target.id->SetVal(GetInteger(v->CoerceToInt() + amount, t), - OP_INCR); - } - break; - - case OP_ASSIGN_IDX: - assert(op1.val); - - if ( t == TYPE_TABLE ) - { - assert(op2); - v->AsTableVal()->Assign(op1.val, op2 ? op2->Ref() : 0); - } - - else if ( t == TYPE_RECORD ) - { - const char* field = op1.val->AsString()->CheckString(); - int idx = v->Type()->AsRecordType()->FieldOffset(field); - - if ( idx >= 0 ) - v->AsRecordVal()->Assign(idx, op2 ? op2->Ref() : 0); - else - reporter->Error("access replay: unknown record field %s for assign", field); - } - - else if ( t == TYPE_VECTOR ) - { - assert(op2); - bro_uint_t index = op1.val->AsCount(); - v->AsVectorVal()->Assign(index, op2 ? op2->Ref() : 0); - } - - else - reporter->InternalError("unknown type in replaying index assign"); - - break; - - case OP_INCR_IDX: - { - assert(op1.val && op2 && op3); - - // We derive the amount as the difference between old - // and new value. - bro_int_t amount = op2->CoerceToInt() - op3->CoerceToInt(); - - if ( t == TYPE_TABLE ) - { - t = v->Type()->AsTableType()->YieldType()->Tag(); - Val* lookup_op1 = v->AsTableVal()->Lookup(op1.val); - int delta = lookup_op1->CoerceToInt() + amount; - Val* new_val = GetInteger(delta, t); - v->AsTableVal()->Assign(op1.val, new_val, OP_INCR ); - } - - else if ( t == TYPE_RECORD ) - { - const char* field = op1.val->AsString()->CheckString(); - int idx = v->Type()->AsRecordType()->FieldOffset(field); - if ( idx >= 0 ) - { - t = v->Type()->AsRecordType()->FieldType(idx)->Tag(); - Val* lookup_field = - v->AsRecordVal()->Lookup(idx); - bro_int_t delta = - lookup_field->CoerceToInt() + amount; - Val* new_val = GetInteger(delta, t); - v->AsRecordVal()->Assign(idx, new_val, OP_INCR); - } - else - reporter->Error("access replay: unknown record field %s for assign", field); - } - - else if ( t == TYPE_VECTOR ) - { - bro_uint_t index = op1.val->AsCount(); - t = v->Type()->AsVectorType()->YieldType()->Tag(); - Val* lookup_op1 = v->AsVectorVal()->Lookup(index); - int delta = lookup_op1->CoerceToInt() + amount; - Val* new_val = GetInteger(delta, t); - v->AsVectorVal()->Assign(index, new_val); - } - - else - reporter->InternalError("unknown type in replaying index increment"); - - break; - } - - case OP_ADD: - assert(op1.val); - if ( t == TYPE_TABLE ) - { - v->AsTableVal()->Assign(op1.val, 0); - } - break; - - case OP_DEL: - assert(op1.val); - if ( t == TYPE_TABLE ) - { - Unref(v->AsTableVal()->Delete(op1.val)); - } - break; - - case OP_EXPIRE: - assert(op1.val); - if ( t == TYPE_TABLE ) - { - // No old check for expire. It may have already - // been deleted by ourselves. Furthermore, we - // ignore the expire_func's return value. - TableVal* tv = v->AsTableVal(); - if ( tv->Lookup(op1.val, false) ) - { - // We want to propagate state updates which - // are performed in the expire_func. - StateAccess::ResumeReplay(); - - tv->CallExpireFunc(op1.val->Ref()); - - StateAccess::SuspendReplay(); - - Unref(tv->AsTableVal()->Delete(op1.val)); - } - } - - break; - - case OP_PRINT: - assert(op1.val); - reporter->InternalError("access replay for print not implemented"); - break; - - case OP_READ_IDX: - if ( t == TYPE_TABLE ) - { - assert(op1.val); - TableVal* tv = v->AsTableVal(); - - // Update the timestamp if we have a read_expire. - if ( tv->FindAttr(ATTR_EXPIRE_READ) ) - { - tv->UpdateTimestamp(op1.val); - } - } - else - reporter->Error("read for non-table"); - break; - - default: - reporter->InternalError("access replay: unknown opcode for StateAccess"); - break; - } - - --replaying; - } - -ID* StateAccess::Target() const - { - return target_type == TYPE_ID ? target.id : target.val->UniqueID(); - } - -void StateAccess::Describe(ODesc* d) const - { - const ID* id; - const char* id_str = ""; - const char* unique_str = ""; - - d->SetShort(); - - if ( target_type == TYPE_ID ) - { - id = target.id; - - if ( ! id ) - { - d->Add("(unknown id)"); - return; - } - - id_str = id->Name(); - - if ( id->ID_Val() && id->ID_Val()->IsMutableVal() && - id->Name()[0] != '#' ) - unique_str = fmt(" [id] (%s)", id->ID_Val()->AsMutableVal()->UniqueID()->Name()); - } - else - { - id = target.val->UniqueID(); - -#ifdef DEBUG - if ( target.val->GetID() ) - { - id_str = target.val->GetID()->Name(); - unique_str = fmt(" [val] (%s)", id->Name()); - } - else -#endif - id_str = id->Name(); - } - - const Val* op1 = op1_type == TYPE_VAL ? - this->op1.val : - id->ID_Val()->AsTableVal()->RecoverIndex(this->op1.key); - - switch ( opcode ) { - case OP_ASSIGN: - assert(op1); - d->Add(id_str); - d->Add(" = "); - op1->Describe(d); - if ( op2 ) - { - d->Add(" ("); - op2->Describe(d); - d->Add(")"); - } - d->Add(unique_str); - break; - - case OP_INCR: - assert(op1 && op2); - d->Add(id_str); - d->Add(" += "); - d->Add(op1->CoerceToInt() - op2->CoerceToInt()); - d->Add(unique_str); - break; - - case OP_ASSIGN_IDX: - assert(op1); - d->Add(id_str); - d->Add("["); - op1->Describe(d); - d->Add("]"); - d->Add(" = "); - if ( op2 ) - op2->Describe(d); - else - d->Add("(null)"); - if ( op3 ) - { - d->Add(" ("); - op3->Describe(d); - d->Add(")"); - } - d->Add(unique_str); - break; - - case OP_INCR_IDX: - assert(op1 && op2 && op3); - d->Add(id_str); - d->Add("["); - op1->Describe(d); - d->Add("]"); - d->Add(" += "); - d->Add(op2->CoerceToInt() - op3->CoerceToInt()); - d->Add(unique_str); - break; - - case OP_ADD: - assert(op1); - d->Add("add "); - d->Add(id_str); - d->Add("["); - op1->Describe(d); - d->Add("]"); - if ( op2 ) - { - d->Add(" ("); - op2->Describe(d); - d->Add(")"); - } - d->Add(unique_str); - break; - - case OP_DEL: - assert(op1); - d->Add("del "); - d->Add(id_str); - d->Add("["); - op1->Describe(d); - d->Add("]"); - if ( op2 ) - { - d->Add(" ("); - op2->Describe(d); - d->Add(")"); - } - d->Add(unique_str); - break; - - case OP_EXPIRE: - assert(op1); - d->Add("expire "); - d->Add(id_str); - d->Add("["); - op1->Describe(d); - d->Add("]"); - if ( op2 ) - { - d->Add(" ("); - op2->Describe(d); - d->Add(")"); - } - d->Add(unique_str); - break; - - case OP_PRINT: - assert(op1); - d->Add("print "); - d->Add(id_str); - op1->Describe(d); - d->Add(unique_str); - break; - - case OP_READ_IDX: - assert(op1); - d->Add("read "); - d->Add(id_str); - d->Add("["); - op1->Describe(d); - d->Add("]"); - break; - - default: - reporter->InternalError("unknown opcode for StateAccess"); - break; - } - - if ( op1_type != TYPE_VAL ) - Unref(const_cast(op1)); - } - -void StateAccess::Log(StateAccess* access) - { - if ( access->target_type == TYPE_ID ) - { - if ( access->target.id->FindAttr(ATTR_TRACKED) ) - notifiers.Modified(access->target.id); - } - else - { - if ( access->target.val->GetProperties() & MutableVal::TRACKED ) - notifiers.Modified(access->target.val); - } - -#ifdef DEBUG - ODesc desc; - access->Describe(&desc); - DBG_LOG(DBG_STATE, "operation: %s%s", - desc.Description(), replaying > 0 ? " (replay)" : ""); -#endif - - delete access; - - } - NotifierRegistry notifiers; NotifierRegistry::~NotifierRegistry() diff --git a/src/StateAccess.h b/src/StateAccess.h index 86f1dbe88c..ac57d6d8f9 100644 --- a/src/StateAccess.h +++ b/src/StateAccess.h @@ -27,69 +27,6 @@ enum Opcode { // Op1 Op2 Op3 (Vals) OP_READ_IDX, // idx }; -class StateAccess { -public: - StateAccess(Opcode opcode, const ID* target, const Val* op1, - const Val* op2 = 0, const Val* op3 = 0); - StateAccess(Opcode opcode, const MutableVal* target, const Val* op1, - const Val* op2 = 0, const Val* op3 = 0); - - // For tables, the idx operand may be given as an index HashKey. - // This is for efficiency. While we need to reconstruct the index - // if we are actually going to serialize the access, we can at - // least skip it if we don't. - StateAccess(Opcode opcode, const ID* target, const HashKey* op1, - const Val* op2 = 0, const Val* op3 = 0); - StateAccess(Opcode opcode, const MutableVal* target, const HashKey* op1, - const Val* op2 = 0, const Val* op3 = 0); - - StateAccess(const StateAccess& sa); - - virtual ~StateAccess(); - - // Replays this access in the our environment. - void Replay(); - - // Returns target ID which may be an internal one for unbound vals. - ID* Target() const; - - void Describe(ODesc* d) const; - - // Main entry point when StateAcesses are performed. - // For every state-changing operation, this has to be called. - static void Log(StateAccess* access); - - // If we're going to make additional non-replaying accesses during a - // Replay(), we have to call these. - static void SuspendReplay() { --replaying; } - static void ResumeReplay() { ++replaying; } - -private: - StateAccess() { target.id = 0; op1.val = op2 = op3 = 0; } - void RefThem(); - - Opcode opcode; - union { - ID* id; - MutableVal* val; - } target; - - union { - Val* val; - const HashKey* key; - } op1; - - Val* op2; - Val* op3; - - enum Type { TYPE_ID, TYPE_VAL, TYPE_MVAL, TYPE_KEY }; - Type target_type; - Type op1_type; - bool delete_op1_key; - - static int replaying; -}; - // We provide a notifier framework to inform interested parties of // modifications to selected global IDs/Vals. To get notified about a change, // derive a class from Notifier and register the interesting IDs/Vals with diff --git a/src/Val.cc b/src/Val.cc index f0c0c031ed..61308bc3b5 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -438,8 +438,6 @@ ID* MutableVal::Bind() const "%u", ++id_counter); name[MAX_NAME_SIZE-1] = '\0'; -// DBG_LOG(DBG_STATE, "new unique ID %s", name); - id = new ID(name, SCOPE_GLOBAL, true); id->SetType(const_cast(this)->Type()->Ref()); @@ -457,8 +455,6 @@ void MutableVal::TransferUniqueID(MutableVal* mv) if ( ! id ) Bind(); - DBG_LOG(DBG_STATE, "transfering ID (new %s, old/alias %s)", new_name, id->Name()); - // Keep old name as alias. aliases.push_back(id); @@ -1178,55 +1174,6 @@ int TableVal::Assign(Val* index, HashKey* k, Val* new_val, Opcode op) subnets->Insert(index, new_entry_val); } - if ( LoggingAccess() && op != OP_NONE ) - { - Val* rec_index = 0; - if ( ! index ) - index = rec_index = RecoverIndex(&k_copy); - - if ( new_val ) - { - // A table. - if ( new_val->IsMutableVal() ) - new_val->AsMutableVal()->AddProperties(GetProperties()); - - bool unref_old_val = false; - Val* old_val = old_entry_val ? - old_entry_val->Value() : 0; - if ( op == OP_INCR && ! old_val ) - // If it's an increment, somebody has already - // checked that the index is there. If it's - // not, that can only be due to using the - // default. - { - old_val = Default(index); - unref_old_val = true; - } - - assert(op != OP_INCR || old_val); - - StateAccess::Log( - new StateAccess( - op == OP_INCR ? - OP_INCR_IDX : OP_ASSIGN_IDX, - this, index, new_val, old_val)); - - if ( unref_old_val ) - Unref(old_val); - } - - else - { - // A set. - StateAccess::Log( - new StateAccess(OP_ADD, this, - index, 0, 0)); - } - - if ( rec_index ) - Unref(rec_index); - } - // Keep old expiration time if necessary. if ( old_entry_val && attrs && attrs->FindAttr(ATTR_EXPIRE_CREATE) ) new_entry_val->SetExpireAccess(old_entry_val->ExpireAccessTime()); @@ -1237,6 +1184,7 @@ int TableVal::Assign(Val* index, HashKey* k, Val* new_val, Opcode op) delete old_entry_val; } + Modified(); return 1; } @@ -1556,11 +1504,7 @@ Val* TableVal::Lookup(Val* index, bool use_default_val) if ( v ) { if ( attrs && attrs->FindAttr(ATTR_EXPIRE_READ) ) - { v->SetExpireAccess(network_time); - if ( LoggingAccess() && ExpirationEnabled() ) - ReadOperation(index, v); - } return v->Value() ? v->Value() : this; } @@ -1587,11 +1531,7 @@ Val* TableVal::Lookup(Val* index, bool use_default_val) if ( v ) { if ( attrs && attrs->FindAttr(ATTR_EXPIRE_READ) ) - { v->SetExpireAccess(network_time); - if ( LoggingAccess() && ExpirationEnabled() ) - ReadOperation(index, v); - } return v->Value() ? v->Value() : this; } @@ -1645,11 +1585,7 @@ TableVal* TableVal::LookupSubnetValues(const SubNetVal* search) if ( entry ) { if ( attrs && attrs->FindAttr(ATTR_EXPIRE_READ) ) - { entry->SetExpireAccess(network_time); - if ( LoggingAccess() && ExpirationEnabled() ) - ReadOperation(s, entry); - } } Unref(s); // assign does not consume index @@ -1679,8 +1615,6 @@ bool TableVal::UpdateTimestamp(Val* index) return false; v->SetExpireAccess(network_time); - if ( LoggingAccess() && attrs->FindAttr(ATTR_EXPIRE_READ) ) - ReadOperation(index, v); return true; } @@ -1699,25 +1633,10 @@ Val* TableVal::Delete(const Val* index) if ( subnets && ! subnets->Remove(index) ) reporter->InternalWarning("index not in prefix table"); - if ( LoggingAccess() ) - { - if ( v ) - { - // A set. - Val* has_old_val = val_mgr->GetInt(1); - StateAccess::Log( - new StateAccess(OP_DEL, this, index, - has_old_val)); - Unref(has_old_val); - } - else - StateAccess::Log( - new StateAccess(OP_DEL, this, index, 0)); - } - delete k; delete v; + Modified(); return va; } @@ -1736,9 +1655,7 @@ Val* TableVal::Delete(const HashKey* k) delete v; - if ( LoggingAccess() ) - StateAccess::Log(new StateAccess(OP_DEL, this, k)); - + Modified(); return va; } @@ -1949,6 +1866,7 @@ void TableVal::DoExpire(double t) HashKey* k = 0; TableEntryVal* v = 0; TableEntryVal* v_saved = 0; + bool modified = false; for ( int i = 0; i < table_incremental_step && (v = tbl->NextEntry(k, expire_cookie)); ++i ) @@ -2001,18 +1919,18 @@ void TableVal::DoExpire(double t) Unref(index); } - if ( LoggingAccess() ) - StateAccess::Log( - new StateAccess(OP_EXPIRE, this, k)); - tbl->RemoveEntry(k); Unref(v->Value()); delete v; + modified = true; } delete k; } + if ( modified ) + Modified(); + if ( ! v ) { expire_cookie = 0; @@ -2124,10 +2042,7 @@ void TableVal::ReadOperation(Val* index, TableEntryVal* v) // practical issues such as latency, we send one update every half // &read_expire. if ( network_time - v->LastReadUpdate() > timeout / 2 ) - { - StateAccess::Log(new StateAccess(OP_READ_IDX, this, index)); v->SetLastReadUpdate(network_time); - } } Val* TableVal::DoClone(CloneState* state) @@ -2307,21 +2222,8 @@ RecordVal::~RecordVal() void RecordVal::Assign(int field, Val* new_val, Opcode op) { Val* old_val = AsNonConstRecord()->replace(field, new_val); - - if ( LoggingAccess() && op != OP_NONE ) - { - if ( new_val && new_val->IsMutableVal() ) - new_val->AsMutableVal()->AddProperties(GetProperties()); - - StringVal* index = new StringVal(Type()->AsRecordType()->FieldName(field)); - StateAccess::Log( - new StateAccess( - op == OP_INCR ? OP_INCR_IDX : OP_ASSIGN_IDX, - this, index, new_val, old_val)); - Unref(index); // The logging may keep a cached copy. - } - Unref(old_val); + Modified(); } Val* RecordVal::Lookup(int field) const @@ -2627,19 +2529,6 @@ bool VectorVal::Assign(unsigned int index, Val* element, Opcode op) else val.vector_val->resize(index + 1); - if ( LoggingAccess() && op != OP_NONE ) - { - if ( element->IsMutableVal() ) - element->AsMutableVal()->AddProperties(GetProperties()); - - Val* ival = val_mgr->GetCount(index); - - StateAccess::Log(new StateAccess(op == OP_INCR ? - OP_INCR_IDX : OP_ASSIGN_IDX, - this, ival, element, val_at_index)); - Unref(ival); - } - Unref(val_at_index); // Note: we do *not* Ref() the element, if any, at this point. @@ -2647,6 +2536,7 @@ bool VectorVal::Assign(unsigned int index, Val* element, Opcode op) // to do it similarly. (*val.vector_val)[index] = element; + Modified(); return true; } diff --git a/src/Val.h b/src/Val.h index 59fef19ac0..2eddac8cb7 100644 --- a/src/Val.h +++ b/src/Val.h @@ -51,8 +51,6 @@ class StringVal; class EnumVal; class MutableVal; -class StateAccess; - class VectorVal; class TableEntryVal; @@ -532,17 +530,6 @@ public: virtual bool AddProperties(Properties state); virtual bool RemoveProperties(Properties state); - // Whether StateAccess:LogAccess needs to be called. - bool LoggingAccess() const - { -#ifndef DEBUG - return props & TRACKED; -#else - return debug_logger.IsVerbose() || - (props & TRACKED); -#endif - } - protected: explicit MutableVal(BroType* t) : Val(t) { props = 0; id = 0; } @@ -553,6 +540,7 @@ protected: friend class Val; void SetID(ID* arg_id) { Unref(id); id = arg_id; } + void Modified() { notifiers.Modified(this); } private: ID* Bind() const; @@ -957,7 +945,6 @@ public: protected: friend class Val; - friend class StateAccess; TableVal() {} void Init(TableType* t); diff --git a/src/bro.bif b/src/bro.bif index 3a082bcc5f..47b713e4e1 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1843,17 +1843,6 @@ function global_ids%(%): id_table TableVal* ids = new TableVal(id_table); PDict(ID)* globals = global_scope()->Vars(); IterCookie* c = globals->InitForIteration(); -#ifdef DEBUG - /** - * Explanation time: c needs to be a robust cookie when one is in debug mode, - * otherwise the Zeek process will crash in ~80% of cases when -B all is specified. - * The reason for this are the RecordVals that we create. RecordVal::Assign triggers - * a StateAccess::Log, which in turn (only in debug mode) triggers StateAccess::Describe, - * which creates a UniqueID for the variable, which triggers an insert into global_scope. - * Which invalidates the iteration cookie if it is not robust. - **/ - globals->MakeRobustCookie(c); -#endif ID* id; while ( (id = globals->NextEntry(c)) ) From 0ba382280c393a40672b88e00c7d9d594d31cd83 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 6 Jun 2019 03:24:13 +0000 Subject: [PATCH 05/22] Remove enum Opcode. --- src/Expr.cc | 34 +++++++++++++++++----------------- src/Expr.h | 12 ++++++------ src/ID.cc | 24 ++---------------------- src/ID.h | 2 +- src/Op.h | 23 ----------------------- src/StateAccess.h | 13 ------------- src/Val.cc | 24 +++++++++++------------- src/Val.h | 14 +++++++------- 8 files changed, 44 insertions(+), 102 deletions(-) delete mode 100644 src/Op.h diff --git a/src/Expr.cc b/src/Expr.cc index ef4d5af6e6..32cef7b0e1 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -97,7 +97,7 @@ void Expr::EvalIntoAggregate(const BroType* /* t */, Val* /* aggr */, Internal("Expr::EvalIntoAggregate called"); } -void Expr::Assign(Frame* /* f */, Val* /* v */, Opcode /* op */) +void Expr::Assign(Frame* /* f */, Val* /* v */) { Internal("Expr::Assign called"); } @@ -261,10 +261,10 @@ Expr* NameExpr::MakeLvalue() return new RefExpr(this); } -void NameExpr::Assign(Frame* f, Val* v, Opcode op) +void NameExpr::Assign(Frame* f, Val* v) { if ( id->IsGlobal() ) - id->SetVal(v, op); + id->SetVal(v); else f->SetElement(id->Offset(), v); } @@ -1007,18 +1007,18 @@ Val* IncrExpr::Eval(Frame* f) const if ( elt ) { Val* new_elt = DoSingleEval(f, elt); - v_vec->Assign(i, new_elt, OP_INCR); + v_vec->Assign(i, new_elt); } else - v_vec->Assign(i, 0, OP_INCR); + v_vec->Assign(i, 0); } - op->Assign(f, v_vec, OP_INCR); + op->Assign(f, v_vec); } else { Val* old_v = v; - op->Assign(f, v = DoSingleEval(f, old_v), OP_INCR); + op->Assign(f, v = DoSingleEval(f, old_v)); Unref(old_v); } @@ -2041,9 +2041,9 @@ Expr* RefExpr::MakeLvalue() return this; } -void RefExpr::Assign(Frame* f, Val* v, Opcode opcode) +void RefExpr::Assign(Frame* f, Val* v) { - op->Assign(f, v, opcode); + op->Assign(f, v); } AssignExpr::AssignExpr(Expr* arg_op1, Expr* arg_op2, int arg_is_init, @@ -2684,7 +2684,7 @@ Val* IndexExpr::Fold(Val* v1, Val* v2) const return 0; } -void IndexExpr::Assign(Frame* f, Val* v, Opcode op) +void IndexExpr::Assign(Frame* f, Val* v) { if ( IsError() ) return; @@ -2704,7 +2704,7 @@ void IndexExpr::Assign(Frame* f, Val* v, Opcode op) switch ( v1->Type()->Tag() ) { case TYPE_VECTOR: - if ( ! v1->AsVectorVal()->Assign(v2, v, op) ) + if ( ! v1->AsVectorVal()->Assign(v2, v) ) { if ( v ) { @@ -2723,7 +2723,7 @@ void IndexExpr::Assign(Frame* f, Val* v, Opcode op) break; case TYPE_TABLE: - if ( ! v1->AsTableVal()->Assign(v2, v, op) ) + if ( ! v1->AsTableVal()->Assign(v2, v) ) { if ( v ) { @@ -2826,7 +2826,7 @@ int FieldExpr::CanDel() const return td->FindAttr(ATTR_DEFAULT) || td->FindAttr(ATTR_OPTIONAL); } -void FieldExpr::Assign(Frame* f, Val* v, Opcode opcode) +void FieldExpr::Assign(Frame* f, Val* v) { if ( IsError() ) return; @@ -2835,14 +2835,14 @@ void FieldExpr::Assign(Frame* f, Val* v, Opcode opcode) if ( op_v ) { RecordVal* r = op_v->AsRecordVal(); - r->Assign(field, v, opcode); + r->Assign(field, v); Unref(r); } } void FieldExpr::Delete(Frame* f) { - Assign(f, 0, OP_ASSIGN_IDX); + Assign(f, 0); } Val* FieldExpr::Fold(Val* v) const @@ -4635,7 +4635,7 @@ Expr* ListExpr::MakeLvalue() return new RefExpr(this); } -void ListExpr::Assign(Frame* f, Val* v, Opcode op) +void ListExpr::Assign(Frame* f, Val* v) { ListVal* lv = v->AsListVal(); @@ -4643,7 +4643,7 @@ void ListExpr::Assign(Frame* f, Val* v, Opcode op) RuntimeError("mismatch in list lengths"); loop_over_list(exprs, i) - exprs[i]->Assign(f, (*lv->Vals())[i]->Ref(), op); + exprs[i]->Assign(f, (*lv->Vals())[i]->Ref()); Unref(lv); } diff --git a/src/Expr.h b/src/Expr.h index 6c9ac5e18f..f146162b04 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -84,7 +84,7 @@ public: const; // Assign to the given value, if appropriate. - virtual void Assign(Frame* f, Val* v, Opcode op = OP_ASSIGN); + virtual void Assign(Frame* f, Val* v); // Returns the type corresponding to this expression interpreted // as an initialization. The type should be Unref()'d when done @@ -225,7 +225,7 @@ public: ID* Id() const { return id; } Val* Eval(Frame* f) const override; - void Assign(Frame* f, Val* v, Opcode op = OP_ASSIGN) override; + void Assign(Frame* f, Val* v) override; Expr* MakeLvalue() override; int IsPure() const override; @@ -572,7 +572,7 @@ class RefExpr : public UnaryExpr { public: explicit RefExpr(Expr* op); - void Assign(Frame* f, Val* v, Opcode op = OP_ASSIGN) override; + void Assign(Frame* f, Val* v) override; Expr* MakeLvalue() override; protected: @@ -615,7 +615,7 @@ public: void Add(Frame* f) override; void Delete(Frame* f) override; - void Assign(Frame* f, Val* v, Opcode op = OP_ASSIGN) override; + void Assign(Frame* f, Val* v) override; Expr* MakeLvalue() override; // Need to override Eval since it can take a vector arg but does @@ -643,7 +643,7 @@ public: int CanDel() const override; - void Assign(Frame* f, Val* v, Opcode op = OP_ASSIGN) override; + void Assign(Frame* f, Val* v) override; void Delete(Frame* f) override; Expr* MakeLvalue() override; @@ -963,7 +963,7 @@ public: BroType* InitType() const override; Val* InitVal(const BroType* t, Val* aggr) const override; Expr* MakeLvalue() override; - void Assign(Frame* f, Val* v, Opcode op = OP_ASSIGN) override; + void Assign(Frame* f, Val* v) override; TraversalCode Traverse(TraversalCallback* cb) const override; diff --git a/src/ID.cc b/src/ID.cc index 915b5002b9..29a9a5a98a 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -59,34 +59,14 @@ void ID::ClearVal() val = 0; } -void ID::SetVal(Val* v, Opcode op, bool arg_weak_ref) +void ID::SetVal(Val* v, bool arg_weak_ref) { - if ( op != OP_NONE ) - { - MutableVal::Properties props = 0; - - if ( attrs && attrs->FindAttr(ATTR_TRACKED) ) - props |= MutableVal::TRACKED; - - if ( props ) - { - if ( v->IsMutableVal() ) - v->AsMutableVal()->AddProperties(props); - } - -#ifndef DEBUG - if ( props ) -#else - if ( debug_logger.IsVerbose() || props ) -#endif - notifiers.Modified(this); - } - if ( ! weak_ref ) Unref(val); val = v; weak_ref = arg_weak_ref; + notifiers.Modified(this); #ifdef DEBUG UpdateValID(); diff --git a/src/ID.h b/src/ID.h index bb9e11ca06..7717a584f3 100644 --- a/src/ID.h +++ b/src/ID.h @@ -46,7 +46,7 @@ public: // reference to the Val, the Val will be destroyed (naturally, // you have to take care that it will not be accessed via // the ID afterwards). - void SetVal(Val* v, Opcode op = OP_ASSIGN, bool weak_ref = false); + void SetVal(Val* v, bool weak_ref = false); void SetVal(Val* v, init_class c); void SetVal(Expr* ev, init_class c); diff --git a/src/Op.h b/src/Op.h deleted file mode 100644 index a628a6bb68..0000000000 --- a/src/Op.h +++ /dev/null @@ -1,23 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef op_h -#define op_h - -// BRO operations. - -typedef enum { - OP_INCR, OP_DECR, OP_NOT, OP_NEGATE, - OP_PLUS, OP_MINUS, OP_TIMES, OP_DIVIDE, OP_MOD, - OP_AND, OP_OR, - OP_LT, OP_LE, OP_EQ, OP_NE, OP_GE, OP_GT, - OP_MATCH, - OP_ASSIGN, - OP_INDEX, OP_FIELD, - OP_IN, - OP_LIST, - OP_CALL, - OP_SCHED, - OP_NAME, OP_CONST, OP_THIS -} BroOP; - -#endif diff --git a/src/StateAccess.h b/src/StateAccess.h index ac57d6d8f9..528e3357d5 100644 --- a/src/StateAccess.h +++ b/src/StateAccess.h @@ -14,19 +14,6 @@ class HashKey; class ODesc; class TableVal; -enum Opcode { // Op1 Op2 Op3 (Vals) - OP_NONE, - OP_ASSIGN, // new old - OP_ASSIGN_IDX, // idx new old - OP_ADD, // idx old - OP_INCR, // idx new old - OP_INCR_IDX, // idx new old - OP_DEL, // idx old - OP_PRINT, // args - OP_EXPIRE, // idx - OP_READ_IDX, // idx -}; - // We provide a notifier framework to inform interested parties of // modifications to selected global IDs/Vals. To get notified about a change, // derive a class from Notifier and register the interesting IDs/Vals with diff --git a/src/Val.cc b/src/Val.cc index 61308bc3b5..75f6f9134d 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -443,7 +443,7 @@ ID* MutableVal::Bind() const global_scope()->Insert(name, id); - id->SetVal(const_cast(this), OP_NONE, true); + id->SetVal(const_cast(this), true); return id; } @@ -461,7 +461,7 @@ void MutableVal::TransferUniqueID(MutableVal* mv) id = new ID(new_name, SCOPE_GLOBAL, true); id->SetType(const_cast(this)->Type()->Ref()); global_scope()->Insert(new_name, id); - id->SetVal(const_cast(this), OP_NONE, true); + id->SetVal(const_cast(this), true); Unref(mv->id); mv->id = 0; @@ -1132,7 +1132,7 @@ void TableVal::CheckExpireAttr(attr_tag at) } } -int TableVal::Assign(Val* index, Val* new_val, Opcode op) +int TableVal::Assign(Val* index, Val* new_val) { HashKey* k = ComputeHash(index); if ( ! k ) @@ -1142,10 +1142,10 @@ int TableVal::Assign(Val* index, Val* new_val, Opcode op) return 0; } - return Assign(index, k, new_val, op); + return Assign(index, k, new_val); } -int TableVal::Assign(Val* index, HashKey* k, Val* new_val, Opcode op) +int TableVal::Assign(Val* index, HashKey* k, Val* new_val) { int is_set = table_type->IsSet(); @@ -1227,15 +1227,13 @@ int TableVal::AddTo(Val* val, int is_first_init, bool propagate_ops) const if ( type->IsSet() ) { - if ( ! t->Assign(v->Value(), k, 0, - propagate_ops ? OP_ASSIGN : OP_NONE) ) + if ( ! t->Assign(v->Value(), k, 0) ) return 0; } else { v->Ref(); - if ( ! t->Assign(0, k, v->Value(), - propagate_ops ? OP_ASSIGN : OP_NONE) ) + if ( ! t->Assign(0, k, v->Value()) ) return 0; } } @@ -1822,7 +1820,7 @@ int TableVal::ExpandCompoundAndInit(val_list* vl, int k, Val* new_val) return 1; } -int TableVal::CheckAndAssign(Val* index, Val* new_val, Opcode op) +int TableVal::CheckAndAssign(Val* index, Val* new_val) { Val* v = 0; if ( subnets ) @@ -1834,7 +1832,7 @@ int TableVal::CheckAndAssign(Val* index, Val* new_val, Opcode op) if ( v ) index->Warn("multiple initializations for index"); - return Assign(index, new_val, op); + return Assign(index, new_val); } void TableVal::InitTimer(double delay) @@ -2219,7 +2217,7 @@ RecordVal::~RecordVal() delete_vals(AsNonConstRecord()); } -void RecordVal::Assign(int field, Val* new_val, Opcode op) +void RecordVal::Assign(int field, Val* new_val) { Val* old_val = AsNonConstRecord()->replace(field, new_val); Unref(old_val); @@ -2513,7 +2511,7 @@ VectorVal::~VectorVal() delete val.vector_val; } -bool VectorVal::Assign(unsigned int index, Val* element, Opcode op) +bool VectorVal::Assign(unsigned int index, Val* element) { if ( element && ! same_type(element->Type(), vector_type->YieldType(), 0) ) diff --git a/src/Val.h b/src/Val.h index 2eddac8cb7..08038a686b 100644 --- a/src/Val.h +++ b/src/Val.h @@ -840,8 +840,8 @@ public: // version takes a HashKey and Unref()'s it when done. If we're a // set, new_val has to be nil. If we aren't a set, index may be nil // in the second version. - int Assign(Val* index, Val* new_val, Opcode op = OP_ASSIGN); - int Assign(Val* index, HashKey* k, Val* new_val, Opcode op = OP_ASSIGN); + int Assign(Val* index, Val* new_val); + int Assign(Val* index, HashKey* k, Val* new_val); Val* SizeVal() const override { return val_mgr->GetCount(Size()); } @@ -951,7 +951,7 @@ protected: void CheckExpireAttr(attr_tag at); int ExpandCompoundAndInit(val_list* vl, int k, Val* new_val); - int CheckAndAssign(Val* index, Val* new_val, Opcode op = OP_ASSIGN); + int CheckAndAssign(Val* index, Val* new_val); bool AddProperties(Properties arg_state) override; bool RemoveProperties(Properties arg_state) override; @@ -995,7 +995,7 @@ public: Val* SizeVal() const override { return val_mgr->GetCount(record_type->NumFields()); } - void Assign(int field, Val* new_val, Opcode op = OP_ASSIGN); + void Assign(int field, Val* new_val); Val* Lookup(int field) const; // Does not Ref() value. Val* LookupWithDefault(int field) const; // Does Ref() value. @@ -1095,11 +1095,11 @@ public: // Note: does NOT Ref() the element! Remember to do so unless // the element was just created and thus has refcount 1. // - bool Assign(unsigned int index, Val* element, Opcode op = OP_ASSIGN); - bool Assign(Val* index, Val* element, Opcode op = OP_ASSIGN) + bool Assign(unsigned int index, Val* element); + bool Assign(Val* index, Val* element) { return Assign(index->AsListVal()->Index(0)->CoerceToUnsigned(), - element, op); + element); } // Assigns the value to how_many locations starting at index. From f8262b65c42aacdf4234b61e80d27726768956b4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 6 Jun 2019 03:28:12 +0000 Subject: [PATCH 06/22] Remove most of MutableVal (but not the class itelf yet) --- src/ID.cc | 20 ----- src/Val.cc | 239 ----------------------------------------------------- src/Val.h | 91 ++------------------ 3 files changed, 7 insertions(+), 343 deletions(-) diff --git a/src/ID.cc b/src/ID.cc index 29a9a5a98a..582c8fcf8e 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -155,16 +155,6 @@ void ID::UpdateValAttrs() if ( ! attrs ) return; - MutableVal::Properties props = 0; - - if ( val && val->IsMutableVal() ) - { - if ( attrs->FindAttr(ATTR_TRACKED) ) - props |= MutableVal::TRACKED; - - val->AsMutableVal()->AddProperties(props); - } - if ( val && val->Type()->Tag() == TYPE_TABLE ) val->AsTableVal()->SetAttrs(attrs); @@ -222,16 +212,6 @@ void ID::RemoveAttr(attr_tag a) { if ( attrs ) attrs->RemoveAttr(a); - - if ( val && val->IsMutableVal() ) - { - MutableVal::Properties props = 0; - - if ( a == ATTR_TRACKED ) - props |= MutableVal::TRACKED; - - val->AsMutableVal()->RemoveProperties(props); - } } void ID::SetOption() diff --git a/src/Val.cc b/src/Val.cc index 75f6f9134d..4ba2ea6dd7 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -351,120 +351,6 @@ void Val::ValDescribeReST(ODesc* d) const MutableVal::~MutableVal() { - for ( list::iterator i = aliases.begin(); i != aliases.end(); ++i ) - { - if ( global_scope() ) - global_scope()->Remove((*i)->Name()); - (*i)->ClearVal(); // just to make sure. - Unref((*i)); - } - - if ( id ) - { - if ( global_scope() ) - global_scope()->Remove(id->Name()); - id->ClearVal(); // just to make sure. - Unref(id); - } - } - -bool MutableVal::AddProperties(Properties arg_props) - { - if ( (props | arg_props) == props ) - // No change. - return false; - - props |= arg_props; - - if ( ! id ) - Bind(); - - return true; - } - - -bool MutableVal::RemoveProperties(Properties arg_props) - { - if ( (props & ~arg_props) == props ) - // No change. - return false; - - props &= ~arg_props; - - return true; - } - -ID* MutableVal::Bind() const - { - static bool initialized = false; - - assert(!id); - - static unsigned int id_counter = 0; - static const int MAX_NAME_SIZE = 128; - static char name[MAX_NAME_SIZE]; - static char* end_of_static_str = 0; - - if ( ! initialized ) - { - // Get local IP. - char host[MAXHOSTNAMELEN]; - strcpy(host, "localhost"); - gethostname(host, MAXHOSTNAMELEN); - host[MAXHOSTNAMELEN-1] = '\0'; -#if 0 - // We ignore errors. - struct hostent* ent = gethostbyname(host); - - uint32 ip; - if ( ent && ent->h_addr_list[0] ) - ip = *(uint32*) ent->h_addr_list[0]; - else - ip = htonl(0x7f000001); // 127.0.0.1 - - safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#", - IPAddr(IPv4, &ip, IPAddr::Network)->AsString().c_str(), - getpid()); -#else - safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#", host, getpid()); -#endif - - end_of_static_str = name + strlen(name); - - initialized = true; - } - - safe_snprintf(end_of_static_str, MAX_NAME_SIZE - (end_of_static_str - name), - "%u", ++id_counter); - name[MAX_NAME_SIZE-1] = '\0'; - - id = new ID(name, SCOPE_GLOBAL, true); - id->SetType(const_cast(this)->Type()->Ref()); - - global_scope()->Insert(name, id); - - id->SetVal(const_cast(this), true); - - return id; - } - -void MutableVal::TransferUniqueID(MutableVal* mv) - { - const char* new_name = mv->UniqueID()->Name(); - - if ( ! id ) - Bind(); - - // Keep old name as alias. - aliases.push_back(id); - - id = new ID(new_name, SCOPE_GLOBAL, true); - id->SetType(const_cast(this)->Type()->Ref()); - global_scope()->Insert(new_name, id); - id->SetVal(const_cast(this), true); - - Unref(mv->id); - mv->id = 0; } IntervalVal::IntervalVal(double quantity, double units) : @@ -2026,23 +1912,6 @@ double TableVal::CallExpireFunc(Val* idx) return secs; } -void TableVal::ReadOperation(Val* index, TableEntryVal* v) - { - double timeout = GetExpireTime(); - - if ( timeout < 0 ) - // Skip in case of unset/invalid expiration value. If it's an - // error, it has been reported already. - return; - - // In theory we need to only propagate one update per &read_expire - // interval to prevent peers from expiring intervals. To account for - // practical issues such as latency, we send one update every half - // &read_expire. - if ( network_time - v->LastReadUpdate() > timeout / 2 ) - v->SetLastReadUpdate(network_time); - } - Val* TableVal::DoClone(CloneState* state) { auto tv = new TableVal(table_type); @@ -2092,48 +1961,6 @@ Val* TableVal::DoClone(CloneState* state) return tv; } -bool TableVal::AddProperties(Properties arg_props) - { - if ( ! MutableVal::AddProperties(arg_props) ) - return false; - - if ( Type()->IsSet() || ! RecursiveProps(arg_props) ) - return true; - - // For a large table, this could get expensive. So, let's hope - // that nobody creates such a table *before* making it persistent - // (for example by inserting it into another table). - TableEntryVal* v; - PDict(TableEntryVal)* tbl = val.table_val; - IterCookie* c = tbl->InitForIteration(); - while ( (v = tbl->NextEntry(c)) ) - if ( v->Value()->IsMutableVal() ) - v->Value()->AsMutableVal()->AddProperties(RecursiveProps(arg_props)); - - return true; - } - -bool TableVal::RemoveProperties(Properties arg_props) - { - if ( ! MutableVal::RemoveProperties(arg_props) ) - return false; - - if ( Type()->IsSet() || ! RecursiveProps(arg_props) ) - return true; - - // For a large table, this could get expensive. So, let's hope - // that nobody creates such a table *before* making it persistent - // (for example by inserting it into another table). - TableEntryVal* v; - PDict(TableEntryVal)* tbl = val.table_val; - IterCookie* c = tbl->InitForIteration(); - while ( (v = tbl->NextEntry(c)) ) - if ( v->Value()->IsMutableVal() ) - v->Value()->AsMutableVal()->RemoveProperties(RecursiveProps(arg_props)); - - return true; - } - unsigned int TableVal::MemoryAllocation() const { unsigned int size = 0; @@ -2428,41 +2255,6 @@ Val* RecordVal::DoClone(CloneState* state) return rv; } -bool RecordVal::AddProperties(Properties arg_props) - { - if ( ! MutableVal::AddProperties(arg_props) ) - return false; - - if ( ! RecursiveProps(arg_props) ) - return true; - - loop_over_list(*val.val_list_val, i) - { - Val* v = (*val.val_list_val)[i]; - if ( v && v->IsMutableVal() ) - v->AsMutableVal()->AddProperties(RecursiveProps(arg_props)); - } - return true; - } - - -bool RecordVal::RemoveProperties(Properties arg_props) - { - if ( ! MutableVal::RemoveProperties(arg_props) ) - return false; - - if ( ! RecursiveProps(arg_props) ) - return true; - - loop_over_list(*val.val_list_val, i) - { - Val* v = (*val.val_list_val)[i]; - if ( v && v->IsMutableVal() ) - v->AsMutableVal()->RemoveProperties(RecursiveProps(arg_props)); - } - return true; - } - unsigned int RecordVal::MemoryAllocation() const { unsigned int size = 0; @@ -2599,37 +2391,6 @@ unsigned int VectorVal::ResizeAtLeast(unsigned int new_num_elements) return Resize(new_num_elements); } -bool VectorVal::AddProperties(Properties arg_props) - { - if ( ! MutableVal::AddProperties(arg_props) ) - return false; - - if ( ! RecursiveProps(arg_props) ) - return true; - - for ( unsigned int i = 0; i < val.vector_val->size(); ++i ) - if ( (*val.vector_val)[i]->IsMutableVal() ) - (*val.vector_val)[i]->AsMutableVal()->AddProperties(RecursiveProps(arg_props)); - - return true; - } - -bool VectorVal::RemoveProperties(Properties arg_props) - { - if ( ! MutableVal::RemoveProperties(arg_props) ) - return false; - - if ( ! RecursiveProps(arg_props) ) - return true; - - for ( unsigned int i = 0; i < val.vector_val->size(); ++i ) - if ( (*val.vector_val)[i]->IsMutableVal() ) - (*val.vector_val)[i]->AsMutableVal()->RemoveProperties(RecursiveProps(arg_props)); - - return true; - } - - Val* VectorVal::DoClone(CloneState* state) { auto vv = new VectorVal(vector_type); diff --git a/src/Val.h b/src/Val.h index 08038a686b..48f456fa17 100644 --- a/src/Val.h +++ b/src/Val.h @@ -325,20 +325,6 @@ public: return IsMutable(type->Tag()); } - const MutableVal* AsMutableVal() const - { - if ( ! IsMutableVal() ) - BadTag("Val::AsMutableVal", type_name(type->Tag())); - return (MutableVal*) this; - } - - MutableVal* AsMutableVal() - { - if ( ! IsMutableVal() ) - BadTag("Val::AsMutableVal", type_name(type->Tag())); - return (MutableVal*) this; - } - void Describe(ODesc* d) const override; virtual void DescribeReST(ODesc* d) const; @@ -499,56 +485,13 @@ private: extern ValManager* val_mgr; class MutableVal : public Val { -public: - // Each MutableVal gets a globally unique ID that can be used to - // reference it no matter if it's directly bound to any user-visible - // ID. This ID is inserted into the global namespace. - ID* UniqueID() const { return id ? id : Bind(); } - - // Returns true if we've already generated a unique ID. - bool HasUniqueID() const { return id; } - - // Transfers the unique ID of the given value to this value. We keep our - // old ID as an alias. - void TransferUniqueID(MutableVal* mv); - - // MutableVals can have properties (let's refrain from calling them - // attributes!). Most properties are recursive. If a derived object - // can contain MutableVals itself, the object has to override - // {Add,Remove}Properties(). RecursiveProp(state) masks out all non- - // recursive properties. If this is non-null, an overriden method must - // call itself with RecursiveProp(state) as argument for all contained - // values. (In any case, don't forget to call the parent's method.) - typedef char Properties; - - // Tracked by NotifierRegistry, not recursive. - static const int TRACKED = 0x04; - - int RecursiveProps(int prop) const { return prop & ~TRACKED; } - - Properties GetProperties() const { return props; } - virtual bool AddProperties(Properties state); - virtual bool RemoveProperties(Properties state); - protected: - explicit MutableVal(BroType* t) : Val(t) - { props = 0; id = 0; } - MutableVal() { props = 0; id = 0; } + explicit MutableVal(BroType* t) : Val(t) {} + + MutableVal() {} ~MutableVal() override; - friend class ID; - friend class Val; - - void SetID(ID* arg_id) { Unref(id); id = arg_id; } void Modified() { notifiers.Modified(this); } - -private: - ID* Bind() const; - - mutable ID* id; - list aliases; - Properties props; - uint64 last_modified; }; #define Microseconds 1e-6 @@ -771,7 +714,7 @@ public: { val = v; last_access_time = network_time; - expire_access_time = last_read_update = + expire_access_time = int(network_time - bro_start_network_time); } @@ -780,7 +723,6 @@ public: auto rval = new TableEntryVal(val ? val->Clone() : nullptr); rval->last_access_time = last_access_time; rval->expire_access_time = expire_access_time; - rval->last_read_update = last_read_update; return rval; } @@ -796,24 +738,16 @@ public: void SetExpireAccess(double time) { expire_access_time = int(time - bro_start_network_time); } - // Returns/sets time of when we propagated the last OP_READ_IDX - // for this item. - double LastReadUpdate() const - { return bro_start_network_time + last_read_update; } - void SetLastReadUpdate(double time) - { last_read_update = int(time - bro_start_network_time); } - protected: friend class TableVal; Val* val; double last_access_time; - // The next two entries store seconds since Bro's start. We use - // ints here to save a few bytes, as we do not need a high resolution - // for these anyway. + // The next entry stores seconds since Bro's start. We use ints here + // to save a few bytes, as we do not need a high resolution for these + // anyway. int expire_access_time; - int last_read_update; }; class TableValTimer : public Timer { @@ -953,9 +887,6 @@ protected: int ExpandCompoundAndInit(val_list* vl, int k, Val* new_val); int CheckAndAssign(Val* index, Val* new_val); - bool AddProperties(Properties arg_state) override; - bool RemoveProperties(Properties arg_state) override; - // Calculates default value for index. Returns 0 if none. Val* Default(Val* index); @@ -971,9 +902,6 @@ protected: // takes ownership of the reference. double CallExpireFunc(Val *idx); - // Propagates a read operation if necessary. - void ReadOperation(Val* index, TableEntryVal *v); - Val* DoClone(CloneState* state) override; TableType* table_type; @@ -1043,9 +971,6 @@ protected: friend class Val; RecordVal() {} - bool AddProperties(Properties arg_state) override; - bool RemoveProperties(Properties arg_state) override; - Val* DoClone(CloneState* state) override; RecordType* record_type; @@ -1133,8 +1058,6 @@ protected: friend class Val; VectorVal() { } - bool AddProperties(Properties arg_state) override; - bool RemoveProperties(Properties arg_state) override; void ValDescribe(ODesc* d) const override; Val* DoClone(CloneState* state) override; From 062a1ee6b37b7686b7eea7c455901797316a937d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 6 Jun 2019 23:13:43 +0000 Subject: [PATCH 07/22] Redo API for notifiers. There's now an notifier::Modifiable interface class that class supposed to signal modifications are to be derived from. This takes the place of the former MutableValue class and also unifies how Val and IDs signal modifications. --- src/ID.cc | 2 +- src/ID.h | 2 +- src/StateAccess.cc | 99 +++++++++-------------------------- src/StateAccess.h | 125 ++++++++++++++++++++++++++------------------- src/Trigger.cc | 30 ++++------- src/Trigger.h | 9 ++-- src/Val.cc | 6 +-- src/Val.h | 16 ++++-- 8 files changed, 126 insertions(+), 163 deletions(-) diff --git a/src/ID.cc b/src/ID.cc index 582c8fcf8e..894de949a5 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -66,7 +66,7 @@ void ID::SetVal(Val* v, bool arg_weak_ref) val = v; weak_ref = arg_weak_ref; - notifiers.Modified(this); + Modified(); #ifdef DEBUG UpdateValID(); diff --git a/src/ID.h b/src/ID.h index 7717a584f3..a1cbe494d3 100644 --- a/src/ID.h +++ b/src/ID.h @@ -15,7 +15,7 @@ class Func; typedef enum { INIT_NONE, INIT_FULL, INIT_EXTRA, INIT_REMOVE, } init_class; typedef enum { SCOPE_FUNCTION, SCOPE_MODULE, SCOPE_GLOBAL } IDScope; -class ID : public BroObj { +class ID : public BroObj, public notifier::Modifiable { public: ID(const char* name, IDScope arg_scope, bool arg_is_export); ~ID() override; diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 853a8a7879..7d7c397981 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -4,110 +4,57 @@ #include "NetVar.h" #include "DebugLogger.h" -NotifierRegistry notifiers; +notifier::Registry notifier::registry; -NotifierRegistry::~NotifierRegistry() +notifier::Registry::~Registry() { - for ( auto i : ids ) - Unref(i.first); - - for ( auto i : vals ) - Unref(i.first); + for ( auto i : registrations ) + Unregister(i.first); } -void NotifierRegistry::Register(ID* id, NotifierRegistry::Notifier* notifier) +void notifier::Registry::Register(Modifiable* m, notifier::Notifier* notifier) { - DBG_LOG(DBG_NOTIFIERS, "registering ID %s for notifier %s", - id->Name(), notifier->Name()); + DBG_LOG(DBG_NOTIFIERS, "registering modifiable %p for notifier %s", + m, notifier->Name()); - Attr* attr = new Attr(ATTR_TRACKED); - - if ( id->Attrs() ) - { - if ( ! id->Attrs()->FindAttr(ATTR_TRACKED) ) - id->Attrs()->AddAttr(attr); - } - else - { - attr_list* a = new attr_list{attr}; - id->SetAttrs(new Attributes(a, id->Type(), false)); - } - - Unref(attr); - - ids.insert({id, notifier}); - Ref(id); + registrations.insert({m, notifier}); + ++m->notifiers; } -void NotifierRegistry::Register(Val* val, NotifierRegistry::Notifier* notifier) +void notifier::Registry::Unregister(Modifiable* m, notifier::Notifier* notifier) { - if ( ! val->IsMutableVal() ) - return; + DBG_LOG(DBG_NOTIFIERS, "unregistering modifiable %p from notifier %s", + m, notifier->Name()); - DBG_LOG(DBG_NOTIFIERS, "registering value %p for notifier %s", - val, notifier->Name()); - - vals.insert({val, notifier}); - Ref(val); - } - -void NotifierRegistry::Unregister(ID* id, NotifierRegistry::Notifier* notifier) - { - DBG_LOG(DBG_NOTIFIERS, "unregistering ID %s for notifier %s", - id->Name(), notifier->Name()); - - auto x = ids.equal_range(id); + auto x = registrations.equal_range(m); for ( auto i = x.first; i != x.second; i++ ) { if ( i->second == notifier ) { - ids.erase(i); - Unref(id); - break; - } - } - - if ( ids.find(id) == ids.end() ) - // Last registered notifier for this ID - id->Attrs()->RemoveAttr(ATTR_TRACKED); - } - -void NotifierRegistry::Unregister(Val* val, NotifierRegistry::Notifier* notifier) - { - DBG_LOG(DBG_NOTIFIERS, "unregistering Val %p for notifier %s", - val, notifier->Name()); - - auto x = vals.equal_range(val); - for ( auto i = x.first; i != x.second; i++ ) - { - if ( i->second == notifier ) - { - vals.erase(i); - Unref(val); + registrations.erase(i); + --i->first->notifiers; break; } } } -void NotifierRegistry::Modified(Val *val) +void notifier::Registry::Unregister(Modifiable* m) { - DBG_LOG(DBG_NOTIFIERS, "modification to tracked value %p", val); - - auto x = vals.equal_range(val); + auto x = registrations.equal_range(m); for ( auto i = x.first; i != x.second; i++ ) - i->second->Modified(val); + Unregister(m, i->second); } -void NotifierRegistry::Modified(ID *id) +void notifier::Registry::Modified(Modifiable* m) { - DBG_LOG(DBG_NOTIFIERS, "modification to tracked ID %s", id->Name()); + DBG_LOG(DBG_NOTIFIERS, "modification to modifiable %p", m); - auto x = ids.equal_range(id); + auto x = registrations.equal_range(m); for ( auto i = x.first; i != x.second; i++ ) - i->second->Modified(id); + i->second->Modified(m); } -const char* NotifierRegistry::Notifier::Name() const +const char* notifier::Notifier::Name() const { return fmt("%p", this); } diff --git a/src/StateAccess.h b/src/StateAccess.h index 528e3357d5..91a7fa6186 100644 --- a/src/StateAccess.h +++ b/src/StateAccess.h @@ -1,4 +1,14 @@ // A class describing a state-modyfing access to a Value or an ID. +// +// TODO UPDATE: We provide a notifier framework to inform interested parties +// of modifications to selected global IDs/Vals. To get notified about a +// change, derive a class from Notifier and register the interesting +// instances with the NotifierRegistry. +// +// Note: For containers (e.g., tables), notifications are only issued if the +// container itself is modified, *not* for changes to the values contained +// therein. + #ifndef STATEACESSS_H #define STATEACESSS_H @@ -7,63 +17,74 @@ #include #include -class Val; -class ID; -class MutableVal; -class HashKey; -class ODesc; -class TableVal; +#include "util.h" -// We provide a notifier framework to inform interested parties of -// modifications to selected global IDs/Vals. To get notified about a change, -// derive a class from Notifier and register the interesting IDs/Vals with -// the NotifierRegistry. -// -// Note: For containers (e.g., tables), notifications are only issued if the -// container itself is modified, *not* for changes to the values contained -// therein. +namespace notifier { -class NotifierRegistry { +class Modifiable; + +class Notifier { public: - class Notifier { - public: - virtual ~Notifier() { } + virtual ~Notifier() { } - // Called when a change is being performed. Note that when - // these methods are called, it is undefined whether the - // change has already been done or is just going to be - // performed soon. - virtual void Modified(ID* id) = 0; - virtual void Modified(Val* val) = 0; - virtual const char* Name() const; // for debugging - }; - - NotifierRegistry() { } - ~NotifierRegistry(); - - // Register a new notifier to be informed when ID/Val changes. Note - // that the registry will store a reference to the target, keeping - // the instance alive for as long as it's registered. - void Register(ID* id, Notifier* notifier); - void Register(Val* val, Notifier* notifier); - - // Cancel a notifier's tracking for this ID/Val, also releasing the - // referencee being held. - void Unregister(ID* id, Notifier* notifier); - void Unregister(Val* val, Notifier* notifier); - - // Inform all registered notifiiers of a modification to a value/ID. - void Modified(ID *id); - void Modified(Val *val); - -private: - typedef std::unordered_multimap ValMap; - typedef std::unordered_multimap IDMap; - - ValMap vals; - IDMap ids; + // Called afger a change has been performed. + virtual void Modified(Modifiable* m) = 0; + virtual const char* Name() const; // for debugging }; -extern NotifierRegistry notifiers; +// Singleton class. +class Registry { +public: + Registry() { } + ~Registry(); + + // Register a new notifier to be informed when an instance changes. + void Register(Modifiable* m, Notifier* notifier); + + // Cancel a notifier's tracking an instace. + void Unregister(Modifiable* m, Notifier* notifier); + + // Cancel all notifiers registered for an instance. + void Unregister(Modifiable* m); + +private: + friend class Modifiable; + + // Inform all registered notifiers of a modification to an instance. + void Modified(Modifiable* m); + + typedef std::unordered_multimap ModifiableMap; + ModifiableMap registrations; +}; + + +class Registry; +extern Registry registry; + +// Base class for objects wanting to signal modifications to the registry. +class Modifiable { +protected: + friend class Registry; + + Modifiable() {} + ~Modifiable() + { + if ( notifiers ) + registry.Unregister(this); + + }; + + void Modified() + { + if ( notifiers ) + registry.Modified(this); + } + + // Number of currently registered notifiers for this instance. + uint64 notifiers; +}; + +} + #endif diff --git a/src/Trigger.cc b/src/Trigger.cc index 213707b6b8..c1295b8218 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -33,7 +33,7 @@ TraversalCode TriggerTraversalCallback::PreExpr(const Expr* expr) trigger->Register(e->Id()); Val* v = e->Id()->ID_Val(); - if ( v && v->IsMutableVal() ) + if ( v && v->Modifiable() ) trigger->Register(v); break; }; @@ -382,38 +382,30 @@ void Trigger::Timeout() void Trigger::Register(ID* id) { assert(! disabled); - notifiers.Register(id, this); + notifier::registry.Register(id, this); Ref(id); - ids.insert(id); + objs.push_back(id); } void Trigger::Register(Val* val) { + if ( ! val->Modifiable() ) + return; + assert(! disabled); - notifiers.Register(val, this); + notifier::registry.Register(val->Modifiable(), this); Ref(val); - vals.insert(val); + objs.push_back(val); } void Trigger::UnregisterAll() { - loop_over_list(ids, i) - { - notifiers.Unregister(ids[i], this); - Unref(ids[i]); - } + for ( auto o : objs ) + Unref(o); // this will unregister with the registry as well - ids.clear(); - - loop_over_list(vals, j) - { - notifiers.Unregister(vals[j], this); - Unref(vals[j]); - } - - vals.clear(); + objs.clear(); } void Trigger::Attach(Trigger *trigger) diff --git a/src/Trigger.h b/src/Trigger.h index 8dc478e049..fe7d425a72 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -13,7 +13,7 @@ class TriggerTimer; class TriggerTraversalCallback; -class Trigger : public NotifierRegistry::Notifier, public BroObj { +class Trigger : public notifier::Notifier, public BroObj { public: // Don't access Trigger objects; they take care of themselves after // instantiation. Note that if the condition is already true, the @@ -61,9 +61,7 @@ public: { d->Add(""); } // Overidden from Notifier. We queue the trigger and evaluate it // later to avoid race conditions. - void Modified(ID* id) override - { QueueTrigger(this); } - void Modified(Val* val) override + void Modified(notifier::Modifiable* m) override { QueueTrigger(this); } const char* Name() const override; @@ -104,8 +102,7 @@ private: bool delayed; // true if a function call is currently being delayed bool disabled; - val_list vals; - id_list ids; + std::vector objs; typedef map ValCache; ValCache cache; diff --git a/src/Val.cc b/src/Val.cc index 4ba2ea6dd7..bea9ce4038 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -899,7 +899,7 @@ static void table_entry_val_delete_func(void* val) delete tv; } -TableVal::TableVal(TableType* t, Attributes* a) : MutableVal(t) +TableVal::TableVal(TableType* t, Attributes* a) : Val(t) { Init(t); SetAttrs(a); @@ -1982,7 +1982,7 @@ unsigned int TableVal::MemoryAllocation() const vector RecordVal::parse_time_records; -RecordVal::RecordVal(RecordType* t, bool init_fields) : MutableVal(t) +RecordVal::RecordVal(RecordType* t, bool init_fields) : Val(t) { origin = 0; record_type = t; @@ -2287,7 +2287,7 @@ Val* EnumVal::DoClone(CloneState* state) return Ref(); } -VectorVal::VectorVal(VectorType* t) : MutableVal(t) +VectorVal::VectorVal(VectorType* t) : Val(t) { vector_type = t->Ref()->AsVectorType(); val.vector_val = new vector(); diff --git a/src/Val.h b/src/Val.h index 48f456fa17..d98c352f6d 100644 --- a/src/Val.h +++ b/src/Val.h @@ -328,6 +328,8 @@ public: void Describe(ODesc* d) const override; virtual void DescribeReST(ODesc* d) const; + virtual notifier::Modifiable* Modifiable() { return 0; } + #ifdef DEBUG // For debugging, we keep a reference to the global ID to which a // value has been bound *last*. @@ -490,8 +492,6 @@ protected: MutableVal() {} ~MutableVal() override; - - void Modified() { notifiers.Modified(this); } }; #define Microseconds 1e-6 @@ -764,7 +764,7 @@ protected: }; class CompositeHash; -class TableVal : public MutableVal { +class TableVal : public Val, public notifier::Modifiable { public: explicit TableVal(TableType* t, Attributes* attrs = 0); ~TableVal() override; @@ -877,6 +877,8 @@ public: HashKey* ComputeHash(const Val* index) const { return table_hash->ComputeHash(index, 1); } + notifier::Modifiable* Modifiable() override { return this; } + protected: friend class Val; TableVal() {} @@ -915,7 +917,7 @@ protected: Val* def_val; }; -class RecordVal : public MutableVal { +class RecordVal : public Val, public notifier::Modifiable { public: explicit RecordVal(RecordType* t, bool init_fields = true); ~RecordVal() override; @@ -962,6 +964,8 @@ public: unsigned int MemoryAllocation() const override; void DescribeReST(ODesc* d) const override; + notifier::Modifiable* Modifiable() override { return this; } + // Extend the underlying arrays of record instances created during // parsing to match the number of fields in the record type (they may // mismatch as a result of parse-time record type redefinitions. @@ -1006,7 +1010,7 @@ protected: }; -class VectorVal : public MutableVal { +class VectorVal : public Val, public notifier::Modifiable { public: explicit VectorVal(VectorType* t); ~VectorVal() override; @@ -1054,6 +1058,8 @@ public: // Won't shrink size. unsigned int ResizeAtLeast(unsigned int new_num_elements); + notifier::Modifiable* Modifiable() override { return this; } + protected: friend class Val; VectorVal() { } From 7bd738865cdab1d5983a96a16f5d199166cb3fcb Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 6 Jun 2019 23:48:31 +0000 Subject: [PATCH 08/22] Remove MutableVal class. --- src/ID.h | 4 ---- src/StateAccess.cc | 23 ++++++++++++++--------- src/StateAccess.h | 21 ++++++++++++--------- src/Stats.cc | 2 +- src/Trigger.cc | 11 ++++++++--- src/Trigger.h | 4 ++-- src/Type.h | 4 ---- src/Val.cc | 4 ---- src/Val.h | 14 -------------- src/bro.bif | 5 +---- src/input/readers/config/Config.cc | 2 +- 11 files changed, 39 insertions(+), 55 deletions(-) diff --git a/src/ID.h b/src/ID.h index a1cbe494d3..9c9a842c39 100644 --- a/src/ID.h +++ b/src/ID.h @@ -70,10 +70,6 @@ public: bool IsRedefinable() const { return FindAttr(ATTR_REDEF) != 0; } - // Returns true if ID is one of those internal globally unique IDs - // to which MutableVals are bound (there name start with a '#'). - bool IsInternalGlobal() const { return name && name[0] == '#'; } - void SetAttrs(Attributes* attr); void AddAttrs(Attributes* attr); void RemoveAttr(attr_tag a); diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 7d7c397981..b580215271 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -14,8 +14,8 @@ notifier::Registry::~Registry() void notifier::Registry::Register(Modifiable* m, notifier::Notifier* notifier) { - DBG_LOG(DBG_NOTIFIERS, "registering modifiable %p for notifier %s", - m, notifier->Name()); + DBG_LOG(DBG_NOTIFIERS, "registering modifiable %p for notifier %p", + m, notifier); registrations.insert({m, notifier}); ++m->notifiers; @@ -23,16 +23,16 @@ void notifier::Registry::Register(Modifiable* m, notifier::Notifier* notifier) void notifier::Registry::Unregister(Modifiable* m, notifier::Notifier* notifier) { - DBG_LOG(DBG_NOTIFIERS, "unregistering modifiable %p from notifier %s", - m, notifier->Name()); + DBG_LOG(DBG_NOTIFIERS, "unregistering modifiable %p from notifier %p", + m, notifier); auto x = registrations.equal_range(m); for ( auto i = x.first; i != x.second; i++ ) { if ( i->second == notifier ) { - registrations.erase(i); --i->first->notifiers; + registrations.erase(i); break; } } @@ -40,9 +40,14 @@ void notifier::Registry::Unregister(Modifiable* m, notifier::Notifier* notifier) void notifier::Registry::Unregister(Modifiable* m) { + DBG_LOG(DBG_NOTIFIERS, "unregistering modifiable %p from all notifiers", + m); + auto x = registrations.equal_range(m); for ( auto i = x.first; i != x.second; i++ ) - Unregister(m, i->second); + --i->first->notifiers; + + registrations.erase(x.first, x.second); } void notifier::Registry::Modified(Modifiable* m) @@ -54,8 +59,8 @@ void notifier::Registry::Modified(Modifiable* m) i->second->Modified(m); } -const char* notifier::Notifier::Name() const +notifier::Modifiable::~Modifiable() { - return fmt("%p", this); + if ( notifiers ) + registry.Unregister(this); } - diff --git a/src/StateAccess.h b/src/StateAccess.h index 91a7fa6186..b769e320c0 100644 --- a/src/StateAccess.h +++ b/src/StateAccess.h @@ -18,6 +18,7 @@ #include #include "util.h" +#include "DebugLogger.h" namespace notifier { @@ -25,11 +26,18 @@ class Modifiable; class Notifier { public: - virtual ~Notifier() { } + Notifier() + { + DBG_LOG(DBG_NOTIFIERS, "creating notifier %p", this); + } - // Called afger a change has been performed. + virtual ~Notifier() + { + DBG_LOG(DBG_NOTIFIERS, "destroying notifier %p", this); + } + + // Called after a change has been performed. virtual void Modified(Modifiable* m) = 0; - virtual const char* Name() const; // for debugging }; // Singleton class. @@ -67,12 +75,7 @@ protected: friend class Registry; Modifiable() {} - ~Modifiable() - { - if ( notifiers ) - registry.Unregister(this); - - }; + virtual ~Modifiable(); void Modified() { diff --git a/src/Stats.cc b/src/Stats.cc index 1d2a2c8ad8..9489f12f93 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -255,7 +255,7 @@ void ProfileLogger::Log() while ( (id = globals->NextEntry(c)) ) // We don't show/count internal globals as they are always // contained in some other global user-visible container. - if ( id->HasVal() && ! id->IsInternalGlobal() ) + if ( id->HasVal() ) { Val* v = id->ID_Val(); diff --git a/src/Trigger.cc b/src/Trigger.cc index c1295b8218..8de5dff5bd 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -385,7 +385,7 @@ void Trigger::Register(ID* id) notifier::registry.Register(id, this); Ref(id); - objs.push_back(id); + objs.push_back({id, id}); } void Trigger::Register(Val* val) @@ -397,13 +397,18 @@ void Trigger::Register(Val* val) notifier::registry.Register(val->Modifiable(), this); Ref(val); - objs.push_back(val); + objs.emplace_back(val, val->Modifiable()); } void Trigger::UnregisterAll() { + DBG_LOG(DBG_NOTIFIERS, "%s: unregistering all", Name()); + for ( auto o : objs ) - Unref(o); // this will unregister with the registry as well + { + notifier::registry.Unregister(o.second, this); + Unref(o.first); + } objs.clear(); } diff --git a/src/Trigger.h b/src/Trigger.h index fe7d425a72..3004d30732 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -64,7 +64,7 @@ public: void Modified(notifier::Modifiable* m) override { QueueTrigger(this); } - const char* Name() const override; + const char* Name() const; static void QueueTrigger(Trigger* trigger); @@ -102,7 +102,7 @@ private: bool delayed; // true if a function call is currently being delayed bool disabled; - std::vector objs; + std::vector> objs; typedef map ValCache; ValCache cache; diff --git a/src/Type.h b/src/Type.h index a97f7360c8..bc8d673443 100644 --- a/src/Type.h +++ b/src/Type.h @@ -693,10 +693,6 @@ bool is_atomic_type(const BroType* t); // True if the given type tag corresponds to a function type. #define IsFunc(t) (t == TYPE_FUNC) -// True if the given type tag corresponds to mutable type. -#define IsMutable(t) \ - (t == TYPE_RECORD || t == TYPE_TABLE || t == TYPE_VECTOR) - // True if the given type type is a vector. #define IsVector(t) (t == TYPE_VECTOR) diff --git a/src/Val.cc b/src/Val.cc index bea9ce4038..0d6f5838ca 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -349,10 +349,6 @@ void Val::ValDescribeReST(ODesc* d) const } } -MutableVal::~MutableVal() - { - } - IntervalVal::IntervalVal(double quantity, double units) : Val(quantity * units, TYPE_INTERVAL) { diff --git a/src/Val.h b/src/Val.h index d98c352f6d..48bbd50cfc 100644 --- a/src/Val.h +++ b/src/Val.h @@ -49,7 +49,6 @@ class RecordVal; class ListVal; class StringVal; class EnumVal; -class MutableVal; class VectorVal; @@ -320,11 +319,6 @@ public: CONST_CONVERTER(TYPE_STRING, StringVal*, AsStringVal) CONST_CONVERTER(TYPE_VECTOR, VectorVal*, AsVectorVal) - bool IsMutableVal() const - { - return IsMutable(type->Tag()); - } - void Describe(ODesc* d) const override; virtual void DescribeReST(ODesc* d) const; @@ -486,14 +480,6 @@ private: extern ValManager* val_mgr; -class MutableVal : public Val { -protected: - explicit MutableVal(BroType* t) : Val(t) {} - - MutableVal() {} - ~MutableVal() override; -}; - #define Microseconds 1e-6 #define Milliseconds 1e-3 #define Seconds 1.0 diff --git a/src/bro.bif b/src/bro.bif index 47b713e4e1..e143cf16e8 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1819,7 +1819,7 @@ function global_sizes%(%): var_sizes ID* id; while ( (id = globals->NextEntry(c)) ) - if ( id->HasVal() && ! id->IsInternalGlobal() ) + if ( id->HasVal() ) { Val* id_name = new StringVal(id->Name()); Val* id_size = val_mgr->GetCount(id->ID_Val()->MemoryAllocation()); @@ -1847,9 +1847,6 @@ function global_ids%(%): id_table ID* id; while ( (id = globals->NextEntry(c)) ) { - if ( id->IsInternalGlobal() ) - continue; - RecordVal* rec = new RecordVal(script_id); rec->Assign(0, new StringVal(type_name(id->Type()->Tag()))); rec->Assign(1, val_mgr->GetBool(id->IsExport())); diff --git a/src/input/readers/config/Config.cc b/src/input/readers/config/Config.cc index eca276281c..7c1708babd 100644 --- a/src/input/readers/config/Config.cc +++ b/src/input/readers/config/Config.cc @@ -33,7 +33,7 @@ Config::Config(ReaderFrontend *frontend) : ReaderBackend(frontend) while ( auto id = globals->NextEntry(c) ) { - if ( id->IsInternalGlobal() || ! id->IsOption() ) + if ( ! id->IsOption() ) continue; if ( id->Type()->Tag() == TYPE_RECORD || From 6adab8d46a98db691fbc1ed3ae4933d519183b1e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 7 Jun 2019 23:38:25 +0000 Subject: [PATCH 09/22] Clean up new code. --- src/StateAccess.cc | 42 +++++++++-------- src/StateAccess.h | 109 +++++++++++++++++++++++++++------------------ src/Trigger.cc | 2 +- src/Trigger.h | 2 +- src/Val.h | 2 + 5 files changed, 94 insertions(+), 63 deletions(-) diff --git a/src/StateAccess.cc b/src/StateAccess.cc index b580215271..dcf5ef8b21 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -1,37 +1,44 @@ -#include "Val.h" +// See the file "COPYING" in the main distribution directory for copyright. + #include "StateAccess.h" -#include "Event.h" -#include "NetVar.h" #include "DebugLogger.h" notifier::Registry notifier::registry; +notifier::Receiver::Receiver() + { + DBG_LOG(DBG_NOTIFIERS, "creating receiver %p", this); + } + +notifier::Receiver::~Receiver() + { + DBG_LOG(DBG_NOTIFIERS, "deleting receiver %p", this); + } + notifier::Registry::~Registry() { for ( auto i : registrations ) Unregister(i.first); } -void notifier::Registry::Register(Modifiable* m, notifier::Notifier* notifier) +void notifier::Registry::Register(Modifiable* m, notifier::Receiver* r) { - DBG_LOG(DBG_NOTIFIERS, "registering modifiable %p for notifier %p", - m, notifier); + DBG_LOG(DBG_NOTIFIERS, "registering object %p for receiver %p", m, r); - registrations.insert({m, notifier}); - ++m->notifiers; + registrations.insert({m, r}); + ++m->num_receivers; } -void notifier::Registry::Unregister(Modifiable* m, notifier::Notifier* notifier) +void notifier::Registry::Unregister(Modifiable* m, notifier::Receiver* r) { - DBG_LOG(DBG_NOTIFIERS, "unregistering modifiable %p from notifier %p", - m, notifier); + DBG_LOG(DBG_NOTIFIERS, "unregistering object %p from receiver %p", m, r); auto x = registrations.equal_range(m); for ( auto i = x.first; i != x.second; i++ ) { - if ( i->second == notifier ) + if ( i->second == r ) { - --i->first->notifiers; + --i->first->num_receivers; registrations.erase(i); break; } @@ -40,19 +47,18 @@ void notifier::Registry::Unregister(Modifiable* m, notifier::Notifier* notifier) void notifier::Registry::Unregister(Modifiable* m) { - DBG_LOG(DBG_NOTIFIERS, "unregistering modifiable %p from all notifiers", - m); + DBG_LOG(DBG_NOTIFIERS, "unregistering object %p from all notifiers", m); auto x = registrations.equal_range(m); for ( auto i = x.first; i != x.second; i++ ) - --i->first->notifiers; + --i->first->num_receivers; registrations.erase(x.first, x.second); } void notifier::Registry::Modified(Modifiable* m) { - DBG_LOG(DBG_NOTIFIERS, "modification to modifiable %p", m); + DBG_LOG(DBG_NOTIFIERS, "object %p has been modified", m); auto x = registrations.equal_range(m); for ( auto i = x.first; i != x.second; i++ ) @@ -61,6 +67,6 @@ void notifier::Registry::Modified(Modifiable* m) notifier::Modifiable::~Modifiable() { - if ( notifiers ) + if ( num_receivers ) registry.Unregister(this); } diff --git a/src/StateAccess.h b/src/StateAccess.h index b769e320c0..2361801ec4 100644 --- a/src/StateAccess.h +++ b/src/StateAccess.h @@ -1,14 +1,9 @@ -// A class describing a state-modyfing access to a Value or an ID. +// See the file "COPYING" in the main distribution directory for copyright. // -// TODO UPDATE: We provide a notifier framework to inform interested parties -// of modifications to selected global IDs/Vals. To get notified about a -// change, derive a class from Notifier and register the interesting -// instances with the NotifierRegistry. -// -// Note: For containers (e.g., tables), notifications are only issued if the -// container itself is modified, *not* for changes to the values contained -// therein. - +// A notification framework to inform interested parties of modifications to +// selected global objects. To get notified about a change, derive a class +// from notifier::Receiver and register the interesting objects with the +// notification::Registry. #ifndef STATEACESSS_H #define STATEACESSS_H @@ -24,70 +19,98 @@ namespace notifier { class Modifiable; -class Notifier { +/** Interface class for receivers of notifications. */ +class Receiver { public: - Notifier() - { - DBG_LOG(DBG_NOTIFIERS, "creating notifier %p", this); - } + Receiver(); + virtual ~Receiver(); - virtual ~Notifier() - { - DBG_LOG(DBG_NOTIFIERS, "destroying notifier %p", this); - } - - // Called after a change has been performed. + /** + * Callback executed when a register object has been modified. + * + * @param m object that was modified + */ virtual void Modified(Modifiable* m) = 0; }; -// Singleton class. +/** Singleton class tracking all notification requests globally. */ class Registry { public: - Registry() { } ~Registry(); - // Register a new notifier to be informed when an instance changes. - void Register(Modifiable* m, Notifier* notifier); + /** + * Registers a receiver to be informed when a modifiable object has + * changed. + * + * @param m object to track. Does not take ownership, but the object + * will automatically unregister itself on destruction. + * + * @param r receiver to notify on changes. Does not take ownershop, + * the receiver must remain valid as long as the registration stays + * in place. + */ + void Register(Modifiable* m, Receiver* r); - // Cancel a notifier's tracking an instace. - void Unregister(Modifiable* m, Notifier* notifier); + /** + * Cancels a receiver's request to be informed about an object's + * modification. The arguments to the method must match what was + * originally registered. + * + * @param m object to no loger track. + * + * @param r receiver to no longer notify. + */ + void Unregister(Modifiable* m, Receiver* Receiver); - // Cancel all notifiers registered for an instance. + /** + * Cancels any active receiver requests to be informed about a + * partilar object's modifications. + * + * @param m object to no loger track. + */ void Unregister(Modifiable* m); private: friend class Modifiable; - // Inform all registered notifiers of a modification to an instance. + // Inform all registered receivers of a modification to an object. + // Will be called from the object itself. void Modified(Modifiable* m); - typedef std::unordered_multimap ModifiableMap; + typedef std::unordered_multimap ModifiableMap; ModifiableMap registrations; }; - -class Registry; +/** + * Singleton object tracking all global notification requests. + */ extern Registry registry; -// Base class for objects wanting to signal modifications to the registry. +/** + * Base class for objects that can trigger notifications to receivers when + * modified. + */ class Modifiable { -protected: - friend class Registry; - - Modifiable() {} - virtual ~Modifiable(); - +public: + /** + * Calling this method signals to all registered receivers that the + * object has been modified. + */ void Modified() { - if ( notifiers ) + if ( num_receivers ) registry.Modified(this); } - // Number of currently registered notifiers for this instance. - uint64 notifiers; +protected: + friend class Registry; + + virtual ~Modifiable(); + + // Number of currently registered receivers. + uint64 num_receivers; }; } - #endif diff --git a/src/Trigger.cc b/src/Trigger.cc index 8de5dff5bd..ae6483e3f5 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -404,7 +404,7 @@ void Trigger::UnregisterAll() { DBG_LOG(DBG_NOTIFIERS, "%s: unregistering all", Name()); - for ( auto o : objs ) + for ( const auto& o : objs ) { notifier::registry.Unregister(o.second, this); Unref(o.first); diff --git a/src/Trigger.h b/src/Trigger.h index 3004d30732..0de5499045 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -13,7 +13,7 @@ class TriggerTimer; class TriggerTraversalCallback; -class Trigger : public notifier::Notifier, public BroObj { +class Trigger : public BroObj, public notifier::Receiver { public: // Don't access Trigger objects; they take care of themselves after // instantiation. Note that if the condition is already true, the diff --git a/src/Val.h b/src/Val.h index 48bbd50cfc..8eb33d1f0e 100644 --- a/src/Val.h +++ b/src/Val.h @@ -322,6 +322,8 @@ public: void Describe(ODesc* d) const override; virtual void DescribeReST(ODesc* d) const; + // To be overridden by mutable derived class to enable change + // notification. virtual notifier::Modifiable* Modifiable() { return 0; } #ifdef DEBUG From 32f30b5c716ccab9cd8a0d9eac6e8a63b473ad14 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 8 Jun 2019 00:27:23 +0000 Subject: [PATCH 10/22] Renaming src/StateAccess.{h,cc} to src/Notifier.{h,cc}. The old names did not reflect the content of the files anymore. --- src/CMakeLists.txt | 2 +- src/ID.h | 2 +- src/{StateAccess.cc => Notifier.cc} | 2 +- src/{StateAccess.h => Notifier.h} | 4 ++-- src/Trigger.h | 2 +- src/Val.h | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) rename src/{StateAccess.cc => Notifier.cc} (98%) rename src/{StateAccess.h => Notifier.h} (98%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1afa5193cc..6620779de8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -302,7 +302,7 @@ set(bro_SRCS Scope.cc SerializationFormat.cc Sessions.cc - StateAccess.cc + Notifier.cc Stats.cc Stmt.cc Tag.cc diff --git a/src/ID.h b/src/ID.h index 9c9a842c39..ee60d4e61c 100644 --- a/src/ID.h +++ b/src/ID.h @@ -5,7 +5,7 @@ #include "Type.h" #include "Attr.h" -#include "StateAccess.h" +#include "Notifier.h" #include "TraverseTypes.h" #include diff --git a/src/StateAccess.cc b/src/Notifier.cc similarity index 98% rename from src/StateAccess.cc rename to src/Notifier.cc index dcf5ef8b21..15bc334af6 100644 --- a/src/StateAccess.cc +++ b/src/Notifier.cc @@ -1,7 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "StateAccess.h" #include "DebugLogger.h" +#include "Notifier.h" notifier::Registry notifier::registry; diff --git a/src/StateAccess.h b/src/Notifier.h similarity index 98% rename from src/StateAccess.h rename to src/Notifier.h index 2361801ec4..0a399e526e 100644 --- a/src/StateAccess.h +++ b/src/Notifier.h @@ -5,8 +5,8 @@ // from notifier::Receiver and register the interesting objects with the // notification::Registry. -#ifndef STATEACESSS_H -#define STATEACESSS_H +#ifndef NOTIFIER_H +#define NOTIFIER_H #include #include diff --git a/src/Trigger.h b/src/Trigger.h index 0de5499045..2e0c91865f 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -4,7 +4,7 @@ #include #include -#include "StateAccess.h" +#include "Notifier.h" #include "Traverse.h" // Triggers are the heart of "when" statements: expressions that when diff --git a/src/Val.h b/src/Val.h index 8eb33d1f0e..959408da8c 100644 --- a/src/Val.h +++ b/src/Val.h @@ -18,7 +18,7 @@ #include "Timer.h" #include "ID.h" #include "Scope.h" -#include "StateAccess.h" +#include "Notifier.h" #include "IPAddr.h" #include "DebugLogger.h" From 632e83de576d50280a192babc224a3ffaf1e47c7 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 19 Jun 2019 09:29:32 -0700 Subject: [PATCH 11/22] Remove old, unmaintained p0f support. Addresses GH-417 --- NEWS | 5 + scripts/base/init-bare.zeek | 27 - scripts/base/misc/p0f.fp | 834 ------------------------------- src/CMakeLists.txt | 1 - src/NetVar.cc | 7 - src/NetVar.h | 3 - src/OSFinger.cc | 689 ------------------------- src/OSFinger.h | 161 ------ src/Sessions.cc | 29 -- src/Sessions.h | 10 - src/analyzer/protocol/tcp/TCP.cc | 198 -------- src/event.bif | 21 +- 12 files changed, 7 insertions(+), 1978 deletions(-) delete mode 100644 scripts/base/misc/p0f.fp delete mode 100644 src/OSFinger.cc delete mode 100644 src/OSFinger.h diff --git a/NEWS b/NEWS index a8fd5ae970..0f1281c911 100644 --- a/NEWS +++ b/NEWS @@ -429,6 +429,11 @@ Removed Functionality - Removed the BroControl ``update`` command, which was deprecated in Bro 2.6. +- Removed p0f (passive OS fingerprinting) support. The version of p0f shipped with zeek was + ancient, probably did not give any reliable support anymore and did not offer a clear upgrade + path. The ``OS_version_found`` event as well as the ``generate_OS_version_event`` configuration + option were removed. + Deprecated Functionality ------------------------ diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index f68bf3a545..3d88809466 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -1844,9 +1844,6 @@ function add_signature_file(sold: string, snew: string): string ## since that can search paths relative to the current script. global signature_files = "" &add_func = add_signature_file; -## ``p0f`` fingerprint file to use. Will be searched relative to ``ZEEKPATH``. -const passive_fingerprint_file = "base/misc/p0f.fp" &redef; - ## Definition of "secondary filters". A secondary filter is a BPF filter given ## as index in this table. For each such filter, the corresponding event is ## raised for all matching packets. @@ -3991,30 +3988,6 @@ type software: record { version: software_version; }; -## Quality of passive fingerprinting matches. -## -## .. zeek:see:: OS_version -type OS_version_inference: enum { - direct_inference, ##< TODO. - generic_inference, ##< TODO. - fuzzy_inference, ##< TODO. -}; - -## Passive fingerprinting match. -## -## .. zeek:see:: OS_version_found -type OS_version: record { - genre: string; ##< Linux, Windows, AIX, ... - detail: string; ##< Kernel version or such. - dist: count; ##< How far is the host away from the sensor (TTL)?. - match_type: OS_version_inference; ##< Quality of the match. -}; - -## Defines for which subnets we should do passive fingerprinting. -## -## .. zeek:see:: OS_version_found -global generate_OS_version_event: set[subnet] &redef; - # Type used to report load samples via :zeek:see:`load_sample`. For now, it's a # set of names (event names, source file names, and perhaps ````), which were seen during the sample. diff --git a/scripts/base/misc/p0f.fp b/scripts/base/misc/p0f.fp deleted file mode 100644 index 0ee37b4e37..0000000000 --- a/scripts/base/misc/p0f.fp +++ /dev/null @@ -1,834 +0,0 @@ -# -# p0f - SYN fingerprints -# ---------------------- -# -# .-------------------------------------------------------------------------. -# | The purpose of this file is to cover signatures for incoming TCP/IP | -# | connections (SYN packets). This is the default mode of operation for | -# | p0f. This is also the biggest and most up-to-date set of signatures | -# | shipped with this project. The file also contains a detailed discussion | -# | of all metrics examined by p0f, and some practical notes on how to | -# | add new signatures. | -# `-------------------------------------------------------------------------' -# -# (C) Copyright 2000-2006 by Michal Zalewski -# -# Each line in this file specifies a single fingerprint. Please read the -# information below carefully before attempting to append any signatures -# reported by p0f as UNKNOWN to this file to avoid mistakes. Note that -# this file is compatible only with the default operation mode, and not -# with -R or -A options (SYN+ACK and RST+ modes). -# -# We use the following set metrics for fingerprinting: -# -# - Window size (WSS) - a highly OS dependent setting used for TCP/IP -# performance control (max. amount of data to be sent without ACK). -# Some systems use a fixed value for initial packets. On other -# systems, it is a multiple of MSS or MTU (MSS+40). In some rare -# cases, the value is just arbitrary. -# -# NEW SIGNATURE: if p0f reported a special value of 'Snn', the number -# appears to be a multiple of MSS (MSS*nn); a special value of 'Tnn' -# means it is a multiple of MTU ((MSS+40)*nn). Unless you notice the -# value of nn is not fixed (unlikely), just copy the Snn or Tnn token -# literally. If you know this device has a simple stack and a fixed -# MTU, you can however multiply S value by MSS, or T value by MSS+40, -# and put it instead of Snn or Tnn. One system may exhibit several T -# or S values. In some situations, this might be a source of some -# additional information about the setup if you have some time to dig -# thru the kernel sources; in some other cases, like Windows, there seem -# to be a multitude of variants and WSS selection algorithms, but it's -# rather difficult to find a pattern without having the source. -# -# If WSS looks like a regular fixed value (for example is a power of two), -# or if you can confirm the value is fixed by looking at several -# fingerprints, please quote it literaly. If there's no apparent pattern -# in WSS chosen, you should consider wildcarding this value - but this -# should be the last option. -# -# NOTE: Some NAT devices, such as Linux iptables with --set-mss, will -# modify MSS, but not WSS. As a result, MSS is changed to reflect -# the MTU of the NAT device, but WSS remains a multiple of the original -# MSS. Fortunately for us, the source device would almost always be -# hooked up to Ethernet. P0f handles it automatically for the original -# MSS of 1460, by adding "NAT!" tag to the result. -# -# In certain configurations, Linux erratically (?) uses MTU from another -# interface on the default gw interface. This only happens on systems with -# two network interfaces. Thus, some Linux systems that do not go thru NAT, -# but have multiple interfaces instead, will be also tagged this way. -# -# P0f recognizes and automatically wildcards WSS of 12345, as generated -# by sendack and sendsyn utilities shipped with the program, when -# reporting a new signature. See test/sendack.c and test/sendsyn.c for more -# information about this. -# -# - Overall packet size - a function of all IP and TCP options and bugs. -# While this is partly redundant in the real world, we record this value -# to capture rare cases when there are IP options (which we do not currently -# examine) or packet data past the headers. Both situations are rare. -# -# Packet size MAY be wildcarded, but the meaning of the wildcard is -# very special, and means the packet must be larger than PACKET_BIG -# (defined in config.h as 100). This is usually not necessary, except -# for some really broken implementations in RST+ mode. For more information, -# see p0fr.fp. P0f automatically wildcards big packets when reporting -# new signatures. -# -# NEW SIGNATURE: Copy this value literally. -# -# - Initial TTL - We check the actual TTL of a received packet. It can't -# be higher than the initial TTL, and also shouldn't be dramatically -# lower (maximum distance is defined in config.h as 40 hops). -# -# NEW SIGNATURE: *Never* copy TTL from a p0f-reported signature literally. -# You need to determine the initial TTL. The best way to do it is to -# check the documentation for a remote system, or check its settings. -# A fairly good method is to simply round the observed TTL up to -# 32, 64, 128, or 255, but it should be noted that some obscure devices -# might not use round TTLs (in particular, some shoddy appliances and -# IRIX and Tru64 are known to use "original" initial TTL settings). If not -# sure, use traceroute or mtr to see how far you are from the host. -# -# Note that -F option overrides this check if no signature can be found. -# -# - Don't fragment flag (DF) - some modern OSes set this to implement PMTU -# discovery. Others do not bother. -# -# NEW SIGNATURE: Copy this value literally. Note: this setting is -# sometimes cleared by firewalls and/or certain connectivity clients. -# Try to find out what's the actual state for a given OS if you see both, -# and add the right one. P0f will automatically detect a case when a -# firewall removed the DF flag and will append "(firewall!)" suffix to -# the signature, so if the DF version is the right one, don't add no-DF -# variant, unless it has a different meaning. -# -# - Maximum segment size (MSS) - this setting is usually link-dependent. P0f -# uses it to determine link type of the remote host. -# -# NEW SIGNATURE: Always wildcard this value, except for rare cases when -# you have an appliance with a fixed value, know the system supports only -# a very limited number of network interface types, or know the system -# is using a value it pulled out of nowhere. I use specific unique MSS -# to tell Google crawlbots from the rest of Linux population, for example. -# -# If a specific MSS/MTU is unique to a certain link type, be sure to -# add it to mtu.h instead of creating several variants of each signature. -# -# - Window scaling (WSCALE) - this feature is used to scale WSS. -# It extends the size of a TCP/IP window to 32 bits, of sorts. Some modern -# systems implement this feature. -# -# NEW SIGNATURE: Observe several signatures. Initial WSCALE is often set -# to zero or other low value. There's usually no need to wildcard this -# parameter. -# -# - Timestamp - some systems that implement timestamps set them to -# zero in the initial SYN. This case is detected and handled appropriately. -# -# NEW SIGNATURE: Copy T or T0 option literally. -# -# - Selective ACK permitted - a flag set by systems that implement -# selective ACK functionality, -# -# NEW SIGNATURE: copy S option literally. -# -# - NOP option - its presence, count and sequence is a useful OS-dependent -# characteristic, -# -# NEW SIGNATURE: copy N options literally. -# -# - Other and unrecognized options (TTCP-related and such) - implemented by -# some eccentric or very buggy TCP/IP stacks ;-), -# -# NEW SIGNATURE: copy ? options literally. -# -# - EOL option. Contrary to the popular belief, the presence of EOL -# option is actually quite rare, most systems just NOP-pad to the -# packet boundary. -# -# NEW SIGNATURE: copy E option literally. -# -# - The sequence of TCP all options mentioned above - this is very -# specific to the implementation, -# -# NEW SIGNATURE: Copy the sequence literally. -# -# - Quirks. Some buggy stacks set certain values that should be zeroed in a -# TCP packet to non-zero values. This has no effect as of today, but is -# a valuable source of information. Some systems actually seem to leak -# memory there. Other systems just exhibit harmful but very specific -# behavior. This section captures all unusual yes-no properties not -# related to the main and expected header layout. We detect the following: -# -# - Data past the headers. Neither SYN nor SYN+ACK packets are supposed -# to carry any payload. If they do, we should take notice. The actual -# payload is not examined, but will be displayed if use the -X option. -# Note that payload is not unusual in RST+ mode (see p0fr.fp), very -# rare otherwise. -# -# - Options past EOL. Some systems have some trailing data past EOL -# in the options section of TCP/IP headers. P0f does not examine this -# data as of today, simply detects its presence. If there is a -# confirmed sizable population of systems that have data past EOL, it -# might be a good idea to look at it. Until then, you have to recompile -# p0f with DEBUG_EXTRAS set or use -x to display this data, -# -# - Zero IP ID. This again is a (mostly) harmless setting to use a fixed -# IP ID for packets with DF set. Some systems reportedly use zero ID, -# most OSes do not. There is a very slight probability of a false -# positive when IP ID is "naturally" chosen to be zero on a system -# that otherwise does set proper values, but the probability is -# neglible (if it becomes a problem, recompile p0f with IGNORE_ZEROID -# set in the sources). -# -# - IP options specified. Usually, packets do not have any IP options -# set, but there can be some. Until there is a confirmed sizable -# population of systems that do have IP options in a packet, p0f -# does not examine those in detail, but it might change (use -# DEBUG_EXTRAS or -x to display IP options if any found), -# -# - URG pointer value. SYN packets do not have URG flag set, so the -# value in URG pointer in TCP header is ignored. Most systems set it -# to zero, but some OSes (some versions of Windows, for example) do -# not zero this field or even simply leak memory; the actual value is -# not examined, because most cases seem to be just random garbage -# (you can use DEBUG_EXTRAS or -x to report this information though); -# see doc/win-memleak.txt for more information, -# -# - "Unused" field value. This should be always zero, but some systems -# forget to clear it. This might result in some funny issues in the -# future. P0f checks for non-zero value (and will display it if -# DEBUG_EXTRAS is set, or you can use -x), -# -# - ACK number non-zero. ACK value in SYN packets with no ACK flag -# is disregarded and is usually set to zero (just like with URG -# pointer), but some systems forget to do it. The exact value is -# not examined (but will be displayed with DEBUG_EXTRAS, or you can -# use -x). Note that this is not an anomaly in SYN+ACK and RST+ modes, -# -# - Non-zero second timestamp. The initial SYN packet should have the -# second timestamp always zeroed. SYN+ACK and RST+ may "legally" have -# this quirk though, -# -# - Unusual flags. If, in addition to SYN (or SYN+ACK), there are some -# auxilinary flags that do not modify the very meaning of a packet, -# p0f records this (this can be URG, PUSH, or something else). -# -# Note: ECN flags (ECE and CWR) are ignored and denoted in a separate -# way. ECN is never by default, because some systems can't handle it, -# and it probably does not make much sense to include it in signatures -# right now. -# -# - TCP option segment parsing problems. If p0f fails to decode options -# because of a badly broken packet, it records this fact. -# -# There are several other quirks valid only in RST+ mode, see p0fr.fp for -# more information. Those quirks are unheard of in SYN and SYN+ACK -# modes. -# -# NEW SIGNATURE: Copy "quirks" section literally. -# -# We DO NOT use ToS for fingerprinting. While the original TCP/IP -# fingerprinting research believed this value would be useful for this -# purpose, it is not. The setting is way too often tweaked by network -# devices. -# -# To wildcard MSS, WSS or WSCALE, replace it with '*'. You can also use a -# modulo operator to match any values that divide by nnn - '%nnn' (and, -# as stated above, WSS also supports special values Snn and Tnn). -# -# Fingerprint entry format: -# -# wwww:ttt:D:ss:OOO...:QQ:OS:Details -# -# wwww - window size (can be * or %nnn or Sxx or Txx) -# "Snn" (multiple of MSS) and "Tnn" (multiple of MTU) are allowed. -# ttt - initial TTL -# D - don't fragment bit (0 - not set, 1 - set) -# ss - overall SYN packet size (* has a special meaning) -# OOO - option value and order specification (see below) -# QQ - quirks list (see below) -# OS - OS genre (Linux, Solaris, Windows) -# details - OS description (2.0.27 on x86, etc) -# -# If OS genre starts with '*', p0f will not show distance, link type -# and timestamp data. It is useful for userland TCP/IP stacks of -# network scanners and so on, where many settings are randomized or -# bogus. -# -# If OS genre starts with @, it denotes an approximate hit for a group -# of operating systems (signature reporting still enabled in this case). -# Use this feature at the end of this file to catch cases for which -# you don't have a precise match, but can tell it's Windows or FreeBSD -# or whatnot by looking at, say, flag layout alone. -# -# If OS genre starts with - (which can prefix @ or *), the entry is -# not considered to be a real operating system (but userland stack -# instead). It is important to mark all scanners and so on with -, -# so that they are not used for masquerade detection (also add this -# prefix for signatures of application-induced behavior, such as -# increased window size with Opera browser). -# -# Option block description is a list of comma or space separated -# options in the order they appear in the packet: -# -# N - NOP option -# E - EOL option -# Wnnn - window scaling option, value nnn (or * or %nnn) -# Mnnn - maximum segment size option, value nnn (or * or %nnn) -# S - selective ACK OK -# T - timestamp -# T0 - timestamp with zero value -# ?n - unrecognized option number n. -# -# P0f can sometimes report ?nn among the options. This means it couldn't -# recognize this option (option number nn). It's either a bug in p0f, or -# a faulty TCP/IP stack, or, if the number is listed here: -# -# http://www.iana.org/assignments/tcp-parameters -# -# ...the stack might be simply quite exotic. -# -# To denote no TCP options, use a single '.'. -# -# Quirks section is usually an empty list ('.') of oddities or bugs of this -# particular stack. List items are not separated in any way. Possible values: -# -# P - options past EOL, -# Z - zero IP ID, -# I - IP options specified, -# U - urg pointer non-zero, -# X - unused (x2) field non-zero, -# A - ACK number non-zero, -# T - non-zero second timestamp, -# F - unusual flags (PUSH, URG, etc), -# D - data payload, -# ! - broken options segment. -# -# WARNING WARNING WARNING -# ----------------------- -# -# Do not add a system X as OS Y just because NMAP says so. It is often -# the case that X is a NAT firewall. While nmap is talking to the -# device itself, p0f is fingerprinting the guy behind the firewall -# instead. -# -# When in doubt, use common sense, don't add something that looks like -# a completely different system as Linux or FreeBSD or LinkSys router. -# Check DNS name, establish a connection to the remote host and look -# at SYN+ACK (p0f -A -S should do) - does it look similar? -# -# Some users tweak their TCP/IP settings - enable or disable RFC1323, -# RFC1644 or RFC2018 support, disable PMTU discovery, change MTU, initial -# TTL and so on. Always compare a new rule to other fingerprints for -# this system, and verify the system isn't "customized". It is OK to -# add signature variants caused by commonly used software (PFs, security -# packages, etc), but it makes no sense to try to add every single -# possible /proc/sys/net/ipv4/* tweak on Linux or so. -# -# KEEP IN MIND: Some packet firewalls configured to normalize outgoing -# traffic (OpenBSD pf with "scrub" enabled, for example) will, well, -# normalize packets. Signatures will not correspond to the originating -# system (and probably not quite to the firewall either). -# -# NOTE: Try to keep this file in some reasonable order, from most to -# least likely systems. This will speed up operation. Also keep most -# generic and broad rules near ehe end. -# -# Still decided to add signature? Let us know - mail a copy of your discovery -# to lcamtuf@coredump.cx. You can help make p0f better, and I can help you -# make your signature more accurate. -# - -########################## -# Standard OS signatures # -########################## - -# ----------------- AIX --------------------- - -# AIX is first because its signatures are close to NetBSD, MacOS X and -# Linux 2.0, but it uses a fairly rare MSSes, at least sometimes... -# This is a shoddy hack, though. - -45046:64:0:44:M*:.:AIX:4.3 - -16384:64:0:44:M512:.:AIX:4.3.2 and earlier - -16384:64:0:60:M512,N,W%2,N,N,T:.:AIX:4.3.3-5.2 (1) -32768:64:0:60:M512,N,W%2,N,N,T:.:AIX:4.3.3-5.2 (2) -65535:64:0:60:M512,N,W%2,N,N,T:.:AIX:4.3.3-5.2 (3) - -65535:64:0:64:M*,N,W1,N,N,T,N,N,S:.:AIX:5.3 ML1 - -# ----------------- Linux ------------------- - -S1:64:0:44:M*:A:Linux:1.2.x -512:64:0:44:M*:.:Linux:2.0.3x (1) -16384:64:0:44:M*:.:Linux:2.0.3x (2) - -# Endian snafu! Nelson says "ha-ha": -2:64:0:44:M*:.:Linux:2.0.3x (MkLinux) on Mac (1) -64:64:0:44:M*:.:Linux:2.0.3x (MkLinux) on Mac (2) - -S4:64:1:60:M1360,S,T,N,W0:.:Linux:2.4 (Google crawlbot) -S4:64:1:60:M1430,S,T,N,W0:.:Linux:2.4-2.6 (Google crawlbot) - -S2:64:1:60:M*,S,T,N,W0:.:Linux:2.4 (large MTU?) -S3:64:1:60:M*,S,T,N,W0:.:Linux:2.4 (newer) -S4:64:1:60:M*,S,T,N,W0:.:Linux:2.4-2.6 - -S3:64:1:60:M*,S,T,N,W1:.:Linux:2.6, seldom 2.4 (older, 1) -S4:64:1:60:M*,S,T,N,W1:.:Linux:2.6, seldom 2.4 (older, 2) -S3:64:1:60:M*,S,T,N,W2:.:Linux:2.6, seldom 2.4 (older, 3) -S4:64:1:60:M*,S,T,N,W2:.:Linux:2.6, seldom 2.4 (older, 4) -T4:64:1:60:M*,S,T,N,W2:.:Linux:2.6 (older, 5) - -S4:64:1:60:M*,S,T,N,W5:.:Linux:2.6 (newer, 1) -S4:64:1:60:M*,S,T,N,W6:.:Linux:2.6 (newer, 2) -S4:64:1:60:M*,S,T,N,W7:.:Linux:2.6 (newer, 3) -T4:64:1:60:M*,S,T,N,W7:.:Linux:2.6 (newer, 4) - - -S20:64:1:60:M*,S,T,N,W0:.:Linux:2.2 (1) -S22:64:1:60:M*,S,T,N,W0:.:Linux:2.2 (2) -S11:64:1:60:M*,S,T,N,W0:.:Linux:2.2 (3) - -# Popular cluster config scripts disable timestamps and -# selective ACK: - -S4:64:1:48:M1460,N,W0:.:Linux:2.4 in cluster - -# This happens only over loopback, but let's make folks happy: -32767:64:1:60:M16396,S,T,N,W0:.:Linux:2.4 (loopback) -32767:64:1:60:M16396,S,T,N,W2:.:Linux:2.6 (newer, loopback) -S8:64:1:60:M3884,S,T,N,W0:.:Linux:2.2 (loopback) - -# Opera visitors: -16384:64:1:60:M*,S,T,N,W0:.:-Linux:2.2 (Opera?) -32767:64:1:60:M*,S,T,N,W0:.:-Linux:2.4 (Opera?) - -# Some fairly common mods & oddities: -S22:64:1:52:M*,N,N,S,N,W0:.:Linux:2.2 (tstamp-) -S4:64:1:52:M*,N,N,S,N,W0:.:Linux:2.4 (tstamp-) -S4:64:1:52:M*,N,N,S,N,W2:.:Linux:2.6 (tstamp-) -S4:64:1:44:M*:.:Linux:2.6? (barebone, rare!) -T4:64:1:60:M1412,S,T,N,W0:.:Linux:2.4 (rare!) - -# ----------------- FreeBSD ----------------- - -16384:64:1:44:M*:.:FreeBSD:2.0-4.2 -16384:64:1:60:M*,N,W0,N,N,T:.:FreeBSD:4.4 (1) - -1024:64:1:60:M*,N,W0,N,N,T:.:FreeBSD:4.4 (2) - -57344:64:1:44:M*:.:FreeBSD:4.6-4.8 (RFC1323-) -57344:64:1:60:M*,N,W0,N,N,T:.:FreeBSD:4.6-4.9 - -32768:64:1:60:M*,N,W0,N,N,T:.:FreeBSD:4.8-5.1 (or MacOS X 10.2-10.3) -65535:64:1:60:M*,N,W0,N,N,T:.:FreeBSD:4.7-5.2 (or MacOS X 10.2-10.4) (1) -65535:64:1:60:M*,N,W1,N,N,T:.:FreeBSD:4.7-5.2 (or MacOS X 10.2-10.4) (2) - -65535:64:1:60:M*,N,W0,N,N,T:Z:FreeBSD:5.1 (1) -65535:64:1:60:M*,N,W1,N,N,T:Z:FreeBSD:5.1 (2) -65535:64:1:60:M*,N,W2,N,N,T:Z:FreeBSD:5.1 (3) -65535:64:1:64:M*,N,N,S,N,W1,N,N,T:.:FreeBSD:5.3-5.4 -65535:64:1:64:M*,N,W1,N,N,T,S,E:P:FreeBSD:6.x (1) -65535:64:1:64:M*,N,W0,N,N,T,S,E:P:FreeBSD:6.x (2) - -65535:64:1:44:M*:Z:FreeBSD:5.2 (RFC1323-) - -# 16384:64:1:60:M*,N,N,N,N,N,N,T:.:FreeBSD:4.4 (tstamp-) - -# ----------------- NetBSD ------------------ - -16384:64:0:60:M*,N,W0,N,N,T:.:NetBSD:1.3 -65535:64:0:60:M*,N,W0,N,N,T0:.:-NetBSD:1.6 (Opera) -16384:64:1:60:M*,N,W0,N,N,T0:.:NetBSD:1.6 -65535:64:1:60:M*,N,W1,N,N,T0:.:NetBSD:1.6W-current (DF) -65535:64:1:60:M*,N,W0,N,N,T0:.:NetBSD:1.6X (DF) -32768:64:1:60:M*,N,W0,N,N,T0:.:NetBSD:1.6Z or 2.0 (DF) -32768:64:1:64:M1416,N,W0,S,N,N,N,N,T0:.:NetBSD:2.0G (DF) -32768:64:1:64:M*,N,W0,S,N,N,N,N,T0:.:NetBSD:3.0 (DF) - -# ----------------- OpenBSD ----------------- - -16384:64:1:64:M*,N,N,S,N,W0,N,N,T:.:OpenBSD:3.0-3.9 -57344:64:1:64:M*,N,N,S,N,W0,N,N,T:.:OpenBSD:3.3-3.4 -16384:64:0:64:M*,N,N,S,N,W0,N,N,T:.:OpenBSD:3.0-3.4 (scrub) -65535:64:1:64:M*,N,N,S,N,W0,N,N,T:.:-OpenBSD:3.0-3.4 (Opera?) -32768:64:1:64:M*,N,N,S,N,W0,N,N,T:.:OpenBSD:3.7 - -# ----------------- Solaris ----------------- - -S17:64:1:64:N,W3,N,N,T0,N,N,S,M*:.:Solaris:8 (RFC1323 on) -S17:64:1:48:N,N,S,M*:.:Solaris:8 (1) -S17:255:1:44:M*:.:Solaris:2.5-7 (1) - -# Sometimes, just sometimes, Solaris feels like coming up with -# rather arbitrary MSS values ;-) - -S6:255:1:44:M*:.:Solaris:2.5-7 (2) -S23:64:1:48:N,N,S,M*:.:Solaris:8 (2) -S34:64:1:48:M*,N,N,S:.:Solaris:9 -S34:64:1:48:M*,N,N,N,N:.:Solaris:9 (no sack) -S44:255:1:44:M*:.:Solaris:7 - -4096:64:0:44:M1460:.:SunOS:4.1.x - -S34:64:1:52:M*,N,W0,N,N,S:.:Solaris:10 (beta) -32850:64:1:64:M*,N,N,T,N,W1,N,N,S:.:Solaris:10 (1203?) -32850:64:1:64:M*,N,W1,N,N,T,N,N,S:.:Solaris:9.1 - -# ----------------- IRIX -------------------- - -49152:60:0:44:M*:.:IRIX:6.2-6.4 -61440:60:0:44:M*:.:IRIX:6.2-6.5 -49152:60:0:52:M*,N,W2,N,N,S:.:IRIX:6.5 (RFC1323+) (1) -49152:60:0:52:M*,N,W3,N,N,S:.:IRIX:6.5 (RFC1323+) (2) - -61440:60:0:48:M*,N,N,S:.:IRIX:6.5.12-6.5.21 (1) -49152:60:0:48:M*,N,N,S:.:IRIX:6.5.12-6.5.21 (2) - -49152:60:0:64:M*,N,W2,N,N,T,N,N,S:.:IRIX:6.5 IP27 - -# ----------------- Tru64 ------------------- -# Tru64 and OpenVMS share the same stack on occassions. -# Relax. - -32768:60:1:48:M*,N,W0:.:Tru64:4.0 (or OS/2 Warp 4) -32768:60:0:48:M*,N,W0:.:Tru64:5.0 (or OpenVMS 7.x on Compaq 5.0 stack) -8192:60:0:44:M1460:.:Tru64:5.1 (no RFC1323) (or QNX 6) -61440:60:0:48:M*,N,W0:.:Tru64:v5.1a JP4 (or OpenVMS 7.x on Compaq 5.x stack) - -# ----------------- OpenVMS ----------------- - -6144:64:1:60:M*,N,W0,N,N,T:.:OpenVMS:7.2 (Multinet 4.3-4.4 stack) - -# ----------------- MacOS ------------------- - -S2:255:1:48:M*,W0,E:.:MacOS:8.6 classic - -16616:255:1:48:M*,W0,E:.:MacOS:7.3-8.6 (OTTCP) -16616:255:1:48:M*,N,N,N,E:.:MacOS:8.1-8.6 (OTTCP) -32768:255:1:48:M*,W0,N:.:MacOS:9.0-9.2 - -32768:255:1:48:M1380,N,N,N,N:.:MacOS:9.1 (OT 2.7.4) (1) -65535:255:1:48:M*,N,N,N,N:.:MacOS:9.1 (OT 2.7.4) (2) - -# ----------------- Windows ----------------- - -# Windows TCP/IP stack is a mess. For most recent XP, 2000 and -# even 98, the pathlevel, not the actual OS version, is more -# relevant to the signature. They share the same code, so it would -# seem. Luckily for us, almost all Windows 9x boxes have an -# awkward MSS of 536, which I use to tell one from another -# in most difficult cases. - -8192:32:1:44:M*:.:Windows:3.11 (Tucows) -S44:64:1:64:M*,N,W0,N,N,T0,N,N,S:.:Windows:95 -8192:128:1:64:M*,N,W0,N,N,T0,N,N,S:.:Windows:95b - -# There were so many tweaking tools and so many stack versions for -# Windows 98 it is no longer possible to tell them from each other -# without some very serious research. Until then, there's an insane -# number of signatures, for your amusement: - -S44:32:1:48:M*,N,N,S:.:Windows:98 (low TTL) (1) -8192:32:1:48:M*,N,N,S:.:Windows:98 (low TTL) (2) -%8192:64:1:48:M536,N,N,S:.:Windows:98 (13) -%8192:128:1:48:M536,N,N,S:.:Windows:98 (15) -S4:64:1:48:M*,N,N,S:.:Windows:98 (1) -S6:64:1:48:M*,N,N,S:.:Windows:98 (2) -S12:64:1:48:M*,N,N,S:.:Windows:98 (3 -T30:64:1:64:M1460,N,W0,N,N,T0,N,N,S:.:Windows:98 (16) -32767:64:1:48:M*,N,N,S:.:Windows:98 (4) -37300:64:1:48:M*,N,N,S:.:Windows:98 (5) -46080:64:1:52:M*,N,W3,N,N,S:.:Windows:98 (RFC1323+) -65535:64:1:44:M*:.:Windows:98 (no sack) -S16:128:1:48:M*,N,N,S:.:Windows:98 (6) -S16:128:1:64:M*,N,W0,N,N,T0,N,N,S:.:Windows:98 (7) -S26:128:1:48:M*,N,N,S:.:Windows:98 (8) -T30:128:1:48:M*,N,N,S:.:Windows:98 (9) -32767:128:1:52:M*,N,W0,N,N,S:.:Windows:98 (10) -60352:128:1:48:M*,N,N,S:.:Windows:98 (11) -60352:128:1:64:M*,N,W2,N,N,T0,N,N,S:.:Windows:98 (12) - -# What's with 1414 on NT? -T31:128:1:44:M1414:.:Windows:NT 4.0 SP6a (1) -64512:128:1:44:M1414:.:Windows:NT 4.0 SP6a (2) -8192:128:1:44:M*:.:Windows:NT 4.0 (older) - -# Windows XP and 2000. Most of the signatures that were -# either dubious or non-specific (no service pack data) -# were deleted and replaced with generics at the end. - -65535:128:1:48:M*,N,N,S:.:Windows:2000 SP4, XP SP1+ -%8192:128:1:48:M*,N,N,S:.:Windows:2000 SP2+, XP SP1+ (seldom 98) -S20:128:1:48:M*,N,N,S:.:Windows:SP3 -S45:128:1:48:M*,N,N,S:.:Windows:2000 SP4, XP SP1+ (2) -40320:128:1:48:M*,N,N,S:.:Windows:2000 SP4 - -S6:128:1:48:M*,N,N,S:.:Windows:XP, 2000 SP2+ -S12:128:1:48:M*,N,N,S:.:Windows:XP SP1+ (1) -S44:128:1:48:M*,N,N,S:.:Windows:XP SP1+, 2000 SP3 -64512:128:1:48:M*,N,N,S:.:Windows:XP SP1+, 2000 SP3 (2) -32767:128:1:48:M*,N,N,S:.:Windows:XP SP1+, 2000 SP4 (3) - -# Windows 2003 & Vista - -8192:128:1:52:M*,W8,N,N,N,S:.:Windows:Vista (beta) -32768:32:1:52:M1460,N,W0,N,N,S:.:Windows:2003 AS -65535:64:1:52:M1460,N,W2,N,N,S:.:Windows:2003 (1) -65535:64:1:48:M1460,N,N,S:.:Windows:2003 (2) - -# Odds, ends, mods: - -S52:128:1:48:M1260,N,N,S:.:Windows:XP/2000 via Cisco -65520:128:1:48:M*,N,N,S:.:Windows:XP bare-bone -16384:128:1:52:M536,N,W0,N,N,S:.:Windows:2000 w/ZoneAlarm? -2048:255:0:40:.:.:Windows:.NET Enterprise Server -44620:64:0:48:M*,N,N,S:.:Windows:ME no SP (?) -S6:255:1:48:M536,N,N,S:.:Windows:95 winsock 2 -32000:128:0:48:M*,N,N,S:.:Windows:XP w/Winroute? -16384:64:1:48:M1452,N,N,S:.:Windows:XP w/Sygate? (1) -17256:64:1:48:M1460,N,N,S:.:Windows:XP w/Sygate? (2) - -# No need to be more specific, it passes: -*:128:1:48:M*,N,N,S:U:-Windows:XP/2000 while downloading (leak!) - -# ----------------- HP/UX ------------------- - -32768:64:1:44:M*:.:HP-UX:B.10.20 -32768:64:1:48:M*,W0,N:.:HP-UX:11.00-11.11 - -# Whoa. Hardcore WSS. -0:64:0:48:M*,W0,N:.:HP-UX:B.11.00 A (RFC1323+) - -# ----------------- RiscOS ------------------ - -16384:64:1:68:M1460,N,W0,N,N,T,N,N,?12:.:RISC OS:3.70-4.36 (inet 5.04) -12288:32:0:44:M536:.:RISC OS:3.70 inet 4.10 -4096:64:1:56:M1460,N,N,T:T:RISC OS:3.70 freenet 2.00 - -# ----------------- BSD/OS ------------------ - -8192:64:1:60:M1460,N,W0,N,N,T:.:BSD/OS:3.1-4.3 (or MacOS X 10.2) - -# ---------------- NetwonOS ----------------- - -4096:64:0:44:M1420:.:NewtonOS:2.1 - -# ---------------- NeXTSTEP ----------------- - -S8:64:0:44:M512:.:NeXTSTEP:3.3 (1) -S4:64:0:44:M1024:.:NeXTSTEP:3.3 (2) - -# ------------------ BeOS ------------------- - -1024:255:0:48:M*,N,W0:.:BeOS:5.0-5.1 -12288:255:0:44:M*:.:BeOS:5.0.x - -# ------------------ OS/400 ----------------- - -8192:64:1:60:M1440,N,W0,N,N,T:.:OS/400:V4R4/R5 -8192:64:0:44:M536:.:OS/400:V4R3/M0 -4096:64:1:60:M1440,N,W0,N,N,T:.:OS/400:V4R5 + CF67032 - -28672:64:0:44:M1460:A:OS/390:? - -# ------------------ ULTRIX ----------------- - -16384:64:0:40:.:.:ULTRIX:4.5 - -# ------------------- QNX ------------------- - -S16:64:0:44:M512:.:QNX:demodisk -16384:64:0:60:M1460,N,W0,N,N,T0:.:QNX:6.x - -# ------------------ Novell ----------------- - -16384:128:1:44:M1460:.:Novell:NetWare 5.0 -6144:128:1:44:M1460:.:Novell:IntranetWare 4.11 -6144:128:1:44:M1368:.:Novell:BorderManager ? - -# According to rfp: -6144:128:1:52:M*,W0,N,S,N,N:.:Novell:Netware 6 SP3 - -# -------------- SCO UnixWare --------------- - -S3:64:1:60:M1460,N,W0,N,N,T:.:SCO:UnixWare 7.1 -S17:64:1:60:M*,N,W0,N,N,T:.:SCO:UnixWare 7.1.x -S23:64:1:44:M1380:.:SCO:OpenServer 5.0 - -# ------------------- DOS ------------------- - -2048:255:0:44:M536:.:DOS:Arachne via WATTCP/1.05 -T2:255:0:44:M984:.:DOS:Arachne via WATTCP/1.05 (eepro) -16383:64:0:44:M536:.:DOS:Unknown via WATTCP (epppd) - -# ------------------ OS/2 ------------------- - -S56:64:0:44:M512:.:OS/2:4 -28672:64:0:44:M1460:.:OS/2:Warp 4.0 - -# ----------------- TOPS-20 ----------------- - -# Another hardcore MSS, one of the ACK leakers hunted down. -0:64:0:44:M1460:A:TOPS-20:version 7 - -# ------------------ AMIGA ------------------ - -S32:64:1:56:M*,N,N,S,N,N,?12:.:AMIGA:3.9 BB2 with Miami stack - -# ------------------ Minix ------------------ - -# Not quite sure. -# 8192:210:0:44:M1460:X:@Minix:? - -# ------------------ Plan9 ------------------ - -65535:255:0:48:M1460,W0,N:.:Plan9:edition 4 - -# ----------------- AMIGAOS ----------------- - -16384:64:1:48:M1560,N,N,S:.:AMIGAOS:3.9 BB2 MiamiDX - -# ----------------- FreeMiNT ---------------- - -S44:255:0:44:M536:.:FreeMiNT:1 patch 16A (Atari) - -########################################### -# Appliance / embedded / other signatures # -########################################### - -# ---------- Firewalls / routers ------------ - -S12:64:1:44:M1460:.:@Checkpoint:(unknown 1) -S12:64:1:48:N,N,S,M1460:.:@Checkpoint:(unknown 2) -4096:32:0:44:M1460:.:ExtremeWare:4.x - -S32:64:0:68:M512,N,W0,N,N,T,N,N,?12:.:Nokia:IPSO w/Checkpoint NG FP3 -S16:64:0:68:M1024,N,W0,N,N,T,N,N,?12:.:Nokia:IPSO 3.7 build 026 - -S4:64:1:60:W0,N,S,T,M1460:.:FortiNet:FortiGate 50 - -8192:64:1:44:M1460:.:@Eagle:Secure Gateway - -# ------- Switches and other stuff ---------- - -4128:255:0:44:M*:Z:Cisco:7200, Catalyst 3500, etc -S8:255:0:44:M*:.:Cisco:12008 -S4:255:0:44:M536:Z:Cisco:IOS 11.0 -60352:128:1:64:M1460,N,W2,N,N,T,N,N,S:.:Alteon:ACEswitch -64512:128:1:44:M1370:.:Nortel:Contivity Client - -# ---------- Caches and whatnots ------------ - -8190:255:0:44:M1428:.:Google:Wireless Transcoder (1) -8190:255:0:44:M1460:.:Google:Wireless Transcoder (2) -8192:64:1:64:M1460,N,N,S,N,W0,N,N,T:.:NetCache:5.2 -16384:64:1:64:M1460,N,N,S,N,W0,N:.:NetCache:5.3 -65535:64:1:64:M1460,N,N,S,N,W*,N,N,T:.:NetCache:5.3-5.5 (or FreeBSD 5.4) -20480:64:1:64:M1460,N,N,S,N,W0,N,N,T:.:NetCache:4.1 -S44:64:1:64:M1460,N,N,S,N,W0,N,N,T:.:NetCache:5.5 - -32850:64:1:64:N,W1,N,N,T,N,N,S,M*:.:NetCache:Data OnTap 5.x - -65535:64:0:60:M1460,N,W0,N,N,T:.:CacheFlow:CacheOS 4.1 -8192:64:0:60:M1380,N,N,N,N,N,N,T:.:CacheFlow:CacheOS 1.1 - -S4:64:0:48:M1460,N,N,S:.:Cisco:Content Engine - -27085:128:0:40:.:.:Dell:PowerApp cache (Linux-based) - -65535:255:1:48:N,W1,M1460:.:Inktomi:crawler -S1:255:1:60:M1460,S,T,N,W0:.:LookSmart:ZyBorg - -16384:255:0:40:.:.:Proxyblocker:(what's this?) - -65535:255:0:48:M*,N,N,S:.:Redline: T|X 2200 - -# ----------- Embedded systems -------------- - -S9:255:0:44:M536:.:PalmOS:Tungsten T3/C -S5:255:0:44:M536:.:PalmOS:3/4 -S4:255:0:44:M536:.:PalmOS:3.5 -2948:255:0:44:M536:.:PalmOS:3.5.3 (Handera) -S29:255:0:44:M536:.:PalmOS:5.0 -16384:255:0:44:M1398:.:PalmOS:5.2 (Clie) -S14:255:0:44:M1350:.:PalmOS:5.2.1 (Treo) -16384:255:0:44:M1400:.:PalmOS:5.2 (Sony) - -S23:64:1:64:N,W1,N,N,T,N,N,S,M1460:.:SymbianOS:7 -8192:255:0:44:M1460:.:SymbianOS:6048 (Nokia 7650?) -8192:255:0:44:M536:.:SymbianOS:(Nokia 9210?) -S22:64:1:56:M1460,T,S:.:SymbianOS:? (SE P800?) -S36:64:1:56:M1360,T,S:.:SymbianOS:60xx (Nokia 6600?) -S36:64:1:60:M1360,T,S,W0,E:.:SymbianOS:60xx - -32768:32:1:44:M1460:.:Windows:CE 3 - -# Perhaps S4? -5840:64:1:60:M1452,S,T,N,W1:.:Zaurus:3.10 - -32768:128:1:64:M1460,N,W0,N,N,T0,N,N,S:.:PocketPC:2002 - -S1:255:0:44:M346:.:Contiki:1.1-rc0 - -4096:128:0:44:M1460:.:Sega:Dreamcast Dreamkey 3.0 -T5:64:0:44:M536:.:Sega:Dreamcast HKT-3020 (browser disc 51027) -S22:64:1:44:M1460:.:Sony:Playstation 2 (SOCOM?) - -S12:64:0:44:M1452:.:AXIS:Printer Server 5600 v5.64 - -3100:32:1:44:M1460:.:Windows:CE 2.0 - -#################### -# Fancy signatures # -#################### - -1024:64:0:40:.:.:-*NMAP:syn scan (1) -2048:64:0:40:.:.:-*NMAP:syn scan (2) -3072:64:0:40:.:.:-*NMAP:syn scan (3) -4096:64:0:40:.:.:-*NMAP:syn scan (4) - -1024:64:0:40:.:A:-*NMAP:TCP sweep probe (1) -2048:64:0:40:.:A:-*NMAP:TCP sweep probe (2) -3072:64:0:40:.:A:-*NMAP:TCP sweep probe (3) -4096:64:0:40:.:A:-*NMAP:TCP sweep probe (4) - -1024:64:0:60:W10,N,M265,T,E:P:-*NMAP:OS detection probe (1) -2048:64:0:60:W10,N,M265,T,E:P:-*NMAP:OS detection probe (2) -3072:64:0:60:W10,N,M265,T,E:P:-*NMAP:OS detection probe (3) -4096:64:0:60:W10,N,M265,T,E:P:-*NMAP:OS detection probe (4) - -1024:64:0:60:W10,N,M265,T,E:PF:-*NMAP:OS detection probe w/flags (1) -2048:64:0:60:W10,N,M265,T,E:PF:-*NMAP:OS detection probe w/flags (2) -3072:64:0:60:W10,N,M265,T,E:PF:-*NMAP:OS detection probe w/flags (3) -4096:64:0:60:W10,N,M265,T,E:PF:-*NMAP:OS detection probe w/flags (4) - -32767:64:0:40:.:.:-*NAST:syn scan - -12345:255:0:40:.:A:-p0f:sendsyn utility - -# UFO - see tmp/*: -56922:128:0:40:.:A:-@Mysterious:port scanner (?) -5792:64:1:60:M1460,S,T,N,W0:T:-@Mysterious:NAT device (2nd tstamp) -S12:128:1:48:M1460,E:P:@Mysterious:Chello proxy (?) -S23:64:1:64:N,W1,N,N,T,N,N,S,M1380:.:@Mysterious:GPRS gateway (?) - -##################################### -# Generic signatures - just in case # -##################################### - -*:128:1:52:M*,N,W0,N,N,S:.:@Windows:XP/2000 (RFC1323+, w, tstamp-) -*:128:1:52:M*,N,W*,N,N,S:.:@Windows:XP/2000 (RFC1323+, w+, tstamp-) -*:128:1:52:M*,N,N,T0,N,N,S:.:@Windows:XP/2000 (RFC1323+, w-, tstamp+) -*:128:1:64:M*,N,W0,N,N,T0,N,N,S:.:@Windows:XP/2000 (RFC1323+, w, tstamp+) -*:128:1:64:M*,N,W*,N,N,T0,N,N,S:.:@Windows:XP/2000 (RFC1323+, w+, tstamp+) - -*:128:1:48:M536,N,N,S:.:@Windows:98 -*:128:1:48:M*,N,N,S:.:@Windows:XP/2000 - - diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f014d0a149..cf0b7201c1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -285,7 +285,6 @@ set(bro_SRCS NetVar.cc Obj.cc OpaqueVal.cc - OSFinger.cc PacketFilter.cc Pipe.cc PolicyFile.cc diff --git a/src/NetVar.cc b/src/NetVar.cc index b9230bece7..16032a43d2 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -140,9 +140,6 @@ RecordType* backdoor_endp_stats; RecordType* software; RecordType* software_version; -RecordType* OS_version; -EnumType* OS_version_inference; -TableVal* generate_OS_version_event; double table_expire_interval; double table_expire_delay; @@ -425,10 +422,6 @@ void init_net_var() software = internal_type("software")->AsRecordType(); software_version = internal_type("software_version")->AsRecordType(); - OS_version = internal_type("OS_version")->AsRecordType(); - OS_version_inference = internal_type("OS_version_inference")->AsEnumType(); - generate_OS_version_event = - opt_internal_table("generate_OS_version_event"); packet_type = internal_type("packet")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index 9fa4d75fa6..ec6e7cab69 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -143,9 +143,6 @@ extern RecordType* backdoor_endp_stats; extern RecordType* software; extern RecordType* software_version; -extern RecordType* OS_version; -extern EnumType* OS_version_inference; -extern TableVal* generate_OS_version_event; extern double table_expire_interval; extern double table_expire_delay; diff --git a/src/OSFinger.cc b/src/OSFinger.cc deleted file mode 100644 index 1b540a1fd0..0000000000 --- a/src/OSFinger.cc +++ /dev/null @@ -1,689 +0,0 @@ -/* - Taken with permission from: - - p0f - passive OS fingerprinting (GNU LESSER GENERAL PUBLIC LICENSE) - ------------------------------------------------------------------- - - "If you sit down at a poker game and don't see a sucker, - get up. You're the sucker." - - (C) Copyright 2000-2003 by Michal Zalewski -*/ - -// To make it easier to upgrade this file to newer releases of p0f, -// it remains in the coding style used by p0f rather than Bro. - -#include "OSFinger.h" -#include "net_util.h" -#include "util.h" -#include "Var.h" -#include -#include -#include - - -void int_delete_func(void* v) - { - delete (int*) v; - } - - -// Initializes data structures for fingerprinting in the given mode. -OSFingerprint::OSFingerprint(FingerprintMode arg_mode) - { - err = 0; - mode = arg_mode; - - sigcnt=gencnt=0; - problems=0; - char* fname; - - memset(sig, 0, sizeof(struct fp_entry)*MAXSIGS); - memset(bh, 0, sizeof(struct fp_entry*)*OSHSIZE); - - os_matches.SetDeleteFunc(int_delete_func); - - if (mode == SYN_FINGERPRINT_MODE) - { - fname = copy_string(internal_val("passive_fingerprint_file")->AsString()->CheckString()); - load_config(fname); - delete [] fname; - } - else if (mode == SYN_ACK_FINGERPRINT_MODE) - {//not yet supported - load_config("p0fsynack.sig"); - } - else if (mode == RST_FINGERPRINT_MODE) - {//not yet supported - load_config("p0frst.sig"); - } - else - { - Error("OS fingerprinting: unknown mode!"); - } -} - -bool OSFingerprint::CacheMatch(const IPAddr& addr, int id) - { - HashKey* key = addr.GetHashKey(); - int* pid = new int; - *pid=id; - int* prev = os_matches.Insert(key, pid); - bool ret = (prev ? *prev != id : 1); - if (prev) - delete prev; - delete key; - return ret; - } - - -// Determines whether the signature file had any collisions. -void OSFingerprint::collide(uint32 id) - { - uint32 i,j; - uint32 cur; - - if (sig[id].ttl % 32 && sig[id].ttl != 255 && sig[id].ttl % 30) - { - problems=1; - reporter->Warning("OS fingerprinting: [!] Unusual TTL (%d) for signature '%s %s' (line %d).", - sig[id].ttl,sig[id].os,sig[id].desc,sig[id].line); - } - - for (i=0;iWarning("OS fingerprinting: [!] Duplicate signature name: '%s %s' (line %d and %d).", - sig[i].os,sig[i].desc,sig[i].line,sig[id].line); - } - - /* If TTLs are sufficiently away from each other, the risk of - a collision is lower. */ - if (abs((int)sig[id].ttl - (int)sig[i].ttl) > 25) continue; - - if (sig[id].df ^ sig[i].df) continue; - if (sig[id].zero_stamp ^ sig[i].zero_stamp) continue; - - /* Zero means >= PACKET_BIG */ - if (sig[id].size) { if (sig[id].size ^ sig[i].size) continue; } - else if (sig[i].size < PACKET_BIG) continue; - - if (sig[id].optcnt ^ sig[i].optcnt) continue; - if (sig[id].quirks ^ sig[i].quirks) continue; - - switch (sig[id].wsize_mod) { - - case 0: /* Current: const */ - - cur=sig[id].wsize; - -do_const: - - switch (sig[i].wsize_mod) { - - case 0: /* Previous is also const */ - - /* A problem if values match */ - if (cur ^ sig[i].wsize) continue; - break; - - case MOD_CONST: /* Current: const, prev: modulo (or *) */ - - /* A problem if current value is a multiple of that modulo */ - if (cur % sig[i].wsize) continue; - break; - - case MOD_MSS: /* Current: const, prev: mod MSS */ - - if (sig[i].mss_mod || sig[i].wsize * - (sig[i].mss ? sig[i].mss : 1460 ) != (int) cur) - continue; - - break; - - case MOD_MTU: /* Current: const, prev: mod MTU */ - - if (sig[i].mss_mod || sig[i].wsize * ( - (sig[i].mss ? sig[i].mss : 1460 )+40) != (int) cur) - continue; - - break; - - } - - break; - - case 1: /* Current signature is modulo something */ - - /* A problem only if this modulo is a multiple of the - previous modulo */ - - if (sig[i].wsize_mod != MOD_CONST) continue; - if (sig[id].wsize % sig[i].wsize) continue; - - break; - - case MOD_MSS: /* Current is modulo MSS */ - - /* There's likely a problem only if the previous one is close - to '*'; we do not check known MTUs, because this particular - signature can be made with some uncommon MTUs in mind. The - problem would also appear if current signature has a fixed - MSS. */ - - if (sig[i].wsize_mod != MOD_CONST || sig[i].wsize >= 8) { - if (!sig[id].mss_mod) { - cur = (sig[id].mss ? sig[id].mss : 1460 ) * sig[id].wsize; - goto do_const; - } - continue; - } - - break; - - case MOD_MTU: /* Current is modulo MTU */ - - if (sig[i].wsize_mod != MOD_CONST || sig[i].wsize <= 8) { - if (!sig[id].mss_mod) { - cur = ( (sig[id].mss ? sig[id].mss : 1460 ) +40) * sig[id].wsize; - goto do_const; - } - continue; - } - - break; - - } - - /* Same for wsc */ - switch (sig[id].wsc_mod) { - - case 0: /* Current: const */ - - cur=sig[id].wsc; - - switch (sig[i].wsc_mod) { - - case 0: /* Previous is also const */ - - /* A problem if values match */ - if (cur ^ sig[i].wsc) continue; - break; - - case 1: /* Current: const, prev: modulo (or *) */ - - /* A problem if current value is a multiple of that modulo */ - if (cur % sig[i].wsc) continue; - break; - - } - - break; - - case MOD_CONST: /* Current signature is modulo something */ - - /* A problem only if this modulo is a multiple of the - previous modulo */ - - if (!sig[i].wsc_mod) continue; - if (sig[id].wsc % sig[i].wsc) continue; - - break; - - } - - /* Same for mss */ - switch (sig[id].mss_mod) { - - case 0: /* Current: const */ - - cur=sig[id].mss; - - switch (sig[i].mss_mod) { - - case 0: /* Previous is also const */ - - /* A problem if values match */ - if (cur ^ sig[i].mss) continue; - break; - - case 1: /* Current: const, prev: modulo (or *) */ - - /* A problem if current value is a multiple of that modulo */ - if (cur % sig[i].mss) continue; - break; - - } - - break; - - case MOD_CONST: /* Current signature is modulo something */ - - /* A problem only if this modulo is a multiple of the - previous modulo */ - - if (!sig[i].mss_mod) continue; - if ((sig[id].mss ? sig[id].mss : 1460 ) % - (sig[i].mss ? sig[i].mss : 1460 )) continue; - - break; - - } - - /* Now check option sequence */ - - for (j=0;jWarning("OS fingerprinting: [!] Signature '%s %s' (line %d)\n" - " is already covered by '%s %s' (line %d).", - sig[id].os,sig[id].desc,sig[id].line,sig[i].os,sig[i].desc, - sig[i].line); - -reloop: - ; - } - } - -// Loads a given file into to classes data structures. -void OSFingerprint::load_config(const char* file) - { - uint32 ln=0; - char buf[MAXLINE]; - char* p; - - FILE* c = open_file(find_file(file, bro_path(), ".osf")); - - if (!c) - { - Error("Can't open OS passive fingerprinting signature file", file); - return; - } - sigcnt=0; //every time we read config we reset it to 0; - while ((p=fgets(buf,sizeof(buf),c))) - { - uint32 l; - - char obuf[MAXLINE],genre[MAXLINE],desc[MAXLINE],quirks[MAXLINE]; - char w[MAXLINE],sb[MAXLINE]; - char* gptr = genre; - uint32 t,d,s; - struct fp_entry* e; - - ln++; - - /* Remove leading and trailing blanks */ - while (isspace(*p)) p++; - l=strlen(p); - while (l && isspace(*(p+l-1))) *(p+(l--)-1)=0; - - /* Skip empty lines and comments */ - if (!l) continue; - if (*p == '#') continue; - - if (sscanf(p,"%[0-9%*()ST]:%d:%d:%[0-9()*]:%[^:]:%[^ :]:%[^:]:%[^:]", - w, &t,&d,sb, obuf, quirks,genre,desc) != 8) - Error("OS fingerprinting: Syntax error in p0f signature config line %d.\n",(uint32)ln); - - gptr = genre; - - if (*sb != '*') s = atoi(sb); else s = 0; - -reparse_ptr: - - switch (*gptr) - { - case '-': sig[sigcnt].userland = 1; gptr++; goto reparse_ptr; - case '*': sig[sigcnt].no_detail = 1; gptr++; goto reparse_ptr; - case '@': sig[sigcnt].generic = 1; gptr++; gencnt++; goto reparse_ptr; - case 0: Error("OS fingerprinting: Empty OS genre in line",(uint32)ln); - } - - sig[sigcnt].os = strdup(gptr); - sig[sigcnt].desc = strdup(desc); - sig[sigcnt].ttl = t; - sig[sigcnt].size = s; - sig[sigcnt].df = d; - - if (w[0] == '*') - { - sig[sigcnt].wsize = 1; - sig[sigcnt].wsize_mod = MOD_CONST; - } - else if (tolower(w[0]) == 's') - { - sig[sigcnt].wsize_mod = MOD_MSS; - if (!isdigit(*(w+1))) - Error("OS fingerprinting: Bad Snn value in WSS in line",(uint32)ln); - sig[sigcnt].wsize = atoi(w+1); - } - else if (tolower(w[0]) == 't') - { - sig[sigcnt].wsize_mod = MOD_MTU; - if (!isdigit(*(w+1))) - Error("OS fingerprinting: Bad Tnn value in WSS in line",(uint32)ln); - sig[sigcnt].wsize = atoi(w+1); - } - else if (w[0] == '%') - { - if (!(sig[sigcnt].wsize = atoi(w+1))) - Error("OS fingerprinting: Null modulo for window size in config line",(uint32)ln); - sig[sigcnt].wsize_mod = MOD_CONST; - } - else - sig[sigcnt].wsize = atoi(w); - - /* Now let's parse options */ - - p=obuf; - - sig[sigcnt].zero_stamp = 1; - - if (*p=='.') p++; - - while (*p) - { - uint8 optcnt = sig[sigcnt].optcnt; - switch (tolower(*p)) - { - case 'n': sig[sigcnt].opt[optcnt] = TCPOPT_NOP; - break; - - case 'e': sig[sigcnt].opt[optcnt] = TCPOPT_EOL; - if (*(p+1)) - Error("OS fingerprinting: EOL not the last option, line",(uint32)ln); - break; - - case 's': sig[sigcnt].opt[optcnt] = TCPOPT_SACK_PERMITTED; - break; - - case 't': sig[sigcnt].opt[optcnt] = TCPOPT_TIMESTAMP; - if (*(p+1)!='0') - { - sig[sigcnt].zero_stamp=0; - if (isdigit(*(p+1))) - Error("OS fingerprinting: Bogus Tstamp specification in line",(uint32)ln); - } - break; - - case 'w': sig[sigcnt].opt[optcnt] = TCPOPT_WINDOW; - if (p[1] == '*') - { - sig[sigcnt].wsc = 1; - sig[sigcnt].wsc_mod = MOD_CONST; - } - else if (p[1] == '%') - { - if (!(sig[sigcnt].wsc = atoi(p+2))) - Error("OS fingerprinting: Null modulo for wscale in config line",(uint32)ln); - sig[sigcnt].wsc_mod = MOD_CONST; - } - else if (!isdigit(*(p+1))) - Error("OS fingerprinting: Incorrect W value in line",(uint32)ln); - else sig[sigcnt].wsc = atoi(p+1); - break; - - case 'm': sig[sigcnt].opt[optcnt] = TCPOPT_MAXSEG; - if (p[1] == '*') - { - sig[sigcnt].mss = 1; - sig[sigcnt].mss_mod = MOD_CONST; - } - else if (p[1] == '%') - { - if (!(sig[sigcnt].mss = atoi(p+2))) - Error("OS fingerprinting: Null modulo for MSS in config line",(uint32)ln); - sig[sigcnt].mss_mod = MOD_CONST; - } - else if (!isdigit(*(p+1))) - Error("OS fingerprinting: Incorrect M value in line",(uint32)ln); - else sig[sigcnt].mss = atoi(p+1); - break; - - /* Yuck! */ - case '?': if (!isdigit(*(p+1))) - Error("OS fingerprinting: Bogus ?nn value in line",(uint32)ln); - else sig[sigcnt].opt[optcnt] = atoi(p+1); - break; - - default: Error("OS fingerprinting: Unknown TCP option in config line",(uint32)ln); - } - - if (++sig[sigcnt].optcnt >= MAXOPT) - Error("OS fingerprinting: Too many TCP options specified in config line",(uint32)ln); - - /* Skip separators */ - do { p++; } while (*p && !isalpha(*p) && *p != '?'); - - } - - sig[sigcnt].line = ln; - - p = quirks; - - while (*p) - switch (toupper(*(p++))) - { - case 'E': - Error("OS fingerprinting: Quirk 'E' is obsolete. Remove it, append E to the options. Line",(uint32)ln); - break; - - case 'K': - if ( mode != RST_FINGERPRINT_MODE ) - Error("OS fingerprinting: Quirk 'K' is valid only in RST+ (-R) mode (wrong config file?). Line",(uint32)ln); - sig[sigcnt].quirks |= QUIRK_RSTACK; - break; - - case 'Q': sig[sigcnt].quirks |= QUIRK_SEQEQ; break; - case '0': sig[sigcnt].quirks |= QUIRK_SEQ0; break; - case 'P': sig[sigcnt].quirks |= QUIRK_PAST; break; - case 'Z': sig[sigcnt].quirks |= QUIRK_ZEROID; break; - case 'I': sig[sigcnt].quirks |= QUIRK_IPOPT; break; - case 'U': sig[sigcnt].quirks |= QUIRK_URG; break; - case 'X': sig[sigcnt].quirks |= QUIRK_X2; break; - case 'A': sig[sigcnt].quirks |= QUIRK_ACK; break; - case 'T': sig[sigcnt].quirks |= QUIRK_T2; break; - case 'F': sig[sigcnt].quirks |= QUIRK_FLAGS; break; - case 'D': sig[sigcnt].quirks |= QUIRK_DATA; break; - case '!': sig[sigcnt].quirks |= QUIRK_BROKEN; break; - case '.': break; - default: Error("OS fingerprinting: Bad quirk in line",(uint32)ln); - } - - e = bh[SIGHASH(s,sig[sigcnt].optcnt,sig[sigcnt].quirks,d)]; - - if (!e) - { - bh[SIGHASH(s,sig[sigcnt].optcnt,sig[sigcnt].quirks,d)] = &sig[sigcnt]; - } - else - { - while (e->next) e = e->next; - e->next = &sig[sigcnt]; - } - - collide(sigcnt); - if (++sigcnt >= MAXSIGS) - Error("OS fingerprinting: Maximum signature count exceeded.\n"); - - } - - fclose(c); - - if (!sigcnt) - Error("OS fingerprinting: no signatures loaded from config file."); - - } - -// Does the actual match between the packet and the signature database. -// Modifies retval and contains OS Type and other useful information. -// Returns config-file line of the matching signature as id. -int OSFingerprint::FindMatch(struct os_type* retval, uint16 tot,uint8 df, - uint8 ttl,uint16 wss,uint8 ocnt,uint8* op, - uint16 mss,uint8 wsc,uint32 tstamp, - uint32 quirks,uint8 ecn) const - { - uint32 j; //used for counter in loops - struct fp_entry* p; - uint8 orig_df = df; - - struct fp_entry* fuzzy = 0; - uint8 fuzzy_now = 0; - int id = 0; //return value: 0 indicates no match. - - retval->os="UNKNOWN"; - retval->desc=NULL; - retval->gadgets=0; - retval->match=0; - retval->uptime=0; - -re_lookup: - - p = bh[SIGHASH(tot,ocnt,quirks,df)]; - - while (p) - { - /* Cheap and specific checks first... */ - /* psize set to zero means >= PACKET_BIG */ - if (p->size) { if (tot ^ p->size) { p = p->next; continue; } } - else if (tot < PACKET_BIG) { p = p->next; continue; } - - if (ocnt ^ p->optcnt) { p = p->next; continue; } - - if (p->zero_stamp ^ (!tstamp)) { p = p->next; continue; } - if (p->df ^ df) { p = p->next; continue; } - if (p->quirks ^ quirks) { p = p->next; continue; } - - /* Check MSS and WSCALE... */ - if (!p->mss_mod) { - if (mss ^ p->mss) { p = p->next; continue; } - } else if (mss % p->mss) { p = p->next; continue; } - - if (!p->wsc_mod) { - if (wsc ^ p->wsc) { p = p->next; continue; } - } else if (wsc % p->wsc) { p = p->next; continue; } - - /* Then proceed with the most complex WSS check... */ - switch (p->wsize_mod) - { - case 0: - if (wss ^ p->wsize) { p = p->next; continue; } - break; - case MOD_CONST: - if (wss % p->wsize) { p = p->next; continue; } - break; - case MOD_MSS: - if (mss && !(wss % mss)) - { - if ((wss / mss) ^ p->wsize) { p = p->next; continue; } - } - else if (!(wss % 1460)) - { - if ((wss / 1460) ^ p->wsize) { p = p->next; continue; } - } - else { p = p->next; continue; } - break; - case MOD_MTU: - if (mss && !(wss % (mss+40))) - { - if ((wss / (mss+40)) ^ p->wsize) { p = p->next; continue; } - } - else if (!(wss % 1500)) - { - if ((wss / 1500) ^ p->wsize) { p = p->next; continue; } - } - else { p = p->next; continue; } - break; - } - - /* Numbers agree. Let's check options */ - for (j=0;jopt[j] ^ op[j]) goto continue_search; - - /* Check TTLs last because we might want to go fuzzy. */ - if (p->ttl < ttl) - { - if ( mode != RST_FINGERPRINT_MODE )fuzzy = p; - p = p->next; - continue; - } - - /* Naah... can't happen ;-) */ - if (!p->no_detail) - if (p->ttl - ttl > MAXDIST) - { - if (mode != RST_FINGERPRINT_MODE ) fuzzy = p; - p = p->next; - continue; - } - -continue_fuzzy: - - /* Match! */ - id = p->line; - if (mss & wss) - { - if (p->wsize_mod == MOD_MSS) - { - if ((wss % mss) && !(wss % 1460)) retval->gadgets|=GADGETNAT; - } - else if (p->wsize_mod == MOD_MTU) - { - if ((wss % (mss+40)) && !(wss % 1500)) retval->gadgets|=GADGETNAT2; - } - } - - retval->os=p->os; - retval->desc=p->desc; - retval->dist=p->ttl-ttl; - - if (ecn) retval->gadgets|=GADGETECN; - if (orig_df ^ df) retval->gadgets|=GADGETFIREWALL; - - if (p->generic) retval->match=MATCHGENERIC; - if (fuzzy_now) retval->match=MATCHFUZZY; - - if (!p->no_detail && tstamp) - { - retval->uptime=tstamp/360000; - retval->gadgets|=GADGETUPTIME; - } - - return id; - -continue_search: - - p = p->next; - - } - - if (!df) { df = 1; goto re_lookup; } //not found with df=0 do df=1 - - if (fuzzy) - { - df = orig_df; - fuzzy_now = 1; - p = fuzzy; - fuzzy = 0; - goto continue_fuzzy; - } - - if (mss & wss) - { - if ((wss % mss) && !(wss % 1460)) retval->gadgets|=GADGETNAT; - else if ((wss % (mss+40)) && !(wss % 1500)) retval->gadgets|=GADGETNAT2; - } - - if (ecn) retval->gadgets|=GADGETECN; - - if (tstamp) - { - retval->uptime=tstamp/360000; - retval->gadgets|=GADGETUPTIME; - } - - return id; - } diff --git a/src/OSFinger.h b/src/OSFinger.h deleted file mode 100644 index b7c731900c..0000000000 --- a/src/OSFinger.h +++ /dev/null @@ -1,161 +0,0 @@ -// Taken with permission from: -// -// p0f - passive OS fingerprinting (GNU LESSER GENERAL PUBLIC LICENSE) -// ------------------------------------------------------------------- -// -// "If you sit down at a poker game and don't see a sucker, -// get up. You're the sucker." -// -// (C) Copyright 2000-2003 by Michal Zalewski - -#ifndef osfinger_h -#define osfinger_h - -#include "util.h" -#include "Dict.h" -#include "Reporter.h" -#include "IPAddr.h" - -// Size limit for size wildcards. -#define PACKET_BIG 100 - -// Maximum number of signatures allowed in the config file. -#define MAXSIGS 1024 - -// Max signature line length. -#define MAXLINE 1024 - -// Maximum distance from a host to be taken seriously. Between 35 and 64 -// is sane. Making it too high might result in some (very rare) false -// positives, too low will result in needless UNKNOWNs. -#define MAXDIST 40 - -// Maximum number of TCP options. A TCP packet can have at most 64 bytes -// of header, 20 of which are non-options. Thus, if a single option -// consumes 1 bytes (the minimum, there can only be 44 bytes of options. -// We err on the safe side. -#define MAXOPT 64 - -declare(PDict,int); - -struct os_type { - const char* os; - char* desc; - uint8 dist; - uint16 gadgets; - uint16 match; - uint32 uptime; -}; - -struct fp_entry { - struct fp_entry* next; - char* os; // OS genre - char* desc; // OS description - uint8 no_detail; // disable guesstimates - uint8 generic; // generic hit - uint8 userland; // userland stack - uint16 wsize; // window size - uint8 wsize_mod; // MOD_* for wsize - uint8 ttl; // TTL - uint8 df; // don't fragment bit - uint8 zero_stamp; // timestamp option but zero value? - uint16 size; // packet size - uint8 optcnt; // option count - uint8 opt[MAXOPT]; // TCPOPT_* - uint16 wsc; // window scaling option - uint16 mss; // MSS option - uint8 wsc_mod; // modulo for WSCALE (NONE or CONST) - uint8 mss_mod; // modulo for MSS (NONE or CONST) - uint32 quirks; // packet quirks and bugs - uint32 line; // config file line -}; - -struct mtu_def { - uint16 mtu; - char* dev; -}; - -enum FingerprintMode { - SYN_FINGERPRINT_MODE, SYN_ACK_FINGERPRINT_MODE, RST_FINGERPRINT_MODE, -}; - -class OSFingerprint { -public: - explicit OSFingerprint(FingerprintMode mode); - ~OSFingerprint() {} - - bool Error() const { return err; } - - int FindMatch(struct os_type* retval, uint16 tot, uint8 DF_flag, - uint8 TTL, uint16 WSS, uint8 ocnt, uint8* op, uint16 MSS, - uint8 win_scale, uint32 tstamp, uint32 quirks, uint8 ECN) const; - bool CacheMatch(const IPAddr& addr, int id); - void load_config(const char* file); - -protected: - void collide(uint32 id); - - void Error(const char* msg) - { - reporter->Error("%s", msg); - err = true; - } - - void Error(const char* msg, int n) - { - reporter->Error(msg, n); - err = true; - } - - void Error(const char* msg, const char* s) - { - reporter->Error(msg, s); - err = true; - } - -private: - bool err; // if true, a fatal error has occurred - unsigned int mode; - uint32 sigcnt, gencnt; - uint8 problems; - struct fp_entry sig[MAXSIGS]; - - /* By hash */ -#define OSHSIZE 16 - struct fp_entry* bh[OSHSIZE]; - - PDict(int) os_matches; -}; - -#define SIGHASH(tsize, optcnt, q, df) \ - ((uint8(((tsize) << 1) ^ ((optcnt) << 1) ^ (df) ^ (q) )) & 0x0f) - -#define MOD_NONE 0 -#define MOD_CONST 1 -#define MOD_MSS 2 -#define MOD_MTU 3 - -#define QUIRK_PAST 0x1 /* P */ -#define QUIRK_ZEROID 0x2 /* Z */ -#define QUIRK_IPOPT 0x4 /* I */ -#define QUIRK_URG 0x8 /* U */ -#define QUIRK_X2 0x10 /* X */ -#define QUIRK_ACK 0x20 /* A */ -#define QUIRK_T2 0x40 /* T */ -#define QUIRK_FLAGS 0x80 /* F */ -#define QUIRK_DATA 0x100 /* D */ -#define QUIRK_BROKEN 0x200 /* ! */ -#define QUIRK_RSTACK 0x400 /* K */ -#define QUIRK_SEQEQ 0x800 /* Q */ -#define QUIRK_SEQ0 0x1000 /* 0 */ - -#define GADGETNAT 0x1 -#define GADGETNAT2 0x2 -#define GADGETFIREWALL 0x4 -#define GADGETECN 0x8 -#define GADGETUPTIME 0x10 - -#define MATCHGENERIC 0x1 -#define MATCHFUZZY 0x2 - -#endif diff --git a/src/Sessions.cc b/src/Sessions.cc index cdad076e27..e668815cfb 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -14,7 +14,6 @@ #include "NetVar.h" #include "Sessions.h" #include "Reporter.h" -#include "OSFinger.h" #include "analyzer/protocol/icmp/ICMP.h" #include "analyzer/protocol/udp/UDP.h" @@ -130,15 +129,6 @@ NetSessions::NetSessions() dump_this_packet = 0; num_packets_processed = 0; - if ( OS_version_found ) - { - SYN_OS_Fingerprinter = new OSFingerprint(SYN_FINGERPRINT_MODE); - if ( SYN_OS_Fingerprinter->Error() ) - exit(1); - } - else - SYN_OS_Fingerprinter = 0; - if ( pkt_profile_mode && pkt_profile_freq > 0 && pkt_profile_file ) pkt_profiler = new PacketProfiler(pkt_profile_mode, pkt_profile_freq, pkt_profile_file->AsFile()); @@ -155,7 +145,6 @@ NetSessions::~NetSessions() { delete ch; delete packet_filter; - delete SYN_OS_Fingerprinter; delete pkt_profiler; Unref(arp_analyzer); delete discarder; @@ -987,24 +976,6 @@ FragReassembler* NetSessions::NextFragment(double t, const IP_Hdr* ip, return f; } -int NetSessions::Get_OS_From_SYN(struct os_type* retval, - uint16 tot, uint8 DF_flag, uint8 TTL, uint16 WSS, - uint8 ocnt, uint8* op, uint16 MSS, uint8 win_scale, - uint32 tstamp, /* uint8 TOS, */ uint32 quirks, - uint8 ECN) const - { - return SYN_OS_Fingerprinter ? - SYN_OS_Fingerprinter->FindMatch(retval, tot, DF_flag, TTL, - WSS, ocnt, op, MSS, win_scale, tstamp, - quirks, ECN) : 0; - } - -bool NetSessions::CompareWithPreviousOSMatch(const IPAddr& addr, int id) const - { - return SYN_OS_Fingerprinter ? - SYN_OS_Fingerprinter->CacheMatch(addr, id) : 0; - } - Connection* NetSessions::FindConnection(Val* v) { BroType* vt = v->Type(); diff --git a/src/Sessions.h b/src/Sessions.h index 880182c7cd..617ab3e52a 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -17,7 +17,6 @@ class EncapsulationStack; class Connection; -class OSFingerprint; class ConnCompressor; struct ConnID; @@ -77,14 +76,6 @@ public: FragReassembler* NextFragment(double t, const IP_Hdr* ip, const u_char* pkt); - int Get_OS_From_SYN(struct os_type* retval, - uint16 tot, uint8 DF_flag, uint8 TTL, uint16 WSS, - uint8 ocnt, uint8* op, uint16 MSS, uint8 win_scale, - uint32 tstamp, /* uint8 TOS, */ uint32 quirks, - uint8 ECN) const; - - bool CompareWithPreviousOSMatch(const IPAddr& addr, int id) const; - // Looks up the connection referred to by the given Val, // which should be a conn_id record. Returns nil if there's // no such connection or the Val is ill-formed. @@ -240,7 +231,6 @@ protected: analyzer::stepping_stone::SteppingStoneManager* stp_manager; Discarder* discarder; PacketFilter* packet_filter; - OSFingerprint* SYN_OS_Fingerprinter; int build_backdoor_analyzer; int dump_this_packet; // if true, current packet should be recorded uint64 num_packets_processed; diff --git a/src/analyzer/protocol/tcp/TCP.cc b/src/analyzer/protocol/tcp/TCP.cc index 51e9960d9f..74e73b80e2 100644 --- a/src/analyzer/protocol/tcp/TCP.cc +++ b/src/analyzer/protocol/tcp/TCP.cc @@ -4,7 +4,6 @@ #include "NetVar.h" #include "File.h" -#include "OSFinger.h" #include "Event.h" #include "analyzer/protocol/pia/PIA.h" @@ -115,201 +114,6 @@ static RecordVal* build_syn_packet_val(int is_orig, const IP_Hdr* ip, return v; } -static RecordVal* build_os_val(int is_orig, const IP_Hdr* ip, - const struct tcphdr* tcp, uint32 tcp_hdr_len) - { - if ( ! is_orig ) - // Later we might use SYN-ACK fingerprinting here. - return 0; - - // Passive OS fingerprinting wants to know a lot about IP and TCP - // options: how many options there are, and in which order. - int winscale = 0; - int MSS = 0; - int optcount = 0; - uint32 quirks = 0; - uint32 tstamp = 0; - uint8 op[MAXOPT]; - - if ( ip->HdrLen() > 20 ) - quirks |= QUIRK_IPOPT; - - if ( ip->ID() == 0 ) - quirks |= QUIRK_ZEROID; - - if ( tcp->th_seq == 0 ) - quirks |= QUIRK_SEQ0; - - if ( tcp->th_seq == tcp->th_ack ) - quirks |= QUIRK_SEQEQ; - - if ( tcp->th_flags & ~(TH_SYN|TH_ACK|TH_RST|TH_ECE|TH_CWR) ) - quirks |= QUIRK_FLAGS; - - if ( ip->TotalLen() - ip->HdrLen() - tcp_hdr_len > 0 ) - quirks |= QUIRK_DATA; // SYN with data - - if ( tcp->th_ack ) - quirks |= QUIRK_ACK; - if ( tcp->th_urp ) - quirks |= QUIRK_URG; - if ( tcp->th_x2 ) - quirks |= QUIRK_X2; - - // Parse TCP options. - u_char* options = (u_char*) tcp + sizeof(struct tcphdr); - u_char* opt_end = (u_char*) tcp + tcp_hdr_len; - - while ( options < opt_end ) - { - unsigned int opt = options[0]; - - if ( opt == TCPOPT_EOL ) - { - op[optcount++] = TCPOPT_EOL; - if ( ++options < opt_end ) - quirks |= QUIRK_PAST; - - // All done - could flag if more junk left over .... - break; - } - - if ( opt == TCPOPT_NOP ) - { - op[optcount++] = TCPOPT_NOP; - ++options; - continue; - } - - if ( options + 1 >= opt_end ) - { - // We've run off the end, no room for the length. - quirks |= QUIRK_BROKEN; - break; - } - - unsigned int opt_len = options[1]; - - if ( options + opt_len > opt_end ) - { - // No room for rest of the options. - quirks |= QUIRK_BROKEN; - break; - } - - if ( opt_len == 0 ) - // Trashed length field. - break; - - switch ( opt ) { - case TCPOPT_SACK_PERMITTED: - // SACKOK LEN - op[optcount] = TCPOPT_SACK_PERMITTED; - break; - - case TCPOPT_MAXSEG: - // MSS LEN D0 D1 - if ( opt_len < 4 ) - break; // bad length - - op[optcount] = TCPOPT_MAXSEG; - MSS = (options[2] << 8) | options[3]; - break; - - case TCPOPT_WINDOW: - // WSCALE LEN D0 - if ( opt_len < 3 ) - break; // bad length - - op[optcount] = TCPOPT_WINDOW; - winscale = options[2]; - break; - - case TCPOPT_TIMESTAMP: - // TSTAMP LEN T0 T1 T2 T3 A0 A1 A2 A3 - if ( opt_len < 10 ) - break; // bad length - - op[optcount] = TCPOPT_TIMESTAMP; - - tstamp = ntohl(extract_uint32(options + 2)); - - if ( extract_uint32(options + 6) ) - quirks |= QUIRK_T2; - break; - - default: // just skip over - op[optcount]=opt; - break; - } - - if ( optcount < MAXOPT - 1 ) - ++optcount; - else - quirks |= QUIRK_BROKEN; - - options += opt_len; - } - - struct os_type os_from_print; - int id = sessions->Get_OS_From_SYN(&os_from_print, - uint16(ip->TotalLen()), - uint8(ip->DF()), uint8(ip->TTL()), - uint16(ntohs(tcp->th_win)), - uint8(optcount), op, - uint16(MSS), uint8(winscale), - tstamp, quirks, - uint8(tcp->th_flags & (TH_ECE|TH_CWR))); - - if ( sessions->CompareWithPreviousOSMatch(ip->SrcAddr(), id) ) - { - RecordVal* os = new RecordVal(OS_version); - - os->Assign(0, new StringVal(os_from_print.os)); - - if ( os_from_print.desc ) - os->Assign(1, new StringVal(os_from_print.desc)); - else - os->Assign(1, val_mgr->GetEmptyString()); - - os->Assign(2, val_mgr->GetCount(os_from_print.dist)); - os->Assign(3, OS_version_inference->GetVal(os_from_print.match)); - - return os; - } - - return 0; - } - - -static void passive_fingerprint(TCP_Analyzer* tcp, bool is_orig, - const IP_Hdr* ip, const struct tcphdr* tp, - uint32 tcp_hdr_len) - { - // is_orig will be removed once we can do SYN-ACK fingerprinting - if ( OS_version_found && is_orig ) - { - const IPAddr& orig_addr = tcp->Conn()->OrigAddr(); - AddrVal* src_addr_val = new AddrVal(orig_addr); - - if ( generate_OS_version_event->Size() == 0 || - generate_OS_version_event->Lookup(src_addr_val) ) - { - RecordVal* OS_val = build_os_val(is_orig, ip, tp, tcp_hdr_len); - - if ( OS_val ) - { // found new OS version - tcp->ConnectionEventFast(OS_version_found, { - tcp->BuildConnVal(), - src_addr_val->Ref(), - OS_val, - }); - } - } - - Unref(src_addr_val); - } - } TCP_Analyzer::TCP_Analyzer(Connection* conn) : TransportLayerAnalyzer("TCP", conn) @@ -1286,8 +1090,6 @@ void TCP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, }); } - passive_fingerprint(this, is_orig, ip, tp, tcp_hdr_len); - Unref(SYN_vals); } diff --git a/src/event.bif b/src/event.bif index 589ff61201..92f3532ef0 100644 --- a/src/event.bif +++ b/src/event.bif @@ -545,7 +545,7 @@ event signature_match%(state: signature_state, msg: string, data: string%); ## descr: The raw (unparsed) software identification string as extracted from ## the protocol. ## -## .. zeek:see:: software_parse_error software_unparsed_version_found OS_version_found +## .. zeek:see:: software_parse_error software_unparsed_version_found event software_version_found%(c: connection, host: addr, s: software, descr: string%); @@ -564,7 +564,6 @@ event software_version_found%(c: connection, host: addr, ## the protocol. ## ## .. zeek:see:: software_version_found software_unparsed_version_found -## OS_version_found event software_parse_error%(c: connection, host: addr, descr: string%); ## Generated when a protocol analyzer finds an identification of a software @@ -581,25 +580,9 @@ event software_parse_error%(c: connection, host: addr, descr: string%); ## ## str: The software identification string as extracted from the protocol. ## -## .. zeek:see:: software_parse_error software_version_found OS_version_found +## .. zeek:see:: software_parse_error software_version_found event software_unparsed_version_found%(c: connection, host: addr, str: string%); -## Generated when an operating system has been fingerprinted. Zeek uses `p0f -## `__ to fingerprint endpoints passively, -## and it raises this event for each system identified. The p0f fingerprints are -## defined by :zeek:id:`passive_fingerprint_file`. -## -## c: The connection. -## -## host: The host running the reported OS. -## -## OS: The OS version string. -## -## .. zeek:see:: passive_fingerprint_file software_parse_error -## software_version_found software_unparsed_version_found -## generate_OS_version_event -event OS_version_found%(c: connection, host: addr, OS: OS_version%); - ## Generated each time Zeek's internal profiling log is updated. The file is ## defined by :zeek:id:`profiling_file`, and its update frequency by ## :zeek:id:`profiling_interval` and :zeek:id:`expensive_profiling_multiple`. From 028294183868d53c9aebe649469eb988855b0481 Mon Sep 17 00:00:00 2001 From: jatkinosn Date: Wed, 19 Jun 2019 15:12:56 -0400 Subject: [PATCH 12/22] Adding options field to RDP::ClientChannelDef Adding Client Cluster Data --- scripts/base/init-bare.zeek | 9 +++- src/analyzer/protocol/rdp/events.bif | 7 +++ src/analyzer/protocol/rdp/rdp-analyzer.pac | 50 +++++++++++++++++----- src/analyzer/protocol/rdp/rdp-protocol.pac | 12 +++++- src/analyzer/protocol/rdp/types.bif | 1 + 5 files changed, 66 insertions(+), 13 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index f68bf3a545..6ed290fb2c 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4278,7 +4278,9 @@ export { ## Name and flags for a single channel requested by the client. type RDP::ClientChannelDef: record { ## A unique name for the channel - name: string; + name: string; + ## Channel Def raw options as count + options: int; ## Absence of this flag indicates that this channel is ## a placeholder and that the server MUST NOT set it up. initialized: bool; @@ -4304,6 +4306,11 @@ export { persistent: bool; }; + type RDP::ClientClusterData: record { + flags: int; + redir_session_id: int; + }; + ## The list of channels requested by the client. type RDP::ClientChannelList: vector of ClientChannelDef; } diff --git a/src/analyzer/protocol/rdp/events.bif b/src/analyzer/protocol/rdp/events.bif index 0931365dc6..91d2fc004a 100644 --- a/src/analyzer/protocol/rdp/events.bif +++ b/src/analyzer/protocol/rdp/events.bif @@ -49,6 +49,13 @@ event rdp_client_security_data%(c: connection, data: RDP::ClientSecurityData%); ## channels: The channels that were requested event rdp_client_network_data%(c: connection, channels: RDP::ClientChannelList%); +## Generated for client clusgter data packets. +## +## c: The connection record for the underlying transport-layer session/flow. +## +## data: The data contained in the client security data structure. +event rdp_client_cluster_data%(c: connection, data: RDP::ClientClusterData%); + ## Generated for MCS server responses. ## ## c: The connection record for the underlying transport-layer session/flow. diff --git a/src/analyzer/protocol/rdp/rdp-analyzer.pac b/src/analyzer/protocol/rdp/rdp-analyzer.pac index 7b7552642f..d52c1153c5 100644 --- a/src/analyzer/protocol/rdp/rdp-analyzer.pac +++ b/src/analyzer/protocol/rdp/rdp-analyzer.pac @@ -130,18 +130,19 @@ refine flow RDP_Flow += { RecordVal* channel_def = new RecordVal(BifType::Record::RDP::ClientChannelDef); channel_def->Assign(0, bytestring_to_val(${cnetwork.channel_def_array[i].name})); + channel_def->Assign(1, val_mgr->GetCount(${cnetwork.channel_def_array[i].options})); - channel_def->Assign(1, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_INITIALIZED})); - channel_def->Assign(2, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_ENCRYPT_RDP})); - channel_def->Assign(3, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_ENCRYPT_SC})); - channel_def->Assign(4, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_ENCRYPT_CS})); - channel_def->Assign(5, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_PRI_HIGH})); - channel_def->Assign(6, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_PRI_MED})); - channel_def->Assign(7, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_PRI_LOW})); - channel_def->Assign(8, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_COMPRESS_RDP})); - channel_def->Assign(9, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_COMPRESS})); - channel_def->Assign(10, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_SHOW_PROTOCOL})); - channel_def->Assign(11, val_mgr->GetBool(${cnetwork.channel_def_array[i].REMOTE_CONTROL_PERSISTENT})); + channel_def->Assign(2, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_INITIALIZED})); + channel_def->Assign(3, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_ENCRYPT_RDP})); + channel_def->Assign(4, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_ENCRYPT_SC})); + channel_def->Assign(5, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_ENCRYPT_CS})); + channel_def->Assign(6, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_PRI_HIGH})); + channel_def->Assign(7, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_PRI_MED})); + channel_def->Assign(8, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_PRI_LOW})); + channel_def->Assign(9, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_COMPRESS_RDP})); + channel_def->Assign(10, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_COMPRESS})); + channel_def->Assign(11, val_mgr->GetBool(${cnetwork.channel_def_array[i].CHANNEL_OPTION_SHOW_PROTOCOL})); + channel_def->Assign(12, val_mgr->GetBool(${cnetwork.channel_def_array[i].REMOTE_CONTROL_PERSISTENT})); channels->Assign(channels->Size(), channel_def); } @@ -154,6 +155,29 @@ refine flow RDP_Flow += { return true; %} + + function proc_rdp_client_cluster_data(ccluster: Client_Cluster_Data): bool + %{ + if ( ! rdp_client_cluster_data ) + return false; + + RecordVal* ccld = new RecordVal(BifType::Record::RDP::ClientClusterData); + ccld->Assign(0, val_mgr->GetCount(${ccluster.flags})); + ccld->Assign(1, val_mgr->GetCount(${ccluster.redir_session_id})); + + ccld->Assign(2, val_mgr->GetBool(${ccluster.REDIRECTION_SUPPORTED})); + ccld->Assign(3, val_mgr->GetCount(${ccluster.SERVER_SESSION_REDIRECTION_VERSION_MASK})); + ccld->Assign(4, val_mgr->GetCount(${ccluster.REDIRECTED_SESSIONID_FIELD_VALID})); + ccld->Assign(5, val_mgr->GetBool(${ccluster.REDIRECTED_SMARTCARD})); + + BifEvent::generate_rdp_client_cluster_data(connection()->bro_analyzer(), + connection()->bro_analyzer()->Conn(), + ccld); + return true; + %} + + + function proc_rdp_server_security(ssd: Server_Security_Data): bool %{ connection()->bro_analyzer()->ProtocolConfirmation(); @@ -226,6 +250,10 @@ refine typeattr Client_Network_Data += &let { proc: bool = $context.flow.proc_rdp_client_network_data(this); }; +refine typeattr Client_Cluster_Data += &let { + proc: bool = $context.flow.proc_rdp_client_cluster_data(this); +}; + refine typeattr GCC_Server_Create_Response += &let { proc: bool = $context.flow.proc_rdp_gcc_server_create_response(this); }; diff --git a/src/analyzer/protocol/rdp/rdp-protocol.pac b/src/analyzer/protocol/rdp/rdp-protocol.pac index 442a0d1292..f10dcf0af4 100644 --- a/src/analyzer/protocol/rdp/rdp-protocol.pac +++ b/src/analyzer/protocol/rdp/rdp-protocol.pac @@ -54,7 +54,7 @@ type Data_Block = record { 0xc001 -> client_core: Client_Core_Data; 0xc002 -> client_security: Client_Security_Data; 0xc003 -> client_network: Client_Network_Data; - #0xc004 -> client_cluster: Client_Cluster_Data; + 0xc004 -> client_cluster: Client_Cluster_Data; #0xc005 -> client_monitor: Client_Monitor_Data; #0xc006 -> client_msgchannel: Client_MsgChannel_Data; #0xc008 -> client_monitor_ex: Client_MonitorExtended_Data; @@ -230,6 +230,16 @@ type Client_Network_Data = record { channel_def_array: Client_Channel_Def[channel_count]; } &byteorder=littleendian; +type Client_Cluster_Data = record { + flags: uint32; + redir_session_id: uint32; +} &let { + REDIRECTION_SUPPORTED: bool = redir_session_id & 0x00000001; + SERVER_SESSION_REDIRECTION_VERSION_MASK: int = (redir_session_id & 0x0000003C); + REDIRECTED_SESSIONID_FIELD_VALID: int = (redir_session_id & 0x00000002); + REDIRECTED_SMARTCARD: bool = redir_session_id & 0x00000040; +} &byteorder=littleendian; + type Client_Channel_Def = record { name: bytestring &length=8; options: uint32; diff --git a/src/analyzer/protocol/rdp/types.bif b/src/analyzer/protocol/rdp/types.bif index 69cbe14dd3..366676d017 100644 --- a/src/analyzer/protocol/rdp/types.bif +++ b/src/analyzer/protocol/rdp/types.bif @@ -5,6 +5,7 @@ type EarlyCapabilityFlags: record; type ClientCoreData: record; type ClientSecurityData: record; +type ClientClusterData: record; type ClientChannelList: vector; type ClientChannelDef: record; From bd0bf3f84f12cb69a13c010431d0933807c426e2 Mon Sep 17 00:00:00 2001 From: jatkinosn Date: Wed, 19 Jun 2019 16:10:29 -0400 Subject: [PATCH 13/22] Removing misc data from Client Cluster data trying to assign values. --- src/analyzer/protocol/rdp/rdp-analyzer.pac | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/analyzer/protocol/rdp/rdp-analyzer.pac b/src/analyzer/protocol/rdp/rdp-analyzer.pac index d52c1153c5..3007529f98 100644 --- a/src/analyzer/protocol/rdp/rdp-analyzer.pac +++ b/src/analyzer/protocol/rdp/rdp-analyzer.pac @@ -165,11 +165,6 @@ refine flow RDP_Flow += { ccld->Assign(0, val_mgr->GetCount(${ccluster.flags})); ccld->Assign(1, val_mgr->GetCount(${ccluster.redir_session_id})); - ccld->Assign(2, val_mgr->GetBool(${ccluster.REDIRECTION_SUPPORTED})); - ccld->Assign(3, val_mgr->GetCount(${ccluster.SERVER_SESSION_REDIRECTION_VERSION_MASK})); - ccld->Assign(4, val_mgr->GetCount(${ccluster.REDIRECTED_SESSIONID_FIELD_VALID})); - ccld->Assign(5, val_mgr->GetBool(${ccluster.REDIRECTED_SMARTCARD})); - BifEvent::generate_rdp_client_cluster_data(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), ccld); From 7b42c3a201b2695d4cf997e69aa5564f9c516fc8 Mon Sep 17 00:00:00 2001 From: jatkinosn Date: Thu, 20 Jun 2019 09:32:37 -0400 Subject: [PATCH 14/22] Correcting types. --- scripts/base/init-bare.zeek | 6 +++--- src/analyzer/protocol/rdp/events.bif | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 6ed290fb2c..f40b1a6fbe 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4280,7 +4280,7 @@ export { ## A unique name for the channel name: string; ## Channel Def raw options as count - options: int; + options: count; ## Absence of this flag indicates that this channel is ## a placeholder and that the server MUST NOT set it up. initialized: bool; @@ -4307,8 +4307,8 @@ export { }; type RDP::ClientClusterData: record { - flags: int; - redir_session_id: int; + flags: count; + redir_session_id: count; }; ## The list of channels requested by the client. diff --git a/src/analyzer/protocol/rdp/events.bif b/src/analyzer/protocol/rdp/events.bif index 91d2fc004a..178860bd42 100644 --- a/src/analyzer/protocol/rdp/events.bif +++ b/src/analyzer/protocol/rdp/events.bif @@ -49,7 +49,7 @@ event rdp_client_security_data%(c: connection, data: RDP::ClientSecurityData%); ## channels: The channels that were requested event rdp_client_network_data%(c: connection, channels: RDP::ClientChannelList%); -## Generated for client clusgter data packets. +## Generated for client cluster data packets. ## ## c: The connection record for the underlying transport-layer session/flow. ## From 3a19af86c59509f35dd16bb199e75a707c628202 Mon Sep 17 00:00:00 2001 From: jatkinosn Date: Thu, 20 Jun 2019 10:47:05 -0400 Subject: [PATCH 15/22] Fixing types. Added handling for fields sub fields. Added test script and output. --- scripts/base/init-bare.zeek | 8 ++++-- src/analyzer/protocol/rdp/rdp-analyzer.pac | 4 +++ src/analyzer/protocol/rdp/rdp-protocol.pac | 4 +-- .../out | 12 ++++++++ .../rdp/rdp-client-cluster-data.zeek | 28 +++++++++++++++++++ 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.rdp.rdp-client-cluster_data/out create mode 100644 testing/btest/scripts/base/protocols/rdp/rdp-client-cluster-data.zeek diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index f40b1a6fbe..728077e062 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4307,8 +4307,12 @@ export { }; type RDP::ClientClusterData: record { - flags: count; - redir_session_id: count; + flags: count; + redir_session_id: count; + redir_supported: bool; + svr_session_redir_version_mask: count; + redir_sessionid_field_valid: count; + redir_smartcard: bool; }; ## The list of channels requested by the client. diff --git a/src/analyzer/protocol/rdp/rdp-analyzer.pac b/src/analyzer/protocol/rdp/rdp-analyzer.pac index 3007529f98..2355ceab79 100644 --- a/src/analyzer/protocol/rdp/rdp-analyzer.pac +++ b/src/analyzer/protocol/rdp/rdp-analyzer.pac @@ -164,6 +164,10 @@ refine flow RDP_Flow += { RecordVal* ccld = new RecordVal(BifType::Record::RDP::ClientClusterData); ccld->Assign(0, val_mgr->GetCount(${ccluster.flags})); ccld->Assign(1, val_mgr->GetCount(${ccluster.redir_session_id})); + ccld->Assign(2, val_mgr->GetBool(${ccluster.REDIRECTION_SUPPORTED})); + ccld->Assign(3, val_mgr->GetCount(${ccluster.SERVER_SESSION_REDIRECTION_VERSION_MASK})); + ccld->Assign(4, val_mgr->GetCount(${ccluster.REDIRECTED_SESSIONID_FIELD_VALID})); + ccld->Assign(5, val_mgr->GetBool(${ccluster.REDIRECTED_SMARTCARD})); BifEvent::generate_rdp_client_cluster_data(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), diff --git a/src/analyzer/protocol/rdp/rdp-protocol.pac b/src/analyzer/protocol/rdp/rdp-protocol.pac index f10dcf0af4..bcf5e89a2e 100644 --- a/src/analyzer/protocol/rdp/rdp-protocol.pac +++ b/src/analyzer/protocol/rdp/rdp-protocol.pac @@ -235,8 +235,8 @@ type Client_Cluster_Data = record { redir_session_id: uint32; } &let { REDIRECTION_SUPPORTED: bool = redir_session_id & 0x00000001; - SERVER_SESSION_REDIRECTION_VERSION_MASK: int = (redir_session_id & 0x0000003C); - REDIRECTED_SESSIONID_FIELD_VALID: int = (redir_session_id & 0x00000002); + SERVER_SESSION_REDIRECTION_VERSION_MASK: uint8 = (redir_session_id & 0x0000003C); + REDIRECTED_SESSIONID_FIELD_VALID: uint8 = (redir_session_id & 0x00000002); REDIRECTED_SMARTCARD: bool = redir_session_id & 0x00000040; } &byteorder=littleendian; diff --git a/testing/btest/Baseline/scripts.base.protocols.rdp.rdp-client-cluster_data/out b/testing/btest/Baseline/scripts.base.protocols.rdp.rdp-client-cluster_data/out new file mode 100644 index 0000000000..53973a2324 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.rdp.rdp-client-cluster_data/out @@ -0,0 +1,12 @@ +RDP Client Cluster Data +Flags: 0000000d +RedirSessionId: 00000000 +Redirection Supported: 00000000 +ServerSessionRedirectionVersionMask: 00000000 +RedirectionSessionIDFieldValid: 00000000 +RedirectedSmartCard: 00000000 +RDP Client Channel List Options +80800000 +c0000000 +c0800000 +c0a00000 diff --git a/testing/btest/scripts/base/protocols/rdp/rdp-client-cluster-data.zeek b/testing/btest/scripts/base/protocols/rdp/rdp-client-cluster-data.zeek new file mode 100644 index 0000000000..97a711209a --- /dev/null +++ b/testing/btest/scripts/base/protocols/rdp/rdp-client-cluster-data.zeek @@ -0,0 +1,28 @@ +# @TEST-EXEC: zeek -r $TRACES/rdp/rdp-proprietary-encryption.pcap %INPUT >out +# @TEST-EXEC: btest-diff out + +@load base/protocol/rdp + + +event rdp_client_cluster_data(c: connection, data: RDP::ClientClusterData) +{ +print "RDP Client Cluster Data"; +#print data; +print fmt("Flags: %08x",data$flags); +print fmt("RedirSessionId: %08x",data$redir_session_id); +print fmt("Redirection Supported: %08x",data$redir_supported); +print fmt("ServerSessionRedirectionVersionMask: %08x",data$svr_session_redir_version_mask); +print fmt("RedirectionSessionIDFieldValid: %08x",data$redir_sessionid_field_valid); +print fmt("RedirectedSmartCard: %08x",data$redir_smartcard); + +} + + +event rdp_client_network_data(c: connection, channels: RDP::ClientChannelList) +{ +print "RDP Client Channel List Options"; +for ( i in channels ) { + print fmt("%08x", channels[i]$options); + } +} + From 437520f45f5da333e8755212ff09f88bf2c9665b Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 20 Jun 2019 11:19:17 -0700 Subject: [PATCH 16/22] Make configure complain if submodules are not checked out. Since people forgetting to checkout submodules is such a common failure case - update configure to give an error message is the cmake directory seems to be missing. This just checks for the presence of cmake/COPYING when a .git directory is found; if cmake/COPYING is not present an error message is displayed. --- configure | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/configure b/configure index ec344d808f..e6b4e1f0b5 100755 --- a/configure +++ b/configure @@ -106,6 +106,20 @@ Usage: $0 [OPTION]... [VAR=VALUE]... sourcedir="$( cd "$( dirname "$0" )" && pwd )" +if [ ! -e "$sourcedir/cmake/COPYING" ] && [ -d "$sourcedir/.git" ]; then + echo "\ +You seem to be missing the content of the cmake directory. + +This typically means that you performed a non-recursive git clone of +Zeek. To check out the required subdirectories, please execute + + git submodule update --recursive --init + +in $sourcedir. +" >&2; + exit 1; +fi + # Function to append a CMake cache entry definition to the # CMakeCacheEntries variable. # $1 is the cache entry variable name From b9d3d4d63b80044d573ff3dbbe06789df3b82953 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 20 Jun 2019 14:00:22 -0700 Subject: [PATCH 17/22] Remove unused SerialInfo.h and SerialTypes.h headers --- CHANGES | 4 + VERSION | 2 +- src/SerialInfo.h | 160 ------------------------------- src/SerialTypes.h | 236 ---------------------------------------------- 4 files changed, 5 insertions(+), 397 deletions(-) delete mode 100644 src/SerialInfo.h delete mode 100644 src/SerialTypes.h diff --git a/CHANGES b/CHANGES index 2ffd5506ba..03c4affe55 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.6-477 | 2019-06-20 14:00:22 -0700 + + * Remove unused SerialInfo.h and SerialTypes.h headers (Jon Siwek, Corelight) + 2.6-476 | 2019-06-20 13:23:22 -0700 * Remove opaque of ocsp_resp. (Johanna Amann, Corelight) diff --git a/VERSION b/VERSION index 2c31edab5c..6aee059b51 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-476 +2.6-477 diff --git a/src/SerialInfo.h b/src/SerialInfo.h deleted file mode 100644 index 04cd79daf7..0000000000 --- a/src/SerialInfo.h +++ /dev/null @@ -1,160 +0,0 @@ -// Helper classes to pass data between serialization methods. - -#ifndef serialinfo_h -#define serialinfo_h - -class SerialInfo { -public: - SerialInfo(Serializer* arg_s) - { - s = arg_s; - may_suspend = clear_containers = false; - cache = globals_as_names = true; - type = SER_NONE; - pid_32bit = false; - include_locations = true; - new_cache_strategy = false; - } - - SerialInfo(const SerialInfo& info) - { - s = info.s; - may_suspend = info.may_suspend; - cache = info.cache; - type = info.type; - clear_containers = info.clear_containers; - globals_as_names = info.globals_as_names; - pid_32bit = info.pid_32bit; - include_locations = info.include_locations; - new_cache_strategy = info.new_cache_strategy; - } - - // Parameters that control serialization. - Serializer* s; // serializer to use - bool cache; // true if object caching is ok - bool may_suspend; // if true, suspending serialization is ok - bool clear_containers; // if true, store container values as empty - bool include_locations; // if true, include locations in serialization - - // If true, for NameExpr's serialize just the names of globals, just - // their value. - bool globals_as_names; - - bool pid_32bit; // if true, use old-style 32-bit permanent IDs - - // If true, we support keeping objs in cache permanently. - bool new_cache_strategy; - - // Attributes set during serialization. - SerialType type; // type of currently serialized object - - // State for suspending/resuming serialization - Continuation cont; -}; - -class UnserialInfo { -public: - UnserialInfo(Serializer* arg_s) - { - s = arg_s; - cache = true; - type = SER_NONE; - install_globals = install_conns = true; - install_uniques = false; - ignore_callbacks = false; - id_policy = Replace; - print = 0; - pid_32bit = false; - new_cache_strategy = false; - } - - UnserialInfo(const UnserialInfo& info) - { - s = info.s; - cache = info.cache; - type = info.type; - install_globals = info.install_globals; - install_uniques = info.install_uniques; - install_conns = info.install_conns; - ignore_callbacks = info.ignore_callbacks; - id_policy = info.id_policy; - print = info.print; - pid_32bit = info.pid_32bit; - new_cache_strategy = info.new_cache_strategy; - } - - // Parameters that control unserialization. - Serializer* s; // serializer to use - bool cache; // if true, object caching is ok - FILE* print; // print read objects to given file (human-readable) - - bool install_globals; // if true, install unknown globals - // in global scope - bool install_conns; // if true, add connections to session table - bool install_uniques; // if true, install unknown globally - // unique IDs in global scope - bool ignore_callbacks; // if true, don't call Got*() callbacks - bool pid_32bit; // if true, use old-style 32-bit permanent IDs. - - // If true, we support keeping objs in cache permanently. - bool new_cache_strategy; - - // If a global ID already exits, of these policies is used. - enum { - Keep, // keep the old ID and ignore the new - Replace, // install the new ID (default) - - // Keep current ID instance but copy the new value into it - // (types have to match). - CopyNewToCurrent, - - // Install the new ID instance but replace its value - // with that of the old one (types have to match). - CopyCurrentToNew, - - // Instantiate a new ID, but do not insert it into the global - // space. - InstantiateNew, - } id_policy; - - // Attributes set during unserialization. - SerialType type; // type of currently unserialized object -}; - -// Helper class to temporarily disable suspending for all next-level calls -// using the given SerialInfo. It saves the current value of info.may_suspend -// and then sets it to false. When it goes out of scope, the original value -// is restored. -// -// We need this because not all classes derived from SerialObj are -// suspension-aware yet, i.e., they don't work correctly if one of the -// next-level functions suspends. Eventually this may change, but actually -// it's not very important: most classes don't need to suspend anyway as -// their data volume is very small. We have to make sure though that those -// which do (e.g. TableVals) support suspension. -class DisableSuspend { -public: - DisableSuspend(SerialInfo* arg_info) - { - info = arg_info; - old_may_suspend = info->may_suspend; - info->may_suspend = false; - } - - ~DisableSuspend() { Restore(); } - - void Release() { info = 0; } - - // Restores the suspension-state to its original value. - void Restore() - { - if ( info ) - info->may_suspend = old_may_suspend; - } - -private: - SerialInfo* info; - bool old_may_suspend; -}; - -#endif diff --git a/src/SerialTypes.h b/src/SerialTypes.h deleted file mode 100644 index a0b3f4231b..0000000000 --- a/src/SerialTypes.h +++ /dev/null @@ -1,236 +0,0 @@ -#ifndef serialtypes_h -#define serialtypes_h - -// Each serializable class gets a type. -// -// The type enables a form of poor man's type-checking: -// Bit 0-7: Number (unique relative to main parent (see below)). -// Bit 8-12: Main parent class (SER_IS_*) -// Bit 13: unused -// Bit 14: 1 if preference is to keep in cache. -// Bit 15: 1 if derived from BroObj. - -typedef uint16 SerialType; - -static const SerialType SER_TYPE_MASK_EXACT = 0x1fff; -static const SerialType SER_TYPE_MASK_PARENT = 0x1f00; -static const SerialType SER_IS_CACHE_STABLE = 0x4000; -static const SerialType SER_IS_BRO_OBJ = 0x8000; - -#define SERIAL_CONST(name, val, type) \ - const SerialType SER_ ## name = val | SER_IS_ ## type; - -#define SERIAL_CONST2(name) SERIAL_CONST(name, 1, name) - -#define SERIAL_IS(name, val) \ - static const SerialType SER_IS_ ## name = val; -#define SERIAL_IS_BO(name, val) \ - static const SerialType SER_IS_ ## name = val | SER_IS_BRO_OBJ; -#define SERIAL_IS_BO_AND_CACHE_STABLE(name, val) \ - static const SerialType SER_IS_ ## name = val | (SER_IS_BRO_OBJ | SER_IS_CACHE_STABLE); - -SERIAL_IS_BO(CONNECTION, 0x0100) -SERIAL_IS(TIMER, 0x0200) -SERIAL_IS(TCP_ENDPOINT, 0x0300) -SERIAL_IS_BO(TCP_ANALYZER, 0x0400) -SERIAL_IS_BO(TCP_ENDPOINT_ANALYZER, 0x0500) -SERIAL_IS(TCP_CONTENTS, 0x0600) -SERIAL_IS(REASSEMBLER, 0x0700) -SERIAL_IS_BO(VAL, 0x0800) -SERIAL_IS_BO_AND_CACHE_STABLE(EXPR, 0x0900) -SERIAL_IS_BO_AND_CACHE_STABLE(BRO_TYPE, 0x0a00) -SERIAL_IS_BO_AND_CACHE_STABLE(STMT, 0x0b00) -SERIAL_IS_BO_AND_CACHE_STABLE(ATTRIBUTES, 0x0c00) -SERIAL_IS_BO_AND_CACHE_STABLE(EVENT_HANDLER, 0x0d00) -SERIAL_IS_BO_AND_CACHE_STABLE(BRO_FILE, 0x0e00) -SERIAL_IS_BO_AND_CACHE_STABLE(FUNC, 0x0f00) -SERIAL_IS_BO(ID, 0x1000) -SERIAL_IS(STATE_ACCESS, 0x1100) -SERIAL_IS_BO(CASE, 0x1200) -SERIAL_IS(LOCATION, 0x1300) -SERIAL_IS(RE_MATCHER, 0x1400) -SERIAL_IS(BITVECTOR, 0x1500) -SERIAL_IS(COUNTERVECTOR, 0x1600) -SERIAL_IS(BLOOMFILTER, 0x1700) -SERIAL_IS(HASHER, 0x1800) - -// These are the externally visible types. -const SerialType SER_NONE = 0; - -SERIAL_CONST2(BRO_OBJ) - -#define SERIAL_CONN(name, val) SERIAL_CONST(name, val, CONNECTION) -SERIAL_CONN(CONNECTION, 1) -SERIAL_CONN(ICMP_ANALYZER, 2) -// We use ICMP_Echo here rather than ICMP_ECHO because the latter gets -// macro expanded :-(. -SERIAL_CONN(ICMP_Echo, 3) -SERIAL_CONN(ICMP_CONTEXT, 4) -SERIAL_CONN(TCP_CONNECTION, 5) -SERIAL_CONN(TCP_CONNECTION_CONTENTS, 6) -SERIAL_CONN(FTP_CONN, 7) -SERIAL_CONN(UDP_CONNECTION, 8) - -#define SERIAL_TIMER(name, val) SERIAL_CONST(name, val, TIMER) -SERIAL_TIMER(TIMER, 1) -SERIAL_TIMER(CONNECTION_TIMER, 2) - -SERIAL_CONST2(TCP_ENDPOINT) -SERIAL_CONST2(TCP_ANALYZER) -SERIAL_CONST2(TCP_ENDPOINT_ANALYZER) - -#define SERIAL_TCP_CONTENTS(name, val) SERIAL_CONST(name, val, TCP_CONTENTS) -SERIAL_TCP_CONTENTS(TCP_CONTENTS, 1) -SERIAL_TCP_CONTENTS(TCP_CONTENT_LINE, 2) -SERIAL_TCP_CONTENTS(TCP_NVT, 3) - -#define SERIAL_REASSEMBLER(name, val) SERIAL_CONST(name, val, REASSEMBLER) -SERIAL_REASSEMBLER(REASSEMBLER, 1) -SERIAL_REASSEMBLER(TCP_REASSEMBLER, 2) -SERIAL_REASSEMBLER(FILE_REASSEMBLER, 3) - -#define SERIAL_VAL(name, val) SERIAL_CONST(name, val, VAL) -SERIAL_VAL(VAL, 1) -SERIAL_VAL(INTERVAL_VAL, 2) -SERIAL_VAL(PORT_VAL, 3) -SERIAL_VAL(ADDR_VAL, 4) -SERIAL_VAL(SUBNET_VAL, 5) -SERIAL_VAL(STRING_VAL, 6) -SERIAL_VAL(PATTERN_VAL, 7) -SERIAL_VAL(LIST_VAL, 8) -SERIAL_VAL(TABLE_VAL, 9) -SERIAL_VAL(RECORD_VAL, 10) -SERIAL_VAL(ENUM_VAL, 11) -SERIAL_VAL(VECTOR_VAL, 12) -SERIAL_VAL(MUTABLE_VAL, 13) -SERIAL_VAL(OPAQUE_VAL, 14) -SERIAL_VAL(HASH_VAL, 15) -SERIAL_VAL(MD5_VAL, 16) -SERIAL_VAL(SHA1_VAL, 17) -SERIAL_VAL(SHA256_VAL, 18) -SERIAL_VAL(ENTROPY_VAL, 19) -SERIAL_VAL(TOPK_VAL, 20) -SERIAL_VAL(BLOOMFILTER_VAL, 21) -SERIAL_VAL(CARDINALITY_VAL, 22) -SERIAL_VAL(X509_VAL, 23) -SERIAL_VAL(COMM_STORE_HANDLE_VAL, 24) -SERIAL_VAL(COMM_DATA_VAL, 25) -SERIAL_VAL(OCSP_RESP_VAL, 26) - -#define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR) -SERIAL_EXPR(EXPR, 1) -SERIAL_EXPR(NAME_EXPR, 2) -SERIAL_EXPR(CONST_EXPR, 3) -SERIAL_EXPR(UNARY_EXPR, 4) -SERIAL_EXPR(BINARY_EXPR, 5) -SERIAL_EXPR(INCR_EXPR, 6) -SERIAL_EXPR(NOT_EXPR, 7) -SERIAL_EXPR(POS_EXPR, 8) -SERIAL_EXPR(NEG_EXPR, 9) -SERIAL_EXPR(ADD_EXPR, 10) -SERIAL_EXPR(SUB_EXPR, 11) -SERIAL_EXPR(TIMES_EXPR, 12) -SERIAL_EXPR(DIVIDE_EXPR, 13) -SERIAL_EXPR(MOD_EXPR, 14) -SERIAL_EXPR(BOOL_EXPR, 15) -SERIAL_EXPR(EQ_EXPR, 16) -SERIAL_EXPR(REL_EXPR, 17) -SERIAL_EXPR(COND_EXPR, 18) -SERIAL_EXPR(REF_EXPR, 19) -SERIAL_EXPR(ASSIGN_EXPR, 20) -SERIAL_EXPR(INDEX_EXPR, 21) -SERIAL_EXPR(FIELD_EXPR, 22) -SERIAL_EXPR(HAS_FIELD_EXPR, 23) -SERIAL_EXPR(RECORD_CONSTRUCTOR_EXPR, 24) -SERIAL_EXPR(FIELD_ASSIGN_EXPR, 25) -// There used to be a SERIAL_EXPR(RECORD_MATCH_EXPR, 26) here -SERIAL_EXPR(ARITH_COERCE_EXPR, 27) -SERIAL_EXPR(RECORD_COERCE_EXPR, 28) -SERIAL_EXPR(FLATTEN_EXPR, 29) -SERIAL_EXPR(SCHEDULE_EXPR, 30) -SERIAL_EXPR(IN_EXPR, 31) -SERIAL_EXPR(CALL_EXPR, 32) -SERIAL_EXPR(EVENT_EXPR, 33) -SERIAL_EXPR(LIST_EXPR, 34) -SERIAL_EXPR(RECORD_ASSIGN_EXPR, 35) -SERIAL_EXPR(ADD_TO_EXPR, 36) -SERIAL_EXPR(REMOVE_FROM_EXPR, 37) -SERIAL_EXPR(SIZE_EXPR, 38) -SERIAL_EXPR(CLONE_EXPR, 39) -SERIAL_EXPR(TABLE_CONSTRUCTOR_EXPR, 40) -SERIAL_EXPR(SET_CONSTRUCTOR_EXPR, 41) -SERIAL_EXPR(VECTOR_CONSTRUCTOR_EXPR, 42) -SERIAL_EXPR(TABLE_COERCE_EXPR, 43) -SERIAL_EXPR(VECTOR_COERCE_EXPR, 44) -SERIAL_EXPR(CAST_EXPR, 45) -SERIAL_EXPR(IS_EXPR_, 46) // Name conflict with internal SER_IS_EXPR constant. -SERIAL_EXPR(BIT_EXPR, 47) -SERIAL_EXPR(COMPLEMENT_EXPR, 48) -SERIAL_EXPR(INDEX_SLICE_ASSIGN_EXPR, 49) - -#define SERIAL_STMT(name, val) SERIAL_CONST(name, val, STMT) -SERIAL_STMT(STMT, 1) -SERIAL_STMT(EXPR_LIST_STMT, 2) -// There used to be ALARM_STMT (3) here. -SERIAL_STMT(PRINT_STMT, 4) -SERIAL_STMT(EXPR_STMT, 5) -SERIAL_STMT(IF_STMT, 6) -SERIAL_STMT(SWITCH_STMT, 7) -SERIAL_STMT(ADD_STMT, 8) -SERIAL_STMT(DEL_STMT, 9) -SERIAL_STMT(EVENT_STMT, 10) -SERIAL_STMT(FOR_STMT, 11) -SERIAL_STMT(NEXT_STMT, 12) -SERIAL_STMT(BREAK_STMT, 13) -SERIAL_STMT(RETURN_STMT, 14) -SERIAL_STMT(STMT_LIST, 15) -SERIAL_STMT(EVENT_BODY_LIST, 16) -SERIAL_STMT(INIT_STMT, 17) -SERIAL_STMT(NULL_STMT, 18) -SERIAL_STMT(WHEN_STMT, 19) -SERIAL_STMT(FALLTHROUGH_STMT, 20) -SERIAL_STMT(WHILE_STMT, 21) - -#define SERIAL_TYPE(name, val) SERIAL_CONST(name, val, BRO_TYPE) -SERIAL_TYPE(BRO_TYPE, 1) -SERIAL_TYPE(TYPE_LIST, 2) -SERIAL_TYPE(INDEX_TYPE, 3) -SERIAL_TYPE(TABLE_TYPE, 4) -SERIAL_TYPE(SET_TYPE, 5) -SERIAL_TYPE(FUNC_TYPE, 6) -SERIAL_TYPE(RECORD_TYPE, 7) -SERIAL_TYPE(SUBNET_TYPE, 8) -SERIAL_TYPE(FILE_TYPE, 9) -SERIAL_TYPE(ENUM_TYPE, 10) -SERIAL_TYPE(VECTOR_TYPE, 11) -SERIAL_TYPE(OPAQUE_TYPE, 12) - -SERIAL_CONST2(ATTRIBUTES) -SERIAL_CONST2(EVENT_HANDLER) -SERIAL_CONST2(BRO_FILE) - -#define SERIAL_FUNC(name, val) SERIAL_CONST(name, val, FUNC) -SERIAL_FUNC(FUNC, 1) -SERIAL_FUNC(BRO_FUNC, 2) -SERIAL_FUNC(DEBUG_FUNC, 3) -SERIAL_FUNC(BUILTIN_FUNC, 4) - -#define SERIAL_BLOOMFILTER(name, val) SERIAL_CONST(name, val, BLOOMFILTER) -SERIAL_BLOOMFILTER(BLOOMFILTER, 1) -SERIAL_BLOOMFILTER(BASICBLOOMFILTER, 2) -SERIAL_BLOOMFILTER(COUNTINGBLOOMFILTER, 3) - -#define SERIAL_HASHER(name, val) SERIAL_CONST(name, val, HASHER) -SERIAL_HASHER(HASHER, 1) -SERIAL_HASHER(DEFAULTHASHER, 2) -SERIAL_HASHER(DOUBLEHASHER, 3) - -SERIAL_CONST2(ID) -SERIAL_CONST2(STATE_ACCESS) -SERIAL_CONST2(CASE) -SERIAL_CONST2(LOCATION) -SERIAL_CONST2(RE_MATCHER) -SERIAL_CONST2(BITVECTOR) -SERIAL_CONST2(COUNTERVECTOR) - -#endif From 61d19d25e18cff3f5e050331f0a752d309c898f7 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 20 Jun 2019 14:19:11 -0700 Subject: [PATCH 18/22] Remove old Broccoli SSL options - ssl_ca_certificate - ssl_private_key - ssl_passphrase --- CHANGES | 8 ++++++++ NEWS | 3 +++ VERSION | 2 +- doc | 2 +- scripts/base/init-bare.zeek | 16 ---------------- src/NetVar.cc | 8 -------- src/NetVar.h | 4 ---- 7 files changed, 13 insertions(+), 30 deletions(-) diff --git a/CHANGES b/CHANGES index 03c4affe55..115d0ef359 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,12 @@ +2.6-478 | 2019-06-20 14:19:11 -0700 + + * Remove old Broccoli SSL options (Jon Siwek, Corelight) + + - ssl_ca_certificate + - ssl_private_key + - ssl_passphrase + 2.6-477 | 2019-06-20 14:00:22 -0700 * Remove unused SerialInfo.h and SerialTypes.h headers (Jon Siwek, Corelight) diff --git a/NEWS b/NEWS index 197419c97d..85c01507a4 100644 --- a/NEWS +++ b/NEWS @@ -418,6 +418,9 @@ Removed Functionality - ``log_encryption_key`` - ``state_dir`` - ``state_write_delay`` + - ``ssl_ca_certificate`` + - ``ssl_private_key`` + - ``ssl_passphrase`` - The following constants were used as part of deprecated functionality in version 2.6 or below and are removed from this release: diff --git a/VERSION b/VERSION index 6aee059b51..97bdea11c7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-477 +2.6-478 diff --git a/doc b/doc index e5b95022ff..a5f2286834 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit e5b95022ffa68ddf4645228d123cf1ea73a55186 +Subproject commit a5f2286834e404df5eb8291fe078732c6b5763ab diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index f68bf3a545..1a57375d4c 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4710,22 +4710,6 @@ const report_gaps_for_partial = F &redef; ## controlled for reproducing results. const exit_only_after_terminate = F &redef; -## The CA certificate file to authorize remote Zeeks/Broccolis. -## -## .. zeek:see:: ssl_private_key ssl_passphrase -const ssl_ca_certificate = "" &redef; - -## File containing our private key and our certificate. -## -## .. zeek:see:: ssl_ca_certificate ssl_passphrase -const ssl_private_key = "" &redef; - -## The passphrase for our private key. Keeping this undefined -## causes Zeek to prompt for the passphrase. -## -## .. zeek:see:: ssl_private_key ssl_ca_certificate -const ssl_passphrase = "" &redef; - ## Default mode for Zeek's user-space dynamic packet filter. If true, packets ## that aren't explicitly allowed through, are dropped from any further ## processing. diff --git a/src/NetVar.cc b/src/NetVar.cc index b9230bece7..3ed77ea277 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -165,10 +165,6 @@ StringVal* log_rotate_base_time; StringVal* peer_description; bro_uint_t chunked_io_buffer_soft_cap; -StringVal* ssl_ca_certificate; -StringVal* ssl_private_key; -StringVal* ssl_passphrase; - Val* profiling_file; double profiling_interval; int expensive_profiling_multiple; @@ -244,10 +240,6 @@ void init_general_global_var() internal_val("peer_description")->AsStringVal(); chunked_io_buffer_soft_cap = opt_internal_unsigned("chunked_io_buffer_soft_cap"); - ssl_ca_certificate = internal_val("ssl_ca_certificate")->AsStringVal(); - ssl_private_key = internal_val("ssl_private_key")->AsStringVal(); - ssl_passphrase = internal_val("ssl_passphrase")->AsStringVal(); - packet_filter_default = opt_internal_int("packet_filter_default"); sig_max_group_size = opt_internal_int("sig_max_group_size"); diff --git a/src/NetVar.h b/src/NetVar.h index 9fa4d75fa6..cbb08a1306 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -168,10 +168,6 @@ extern StringVal* log_rotate_base_time; extern StringVal* peer_description; extern bro_uint_t chunked_io_buffer_soft_cap; -extern StringVal* ssl_ca_certificate; -extern StringVal* ssl_private_key; -extern StringVal* ssl_passphrase; - extern Val* profiling_file; extern double profiling_interval; extern int expensive_profiling_multiple; From aefd9322fdef9c1a34ef18de9fe720ddc73ae066 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 20 Jun 2019 18:31:58 -0700 Subject: [PATCH 19/22] Fix TableVal::DoClone to use CloneState cache --- CHANGES | 4 +++ VERSION | 2 +- src/Val.cc | 2 +- src/Val.h | 7 +++-- testing/btest/Baseline/language.copy/out | 3 +++ testing/btest/language/copy.zeek | 33 +++++++++++++++++++----- 6 files changed, 38 insertions(+), 13 deletions(-) diff --git a/CHANGES b/CHANGES index 115d0ef359..516943f29d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.6-479 | 2019-06-20 18:31:58 -0700 + + * Fix TableVal::DoClone to use CloneState cache (Jon Siwek, Corelight) + 2.6-478 | 2019-06-20 14:19:11 -0700 * Remove old Broccoli SSL options (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index 97bdea11c7..6802e60598 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-478 +2.6-479 diff --git a/src/Val.cc b/src/Val.cc index 409268d801..04e08feb61 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2181,7 +2181,7 @@ Val* TableVal::DoClone(CloneState* state) TableEntryVal* val; while ( (val = tbl->NextEntry(key, cookie)) ) { - TableEntryVal* nval = val->Clone(); + TableEntryVal* nval = val->Clone(state); tv->AsNonConstTable()->Insert(key, nval); if ( subnets ) diff --git a/src/Val.h b/src/Val.h index 8168113acd..4d76ccc804 100644 --- a/src/Val.h +++ b/src/Val.h @@ -3,8 +3,6 @@ #ifndef val_h #define val_h -// BRO values. - #include #include #include @@ -374,6 +372,7 @@ protected: friend class RecordVal; friend class VectorVal; friend class ValManager; + friend class TableEntryVal; virtual void ValDescribe(ODesc* d) const; virtual void ValDescribeReST(ODesc* d) const; @@ -804,9 +803,9 @@ public: int(network_time - bro_start_network_time); } - TableEntryVal* Clone() + TableEntryVal* Clone(Val::CloneState* state) { - auto rval = new TableEntryVal(val ? val->Clone() : nullptr); + auto rval = new TableEntryVal(val ? val->Clone(state) : nullptr); rval->last_access_time = last_access_time; rval->expire_access_time = expire_access_time; rval->last_read_update = last_read_update; diff --git a/testing/btest/Baseline/language.copy/out b/testing/btest/Baseline/language.copy/out index 675d38aa5d..fbc2c4b04d 100644 --- a/testing/btest/Baseline/language.copy/out +++ b/testing/btest/Baseline/language.copy/out @@ -1,2 +1,5 @@ direct assignment (PASS) using copy (PASS) +F, T +F, T +[a=42], [a=42], [a=42], [a=42] diff --git a/testing/btest/language/copy.zeek b/testing/btest/language/copy.zeek index 9ac1e577ea..638976295d 100644 --- a/testing/btest/language/copy.zeek +++ b/testing/btest/language/copy.zeek @@ -2,14 +2,12 @@ # @TEST-EXEC: btest-diff out function test_case(msg: string, expect: bool) - { - print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL"); - } - - + { + print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL"); + } event zeek_init() -{ + { # "b" is not a copy of "a" local a: set[string] = set("this", "test"); local b: set[string] = a; @@ -25,6 +23,27 @@ event zeek_init() delete c["this"]; test_case( "using copy", |d| == 2 && "this" in d); + } -} +type myrec: record { + a: count; +}; + +event zeek_init() + { + local v: vector of myrec; + local t: table[count] of myrec; + local mr = myrec($a = 42); + + t[0] = mr; + t[1] = mr; + local tc = copy(t); + print same_object(t, tc), same_object(tc[0], tc[1]); + + v[0] = mr; + v[1] = mr; + local vc = copy(v); + print same_object(v, vc), same_object(vc[0], vc[1]); + print tc[0], tc[1], vc[0], vc[1]; + } From 8f19bbe589ea1d2f056e01c9cf61a5fd080eaa84 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 20 Jun 2019 19:50:23 -0700 Subject: [PATCH 20/22] Improve C++ header includes to improve build time Recent changes ended up including all the Broker headers more places than necessary, causing compile time to increase 2x. --- src/OpaqueVal.cc | 2 ++ src/OpaqueVal.h | 3 +++ src/Val.h | 2 -- src/broker/Data.cc | 3 +++ src/broker/Data.h | 4 +++- src/broker/Manager.h | 12 +++++++++++- src/broker/Store.h | 4 +++- src/file_analysis/analyzer/x509/X509.cc | 2 ++ src/probabilistic/BitVector.h | 3 ++- src/probabilistic/BloomFilter.cc | 2 ++ src/probabilistic/BloomFilter.h | 3 ++- src/probabilistic/CardinalityCounter.h | 6 +++++- src/probabilistic/CounterVector.cc | 3 +++ src/probabilistic/CounterVector.h | 3 ++- src/probabilistic/Hasher.h | 5 ++++- src/probabilistic/Topk.cc | 2 ++ 16 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index 4d7c66b431..93adc5ca06 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -6,6 +6,8 @@ #include "probabilistic/BloomFilter.h" #include "probabilistic/CardinalityCounter.h" +#include + // Helper to retrieve a broker value out of a broker::vector at a specified // index, and casted to the expected destination type. template diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 17ee78e2fd..3e1b91f0ab 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -3,6 +3,9 @@ #ifndef OPAQUEVAL_H #define OPAQUEVAL_H +#include +#include + #include "RandTest.h" #include "Val.h" #include "digest.h" diff --git a/src/Val.h b/src/Val.h index 4d76ccc804..5bc5df4da9 100644 --- a/src/Val.h +++ b/src/Val.h @@ -8,8 +8,6 @@ #include #include -#include - #include "net_util.h" #include "Type.h" #include "Dict.h" diff --git a/src/broker/Data.cc b/src/broker/Data.cc index 966fed6426..d2e53fe45b 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -1,6 +1,9 @@ #include "Data.h" #include "File.h" #include "broker/data.bif.h" + +#include + #include #include #include diff --git a/src/broker/Data.h b/src/broker/Data.h index eda8f6550c..b134656123 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -1,7 +1,9 @@ #ifndef BRO_COMM_DATA_H #define BRO_COMM_DATA_H -#include +#include +#include + #include "OpaqueVal.h" #include "Reporter.h" #include "Frame.h" diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 5dfd2eb235..569355b533 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -1,7 +1,17 @@ #ifndef BRO_COMM_MANAGER_H #define BRO_COMM_MANAGER_H -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include diff --git a/src/broker/Store.h b/src/broker/Store.h index 7e5b2bde07..46d19ee923 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -8,7 +8,9 @@ #include "OpaqueVal.h" #include "Trigger.h" -#include +#include +#include +#include namespace bro_broker { diff --git a/src/file_analysis/analyzer/x509/X509.cc b/src/file_analysis/analyzer/x509/X509.cc index 74b903f585..33f2cb4d07 100644 --- a/src/file_analysis/analyzer/x509/X509.cc +++ b/src/file_analysis/analyzer/x509/X509.cc @@ -10,6 +10,8 @@ #include "file_analysis/Manager.h" +#include + #include #include #include diff --git a/src/probabilistic/BitVector.h b/src/probabilistic/BitVector.h index 12d628cacf..ecec6f5714 100644 --- a/src/probabilistic/BitVector.h +++ b/src/probabilistic/BitVector.h @@ -6,7 +6,8 @@ #include #include -#include +#include +#include namespace probabilistic { diff --git a/src/probabilistic/BloomFilter.cc b/src/probabilistic/BloomFilter.cc index f449fad8b6..dd89bf9c19 100644 --- a/src/probabilistic/BloomFilter.cc +++ b/src/probabilistic/BloomFilter.cc @@ -4,6 +4,8 @@ #include #include +#include + #include "BloomFilter.h" #include "CounterVector.h" diff --git a/src/probabilistic/BloomFilter.h b/src/probabilistic/BloomFilter.h index 6f2362de44..bc22c91014 100644 --- a/src/probabilistic/BloomFilter.h +++ b/src/probabilistic/BloomFilter.h @@ -5,7 +5,8 @@ #include -#include +#include +#include #include "BitVector.h" #include "Hasher.h" diff --git a/src/probabilistic/CardinalityCounter.h b/src/probabilistic/CardinalityCounter.h index a2d69d0809..63047172ed 100644 --- a/src/probabilistic/CardinalityCounter.h +++ b/src/probabilistic/CardinalityCounter.h @@ -4,7 +4,11 @@ #define PROBABILISTIC_CARDINALITYCOUNTER_H #include -#include +#include +#include + +#include +#include namespace probabilistic { diff --git a/src/probabilistic/CounterVector.cc b/src/probabilistic/CounterVector.cc index a847e06ea7..b9a173356e 100644 --- a/src/probabilistic/CounterVector.cc +++ b/src/probabilistic/CounterVector.cc @@ -5,6 +5,9 @@ #include #include #include "BitVector.h" +#include "util.h" + +#include using namespace probabilistic; diff --git a/src/probabilistic/CounterVector.h b/src/probabilistic/CounterVector.h index 41674efd11..f8209fabca 100644 --- a/src/probabilistic/CounterVector.h +++ b/src/probabilistic/CounterVector.h @@ -6,7 +6,8 @@ #include #include -#include +#include +#include namespace probabilistic { diff --git a/src/probabilistic/Hasher.h b/src/probabilistic/Hasher.h index 3218ec4d7a..3d60a264c0 100644 --- a/src/probabilistic/Hasher.h +++ b/src/probabilistic/Hasher.h @@ -3,7 +3,10 @@ #ifndef PROBABILISTIC_HASHER_H #define PROBABILISTIC_HASHER_H -#include +#include +#include + +#include #include "Hash.h" diff --git a/src/probabilistic/Topk.cc b/src/probabilistic/Topk.cc index 56b9030f21..8ff158e10d 100644 --- a/src/probabilistic/Topk.cc +++ b/src/probabilistic/Topk.cc @@ -1,5 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include + #include "broker/Data.h" #include "probabilistic/Topk.h" #include "CompHash.h" From a5e03d25d494571d04522aad917b8edfbbe3bacc Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 20 Jun 2019 15:00:33 -0700 Subject: [PATCH 21/22] GH-171: support warning messages alongside deprecated attributes --- src/Expr.cc | 19 ++-- src/ID.cc | 24 ++++- src/ID.h | 4 +- src/Type.cc | 41 +++++++-- src/Type.h | 14 ++- src/Var.cc | 17 ++-- src/parse.y | 25 +++-- src/plugin/ComponentManager.h | 2 +- .../language.deprecated/no-warnings.out | 28 ++++++ .../btest/Baseline/language.deprecated/out | 28 ------ .../Baseline/language.deprecated/warnings.out | 28 ++++++ testing/btest/language/deprecated.zeek | 91 ++++++++++++++++++- 12 files changed, 247 insertions(+), 74 deletions(-) create mode 100644 testing/btest/Baseline/language.deprecated/no-warnings.out delete mode 100644 testing/btest/Baseline/language.deprecated/out create mode 100644 testing/btest/Baseline/language.deprecated/warnings.out diff --git a/src/Expr.cc b/src/Expr.cc index e19536359a..3a4ce6aed7 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2884,9 +2884,8 @@ FieldExpr::FieldExpr(Expr* arg_op, const char* arg_field_name) SetType(rt->FieldType(field)->Ref()); td = rt->FieldDecl(field); - if ( td->FindAttr(ATTR_DEPRECATED) ) - reporter->Warning("deprecated (%s$%s)", rt->GetName().c_str(), - field_name); + if ( rt->IsFieldDeprecated(field) ) + reporter->Warning("%s", rt->GetFieldDeprecationWarning(field, false).c_str()); } } } @@ -2975,9 +2974,8 @@ HasFieldExpr::HasFieldExpr(Expr* arg_op, const char* arg_field_name) if ( field < 0 ) ExprError("no such field in record"); - else if ( rt->FieldDecl(field)->FindAttr(ATTR_DEPRECATED) ) - reporter->Warning("deprecated (%s?$%s)", rt->GetName().c_str(), - field_name); + else if ( rt->IsFieldDeprecated(field) ) + reporter->Warning("%s", rt->GetFieldDeprecationWarning(field, true).c_str()); SetType(base_type(TYPE_BOOL)); } @@ -3659,13 +3657,8 @@ RecordCoerceExpr::RecordCoerceExpr(Expr* op, RecordType* r) break; } } - else - { - if ( t_r->FieldDecl(i)->FindAttr(ATTR_DEPRECATED) ) - reporter->Warning("deprecated (%s$%s)", - t_r->GetName().c_str(), - t_r->FieldName(i)); - } + else if ( t_r->IsFieldDeprecated(i) ) + reporter->Warning("%s", t_r->GetFieldDeprecationWarning(i, false).c_str()); } } } diff --git a/src/ID.cc b/src/ID.cc index 12677bec75..5a8944f035 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -219,15 +219,35 @@ void ID::UpdateValAttrs() } } -void ID::MakeDeprecated() +void ID::MakeDeprecated(Expr* deprecation) { if ( IsDeprecated() ) return; - attr_list* attr = new attr_list{new Attr(ATTR_DEPRECATED)}; + attr_list* attr = new attr_list{new Attr(ATTR_DEPRECATED, deprecation)}; AddAttrs(new Attributes(attr, Type(), false)); } +string ID::GetDeprecationWarning() const + { + string result; + Attr* depr_attr = FindAttr(ATTR_DEPRECATED); + if ( depr_attr ) + { + ConstExpr* expr = static_cast(depr_attr->AttrExpr()); + if ( expr ) + { + StringVal* text = expr->Value()->AsStringVal(); + result = text->CheckString(); + } + } + + if ( result.empty() ) + return fmt("deprecated (%s)", Name()); + else + return fmt("deprecated (%s): %s", Name(), result.c_str()); + } + void ID::AddAttrs(Attributes* a) { if ( attrs ) diff --git a/src/ID.h b/src/ID.h index bb9e11ca06..405739476e 100644 --- a/src/ID.h +++ b/src/ID.h @@ -86,7 +86,9 @@ public: bool IsDeprecated() const { return FindAttr(ATTR_DEPRECATED) != 0; } - void MakeDeprecated(); + void MakeDeprecated(Expr* deprecation); + + string GetDeprecationWarning() const; void Error(const char* msg, const BroObj* o2 = 0); diff --git a/src/Type.cc b/src/Type.cc index 7b65dd0ffa..60461e026f 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -985,6 +985,33 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const d->PopIndentNoNL(); } +string RecordType::GetFieldDeprecationWarning(int field, bool has_check) const + { + const TypeDecl* decl = FieldDecl(field); + if ( decl) + { + string result; + if ( const Attr* deprecation = decl->FindAttr(ATTR_DEPRECATED) ) + { + ConstExpr* expr = static_cast(deprecation->AttrExpr()); + if ( expr ) + { + StringVal* text = expr->Value()->AsStringVal(); + result = text->CheckString(); + } + } + + if ( result.empty() ) + return fmt("deprecated (%s%s$%s)", GetName().c_str(), has_check ? "?" : "", + FieldName(field)); + else + return fmt("deprecated (%s%s$%s): %s", GetName().c_str(), has_check ? "?" : "", + FieldName(field), result.c_str()); + } + + return ""; + } + SubNetType::SubNetType() : BroType(TYPE_SUBNET) { } @@ -1083,7 +1110,7 @@ EnumType::~EnumType() // Note, we use reporter->Error() here (not Error()) to include the current script // location in the error message, rather than the one where the type was // originally defined. -void EnumType::AddName(const string& module_name, const char* name, bool is_export, bool deprecated) +void EnumType::AddName(const string& module_name, const char* name, bool is_export, Expr* deprecation) { /* implicit, auto-increment */ if ( counter < 0) @@ -1092,11 +1119,11 @@ void EnumType::AddName(const string& module_name, const char* name, bool is_expo SetError(); return; } - CheckAndAddName(module_name, name, counter, is_export, deprecated); + CheckAndAddName(module_name, name, counter, is_export, deprecation); counter++; } -void EnumType::AddName(const string& module_name, const char* name, bro_int_t val, bool is_export, bool deprecated) +void EnumType::AddName(const string& module_name, const char* name, bro_int_t val, bool is_export, Expr* deprecation) { /* explicit value specified */ if ( counter > 0 ) @@ -1106,11 +1133,11 @@ void EnumType::AddName(const string& module_name, const char* name, bro_int_t va return; } counter = -1; - CheckAndAddName(module_name, name, val, is_export, deprecated); + CheckAndAddName(module_name, name, val, is_export, deprecation); } void EnumType::CheckAndAddName(const string& module_name, const char* name, - bro_int_t val, bool is_export, bool deprecated) + bro_int_t val, bool is_export, Expr* deprecation) { if ( Lookup(val) ) { @@ -1127,8 +1154,8 @@ void EnumType::CheckAndAddName(const string& module_name, const char* name, id->SetType(this->Ref()); id->SetEnumConst(); - if ( deprecated ) - id->MakeDeprecated(); + if ( deprecation ) + id->MakeDeprecated(deprecation); zeekygen_mgr->Identifier(id); } diff --git a/src/Type.h b/src/Type.h index 043ec5c928..78c56f99e8 100644 --- a/src/Type.h +++ b/src/Type.h @@ -494,6 +494,14 @@ public: void DescribeFields(ODesc* d) const; void DescribeFieldsReST(ODesc* d, bool func_args) const; + bool IsFieldDeprecated(int field) const + { + const TypeDecl* decl = FieldDecl(field); + return decl && decl->FindAttr(ATTR_DEPRECATED) != 0; + } + + string GetFieldDeprecationWarning(int field, bool has_check) const; + protected: RecordType() { types = 0; } @@ -551,12 +559,12 @@ public: // The value of this name is next internal counter value, starting // with zero. The internal counter is incremented. - void AddName(const string& module_name, const char* name, bool is_export, bool deprecated); + void AddName(const string& module_name, const char* name, bool is_export, Expr* deprecation = nullptr); // The value of this name is set to val. Once a value has been // explicitly assigned using this method, no further names can be // added that aren't likewise explicitly initalized. - void AddName(const string& module_name, const char* name, bro_int_t val, bool is_export, bool deprecated); + void AddName(const string& module_name, const char* name, bro_int_t val, bool is_export, Expr* deprecation = nullptr); // -1 indicates not found. bro_int_t Lookup(const string& module_name, const char* name) const; @@ -578,7 +586,7 @@ protected: void CheckAndAddName(const string& module_name, const char* name, bro_int_t val, bool is_export, - bool deprecated); + Expr* deprecation = nullptr); typedef std::map NameMap; NameMap names; diff --git a/src/Var.cc b/src/Var.cc index dcdd1cc279..74cfee291f 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -312,16 +312,21 @@ static void transfer_arg_defaults(RecordType* args, RecordType* recv) } } -static bool has_attr(const attr_list* al, attr_tag tag) +static Attr* find_attr(const attr_list* al, attr_tag tag) { if ( ! al ) - return false; + return nullptr; for ( int i = 0; i < al->length(); ++i ) if ( (*al)[i]->Tag() == tag ) - return true; + return (*al)[i]; - return false; + return nullptr; + } + +static bool has_attr(const attr_list* al, attr_tag tag) + { + return find_attr(al, tag) != nullptr; } void begin_func(ID* id, const char* module_name, function_flavor flavor, @@ -398,8 +403,8 @@ void begin_func(ID* id, const char* module_name, function_flavor flavor, arg_id->SetType(arg_i->type->Ref()); } - if ( has_attr(attrs, ATTR_DEPRECATED) ) - id->MakeDeprecated(); + if ( Attr* depr_attr = find_attr(attrs, ATTR_DEPRECATED) ) + id->MakeDeprecated(depr_attr->AttrExpr()); } class OuterIDBindingFinder : public TraversalCallback { diff --git a/src/parse.y b/src/parse.y index 48c8143f35..9ab7dd5e66 100644 --- a/src/parse.y +++ b/src/parse.y @@ -5,7 +5,7 @@ // Switching parser table type fixes ambiguity problems. %define lr.type ielr -%expect 104 +%expect 106 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -50,14 +50,14 @@ %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR %nonassoc TOK_AS TOK_IS -%type opt_no_test opt_no_test_block opt_deprecated TOK_PATTERN_END +%type opt_no_test opt_no_test_block TOK_PATTERN_END %type TOK_ID TOK_PATTERN_TEXT %type local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func case_type %type local_id_list case_type_list %type init_class %type opt_init %type TOK_CONSTANT -%type expr opt_expr init anonymous_function index_slice +%type expr opt_expr init anonymous_function index_slice opt_deprecated %type event %type stmt stmt_list func_body for_head %type type opt_type enum_body @@ -699,8 +699,8 @@ expr: else $$ = new NameExpr(id); - if ( id->IsDeprecated() ) - reporter->Warning("deprecated (%s)", id->Name()); + if ( Attr* depr_attr = id->FindAttr(ATTR_DEPRECATED) ) + reporter->Warning("%s", id->GetDeprecationWarning().c_str()); } } @@ -1002,7 +1002,7 @@ type: Ref($$); if ( $1->IsDeprecated() ) - reporter->Warning("deprecated (%s)", $1->Name()); + reporter->Warning("%s", $1->GetDeprecationWarning().c_str()); } } ; @@ -1327,6 +1327,8 @@ attr: { $$ = new Attr(ATTR_ERROR_HANDLER); } | TOK_ATTR_DEPRECATED { $$ = new Attr(ATTR_DEPRECATED); } + | TOK_ATTR_DEPRECATED '=' expr + { $$ = new Attr(ATTR_DEPRECATED, $3); } ; stmt: @@ -1535,7 +1537,7 @@ event: YYERROR; } if ( id->IsDeprecated() ) - reporter->Warning("deprecated (%s)", id->Name()); + reporter->Warning("%s", id->GetDeprecationWarning().c_str()); } $$ = new EventExpr($1, $3); @@ -1741,7 +1743,7 @@ global_or_event_id: if ( t->Tag() != TYPE_FUNC || t->AsFuncType()->Flavor() != FUNC_FLAVOR_FUNCTION ) - reporter->Warning("deprecated (%s)", $$->Name()); + reporter->Warning("%s", $$->GetDeprecationWarning().c_str()); } delete [] $1; @@ -1787,9 +1789,12 @@ opt_no_test_block: opt_deprecated: TOK_ATTR_DEPRECATED - { $$ = true; } + { $$ = new ConstExpr(new StringVal("")); } | - { $$ = false; } + TOK_ATTR_DEPRECATED '=' expr + { $$ = $3; } + | + { $$ = nullptr; } %% diff --git a/src/plugin/ComponentManager.h b/src/plugin/ComponentManager.h index 399c704551..30b3628588 100644 --- a/src/plugin/ComponentManager.h +++ b/src/plugin/ComponentManager.h @@ -244,7 +244,7 @@ void ComponentManager::RegisterComponent(C* component, string id = fmt("%s%s", prefix.c_str(), cname.c_str()); tag_enum_type->AddName(module, id.c_str(), component->Tag().AsEnumVal()->InternalInt(), true, - false); + nullptr); } } // namespace plugin diff --git a/testing/btest/Baseline/language.deprecated/no-warnings.out b/testing/btest/Baseline/language.deprecated/no-warnings.out new file mode 100644 index 0000000000..42930b1690 --- /dev/null +++ b/testing/btest/Baseline/language.deprecated/no-warnings.out @@ -0,0 +1,28 @@ +warning in ./no-warnings.zeek, line 27: deprecated (ONE) +warning in ./no-warnings.zeek, line 28: deprecated (TWO) +warning in ./no-warnings.zeek, line 30: deprecated (GREEN) +warning in ./no-warnings.zeek, line 31: deprecated (BLUE) +warning in ./no-warnings.zeek, line 33: deprecated (blah) +warning in ./no-warnings.zeek, line 37: deprecated (my_event) +warning in ./no-warnings.zeek, line 38: deprecated (my_event) +warning in ./no-warnings.zeek, line 39: deprecated (my_hook) +warning in ./no-warnings.zeek, line 41: deprecated (my_record$b) +warning in ./no-warnings.zeek, line 42: deprecated (my_record$b) +warning in ./no-warnings.zeek, line 43: deprecated (my_record$b) +warning in ./no-warnings.zeek, line 45: deprecated (my_record?$b) +warning in ./no-warnings.zeek, line 46: deprecated (my_record$b) +warning in ./no-warnings.zeek, line 49: deprecated (my_record$b) +warning in ./no-warnings.zeek, line 52: deprecated (my_event) +warning in ./no-warnings.zeek, line 57: deprecated (my_hook) +warning in ./no-warnings.zeek, line 62: deprecated (blah) +warning in ./no-warnings.zeek, line 71: deprecated (dont_use_me) +warning in ./no-warnings.zeek, line 76: deprecated (dont_use_me_either) +ZERO +ONE +TWO +RED +GREEN +BLUE +generate my_hook please +generate my_event please +schedule my_event please diff --git a/testing/btest/Baseline/language.deprecated/out b/testing/btest/Baseline/language.deprecated/out deleted file mode 100644 index 3126b1e78b..0000000000 --- a/testing/btest/Baseline/language.deprecated/out +++ /dev/null @@ -1,28 +0,0 @@ -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 30: deprecated (ONE) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 31: deprecated (TWO) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 33: deprecated (GREEN) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 34: deprecated (BLUE) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 36: deprecated (blah) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 40: deprecated (my_event) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 41: deprecated (my_event) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 42: deprecated (my_hook) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 44: deprecated (my_record$b) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 45: deprecated (my_record$b) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 46: deprecated (my_record$b) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 48: deprecated (my_record?$b) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 49: deprecated (my_record$b) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 52: deprecated (my_record$b) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 55: deprecated (my_event) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 60: deprecated (my_hook) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 65: deprecated (blah) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 74: deprecated (dont_use_me) -warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.deprecated/deprecated.zeek, line 79: deprecated (dont_use_me_either) -ZERO -ONE -TWO -RED -GREEN -BLUE -generate my_hook please -generate my_event please -schedule my_event please diff --git a/testing/btest/Baseline/language.deprecated/warnings.out b/testing/btest/Baseline/language.deprecated/warnings.out new file mode 100644 index 0000000000..5f0ee7bdc8 --- /dev/null +++ b/testing/btest/Baseline/language.deprecated/warnings.out @@ -0,0 +1,28 @@ +warning in ./warnings.zeek, line 27: deprecated (ONE): one warning +warning in ./warnings.zeek, line 28: deprecated (TWO): two warning +warning in ./warnings.zeek, line 30: deprecated (GREEN): green warning +warning in ./warnings.zeek, line 31: deprecated (BLUE): red warning +warning in ./warnings.zeek, line 33: deprecated (blah): type warning +warning in ./warnings.zeek, line 37: deprecated (my_event): event warning +warning in ./warnings.zeek, line 38: deprecated (my_event): event warning +warning in ./warnings.zeek, line 39: deprecated (my_hook): hook warning +warning in ./warnings.zeek, line 41: deprecated (my_record$b): record warning +warning in ./warnings.zeek, line 42: deprecated (my_record$b): record warning +warning in ./warnings.zeek, line 43: deprecated (my_record$b): record warning +warning in ./warnings.zeek, line 45: deprecated (my_record?$b): record warning +warning in ./warnings.zeek, line 46: deprecated (my_record$b): record warning +warning in ./warnings.zeek, line 49: deprecated (my_record$b): record warning +warning in ./warnings.zeek, line 52: deprecated (my_event): event warning +warning in ./warnings.zeek, line 57: deprecated (my_hook): hook warning +warning in ./warnings.zeek, line 62: deprecated (blah): type warning +warning in ./warnings.zeek, line 71: deprecated (dont_use_me): global function warning +warning in ./warnings.zeek, line 76: deprecated (dont_use_me_either): function warning +ZERO +ONE +TWO +RED +GREEN +BLUE +generate my_hook please +generate my_event please +schedule my_event please diff --git a/testing/btest/language/deprecated.zeek b/testing/btest/language/deprecated.zeek index 6e10d7d744..b10b5674d3 100644 --- a/testing/btest/language/deprecated.zeek +++ b/testing/btest/language/deprecated.zeek @@ -1,6 +1,10 @@ -# @TEST-EXEC: zeek -b %INPUT >out 2>&1 -# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: zeek -b no-warnings.zeek >no-warnings.out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff no-warnings.out +# @TEST-EXEC: zeek -b warnings.zeek >warnings.out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff warnings.out + +@TEST-START-FILE no-warnings.zeek type blah: string &deprecated; global my_event: event(arg: string) &deprecated; @@ -21,7 +25,7 @@ type my_enum: enum { type my_other_enum: enum { ZERO = 0, ONE = 1 &deprecated, - TWO = 2 &deprecated + TWO = 2 &deprecated, }; event zeek_init() @@ -78,3 +82,84 @@ function dont_use_me_either() &deprecated { dont_use_me_either(); } +@TEST-END-FILE + +@TEST-START-FILE warnings.zeek +type blah: string &deprecated="type warning"; + +global my_event: event(arg: string) &deprecated="event warning"; + +global my_hook: hook(arg: string) &deprecated="hook warning"; + +type my_record: record { + a: count &default = 1; + b: string &optional &deprecated="record warning"; +}; + +type my_enum: enum { + RED, + GREEN &deprecated="green warning", + BLUE &deprecated="red warning" +}; + +type my_other_enum: enum { + ZERO = 0, + ONE = 1 &deprecated="one warning", + TWO = 2 &deprecated="two warning", +}; + +event zeek_init() + { + print ZERO; + print ONE; + print TWO; + print RED; + print GREEN; + print BLUE; + + local l: blah = "testing"; + + local ls: string = " test"; + + event my_event("generate my_event please"); + schedule 1sec { my_event("schedule my_event please") }; + hook my_hook("generate my_hook please"); + + local mr = my_record($a = 3, $b = "yeah"); + mr = [$a = 4, $b = "ye"]; + mr = record($a = 5, $b = "y"); + + if ( ! mr?$b ) + mr$b = "nooooooo"; + + mr$a = 2; + mr$b = "noooo"; + } + +event my_event(arg: string) + { + print arg; + } + +hook my_hook(arg: string) + { + print arg; + } + +function hmm(b: blah) + { + print b; + } + +global dont_use_me: function() &deprecated="global function warning"; + +function dont_use_me() + { + dont_use_me(); + } + +function dont_use_me_either() &deprecated="function warning" + { + dont_use_me_either(); + } +@TEST-END-FILE From 6918e5493664fd420d02dcc4ec3e2c4cfc0f5961 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 21 Jun 2019 09:56:57 -0700 Subject: [PATCH 22/22] Updating submodule(s). [nomail] --- aux/zeekctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/zeekctl b/aux/zeekctl index 67c8f308ee..b6642b80f1 160000 --- a/aux/zeekctl +++ b/aux/zeekctl @@ -1 +1 @@ -Subproject commit 67c8f308eeeaf49b3f79a4dfd363b07644f45d40 +Subproject commit b6642b80f1436751884e0dd12fa16930fabc66ca