Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-04 23:58:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/topic/bernhard/topk' into topic/robin/topk-merge

Browse source 
* origin/topic/bernhard/topk:
  update documentation, rename get* to Get* and make hasher persistent

Conflicts:
	src/probabilistic/Topk.cc
	src/probabilistic/Topk.h
	src/probabilistic/top-k.bif
This commit is contained in:
Robin Sommer 2013-08-01 14:13:25 -07:00
commit f6e5de91fa
4 changed files with 178 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

54
src/probabilistic/Topk.cc
View file

@ -19,19 +19,22 @@ static void topk_element_hash_delete_func(void* val)
Element::~Element() Element::~Element()
{ {
Unref(value); Unref(value);
value = 0; }
void TopkVal::Typify(BroType* t)
{
assert(!hash && !type);
type = t->Ref();
TypeList* tl = new TypeList(t);
tl->Append(t->Ref());
hash = new CompositeHash(tl);
Unref(tl);
} }
HashKey* TopkVal::GetHash(Val* v) const HashKey* TopkVal::GetHash(Val* v) const
{ {
TypeList* tl = new TypeList(v->Type()); HashKey* key = hash->ComputeHash(v, 1);
tl->Append(v->Type()->Ref());
CompositeHash* topk_hash = new CompositeHash(tl);
Unref(tl);
HashKey* key = topk_hash->ComputeHash(v, 1);
assert(key); assert(key);
delete topk_hash;
return key; return key;
} }
@ -43,6 +46,7 @@ TopkVal::TopkVal(uint64 arg_size) : OpaqueVal(topk_type)
type = 0; type = 0;
numElements = 0; numElements = 0;
pruned = false; pruned = false;
hash = 0;
} }
TopkVal::TopkVal() : OpaqueVal(topk_type) TopkVal::TopkVal() : OpaqueVal(topk_type)
@ -52,6 +56,7 @@ TopkVal::TopkVal() : OpaqueVal(topk_type)
size = 0; size = 0;
type = 0; type = 0;
numElements = 0; numElements = 0;
hash = 0;
} }
TopkVal::~TopkVal() TopkVal::~TopkVal()
@ -68,7 +73,7 @@ TopkVal::~TopkVal()
} }
Unref(type); Unref(type);
type = 0; delete hash;
} }
void TopkVal::Merge(const TopkVal* value, bool doPrune) void TopkVal::Merge(const TopkVal* value, bool doPrune)
@ -76,7 +81,7 @@ void TopkVal::Merge(const TopkVal* value, bool doPrune)
if ( type == 0 ) if ( type == 0 )
{ {
assert(numElements == 0); assert(numElements == 0);
type = value->type->Ref(); Typify(value->type);
} }
else else
@ -230,7 +235,10 @@ bool TopkVal::DoUnserialize(UnserialInfo* info)
v &= UNSERIALIZE(&type_present); v &= UNSERIALIZE(&type_present);
if ( type_present ) if ( type_present )
{ {
type = BroType::Unserialize(info); BroType* deserialized_type = BroType::Unserialize(info);
Typify(deserialized_type);
Unref(deserialized_type);
assert(type); assert(type);
} }
else else
@ -270,7 +278,7 @@ bool TopkVal::DoUnserialize(UnserialInfo* info)
} }
VectorVal* TopkVal::getTopK(int k) const // returns vector VectorVal* TopkVal::GetTopK(int k) const // returns vector
{ {
if ( numElements == 0 ) if ( numElements == 0 )
{ {
@ -311,14 +319,14 @@ VectorVal* TopkVal::getTopK(int k) const // returns vector
return t; return t;
} }
uint64_t TopkVal::getCount(Val* value) const uint64_t TopkVal::GetCount(Val* value) const
{ {
HashKey* key = GetHash(value); HashKey* key = GetHash(value);
Element* e = (Element*) elementDict->Lookup(key); Element* e = (Element*) elementDict->Lookup(key);
if ( e == 0 ) if ( e == 0 )
{ {
reporter->Error("getCount for element that is not in top-k"); reporter->Error("GetCount for element that is not in top-k");
return 0; return 0;
} }
@ -326,14 +334,14 @@ uint64_t TopkVal::getCount(Val* value) const
return e->parent->count; return e->parent->count;
} }
uint64_t TopkVal::getEpsilon(Val* value) const uint64_t TopkVal::GetEpsilon(Val* value) const
{ {
HashKey* key = GetHash(value); HashKey* key = GetHash(value);
Element* e = (Element*) elementDict->Lookup(key); Element* e = (Element*) elementDict->Lookup(key);
if ( e == 0 ) if ( e == 0 )
{ {
reporter->Error("getEpsilon for element that is not in top-k"); reporter->Error("GetEpsilon for element that is not in top-k");
return 0; return 0;
} }
@ -341,7 +349,7 @@ uint64_t TopkVal::getEpsilon(Val* value) const
return e->epsilon; return e->epsilon;
} }
uint64_t TopkVal::getSum() const uint64_t TopkVal::GetSum() const
{ {
uint64_t sum = 0; uint64_t sum = 0;
@ -353,8 +361,8 @@ uint64_t TopkVal::getSum() const
it++; it++;
} }
if ( pruned ) if ( pruned )
reporter->Warning("TopkVal::getSum() was used on a pruned data structure. Result values do not represent total element count"); reporter->Warning("TopkVal::GetSum() was used on a pruned data structure. Result values do not represent total element count");
return sum; return sum;
} }
@ -362,11 +370,9 @@ uint64_t TopkVal::getSum() const
void TopkVal::Encountered(Val* encountered) void TopkVal::Encountered(Val* encountered)
{ {
// ok, let's see if we already know this one. // ok, let's see if we already know this one.
//printf("NumElements: %d\n", numElements); if ( numElements == 0 )
// check type compatibility Typify(encountered->Type());
if ( numElements == 0 )
type = encountered->Type()->Ref();
else else
if ( ! same_type(type, encountered->Type()) ) if ( ! same_type(type, encountered->Type()) )
{ {

106
src/probabilistic/Topk.h
View file

@ -38,51 +38,101 @@ declare(PDict, Element);
class TopkVal : public OpaqueVal { class TopkVal : public OpaqueVal {
public: public:
// Initialize a TopkVal. Size specifies how many total elements are /**
// tracked * Construct a TopkVal.
*
* @param size specifies how many total elements are tracked
*
* @return A newly initialized TopkVal
*/
TopkVal(uint64 size); TopkVal(uint64 size);
/**
* Destructor.
*/
~TopkVal(); ~TopkVal();
// Call this, when a new value is encountered. Note that on the first call, /**
// the Bro-Type of the value types that are counted is set. All following calls * Call this, when a new value is encountered. Note that on the first call,
// to encountered have to specify the same type * the Bro-Type of the value types that are counted is set. All following calls
void Encountered(Val* value); * to encountered have to specify the same type.
*
* @param value The encountered element
*/
void Encountered(Val* value);
// Return the first k elements of the result vector. At the moment, this does /**
// not check if it is in the right order or if we can prove that these are * Get the first k elements of the result vector. At the moment, this does
// the correct top-k. Use count and epsilon for this. * not check if it is in the right order or if we can prove that these are
VectorVal* getTopK(int k) const; // returns vector * the correct top-k. Use count and epsilon for this.
*
* @param k Number of top-elements to return
*
* @returns The top-k encountered elements
*/
VectorVal* GetTopK(int k) const;
// Get the current count tracked in the top-k data structure for a certain val. /**
// Returns 0 if the val is unknown (and logs the error to reporter) * Get the current count tracked in the top-k data structure for a certain val.
uint64_t getCount(Val* value) const; * Returns 0 if the val is unknown (and logs the error to reporter)
*
* @param value Bro value to get counts for
*
* @returns internal count for val, 0 if unknown
*/
uint64_t GetCount(Val* value) const;
// Get the current epsilon tracked in the top-k data structure for a certain val. /**
// Returns 0 if the val is unknown (and logs the error to reporter) * Get the current epsilon tracked in the top-k data structure for a certain val.
uint64_t getEpsilon(Val* value) const; *
* @param value Bro value to get epsilons for
*
* @returns the epsilon. Returns 0 if the val is unknown (and logs the error to reporter)
*/
uint64_t GetEpsilon(Val* value) const;
// Get the size set in the constructor /**
uint64_t getSize() const { return size; } * Get the size set in the constructor
*
* @returns size of the top-k structure
*/
uint64_t GetSize() const { return size; }
// Get the sum of all counts of all tracked elements. This is equal to the number /**
// of total observations up to this moment, if no elements were pruned from the data * Get the sum of all counts of all tracked elements. This is equal to the number
// structure. * of total observations up to this moment, if no elements were pruned from the data
uint64_t getSum() const; * structure.
*
* @returns sum of all counts
*/
uint64_t GetSum() const;
// Merge another top-k data structure in this one. /**
// doPrune specifies if the total count of elements is limited to size after * Merge another top-k data structure into this one.
// merging. * doPrune specifies if the total count of elements is limited to size after
// Please note, that pruning will invalidate the results of getSum. * merging.
* Please note, that pruning will invalidate the results of getSum.
*
* @param value TopkVal to merge into this TopkVal
*
* @param doPrune prune resulting TopkVal to size after merging
*/
void Merge(const TopkVal* value, bool doPrune=false); void Merge(const TopkVal* value, bool doPrune=false);
protected: protected:
TopkVal(); // for deserialize /**
* Construct an empty TopkVal.
* Only used for deserialization
*/
TopkVal();
private: private:
void IncrementCounter(Element* e, unsigned int count = 1); void IncrementCounter(Element* e, unsigned int count = 1);
HashKey* GetHash(Val*) const; // this probably should go somewhere else. HashKey* GetHash(Val*) const; // this probably should go somewhere else.
void Typify(BroType*);
BroType* type; BroType* type;
CompositeHash* hash;
std::list<Bucket*> buckets; std::list<Bucket*> buckets;
PDict(Element)* elementDict; PDict(Element)* elementDict;
uint64 size; // how many elements are we tracking? uint64 size; // how many elements are we tracking?

82
src/probabilistic/top-k.bif
View file

@ -11,6 +11,8 @@
## Creates a top-k data structure which tracks *size* elements. ## Creates a top-k data structure which tracks *size* elements.
## ##
## size: number of elements to track
##
## Returns: Opaque pointer to the data structure. ## Returns: Opaque pointer to the data structure.
function topk_init%(size: count%): opaque of topk function topk_init%(size: count%): opaque of topk
%{ %{
@ -18,9 +20,14 @@ function topk_init%(size: count%): opaque of topk
return v; return v;
%} %}
## Add a new observed object to the data structure. The first ## Add a new observed object to the data structure.
## added object sets the type of data tracked by the top-k data ##
## structure. All following values have to be of the same type ## .. note:: The first added object sets the type of data tracked by
## the top-k data structure. All following values have to be of the same type
##
## handle: the TopK handle
##
## value: observed value
function topk_add%(handle: opaque of topk, value: any%): any function topk_add%(handle: opaque of topk, value: any%): any
%{ %{
assert(handle); assert(handle);
@ -32,63 +39,86 @@ function topk_add%(handle: opaque of topk, value: any%): any
## Get the first k elements of the top-k data structure ## Get the first k elements of the top-k data structure
## ##
## handle: the TopK handle
##
## k: number of elements to return
##
## Returns: vector of the first k elements ## Returns: vector of the first k elements
function topk_get_top%(handle: opaque of topk, k: count%): any function topk_get_top%(handle: opaque of topk, k: count%): any
%{ %{
assert(handle); assert(handle);
probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle; probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle;
return h->getTopK(k); return h->GetTopK(k);
%} %}
## Get an overestimated count of how often value has been encountered. ## Get an overestimated count of how often value has been encountered.
## value has to be part of the currently tracked elements, otherwise ##
## 0 will be returned and an error message will be added to reporter. ## .. note:: value has to be part of the currently tracked elements, otherwise
## 0 will be returned and an error message will be added to reporter.
##
## handle: the TopK handle
##
## value: Value to look up count for.
## ##
## Returns: Overestimated number for how often the element has been encountered ## Returns: Overestimated number for how often the element has been encountered
function topk_count%(handle: opaque of topk, value: any%): count function topk_count%(handle: opaque of topk, value: any%): count
%{ %{
assert(handle); assert(handle);
probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle; probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle;
return new Val(h->getCount(value), TYPE_COUNT); return new Val(h->GetCount(value), TYPE_COUNT);
%} %}
## Get a the maximal overestimation for count. Same restrictiosn as for topk_count ## Get a the maximal overestimation for count.
## apply. ##
## .. note:: Same restrictiosn as for topk_count apply.
##
## handle: the TopK handle
##
## value: Value to look up epsilon for.
## ##
## Returns: Number which represents the maximal overesimation for the count of this element. ## Returns: Number which represents the maximal overesimation for the count of this element.
function topk_epsilon%(handle: opaque of topk, value: any%): count function topk_epsilon%(handle: opaque of topk, value: any%): count
%{ %{
assert(handle); assert(handle);
probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle; probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle;
return new Val(h->getEpsilon(value), TYPE_COUNT); return new Val(h->GetEpsilon(value), TYPE_COUNT);
%} %}
## Get the number of elements this data structure is supposed to track (given on init). ## Get the number of elements this data structure is supposed to track (given on init).
## Note that the actual number of elements in the data structure can be lower or higher
## than this. (higher due to non-pruned merges)
## ##
## Returns: size given during initialization ## .. note ::Note that the actual number of elements in the data structure can be lower
## or higher (due to non-pruned merges) than thiz
##
## handle: the TopK handle
##
## Returns: size given during initialization
function topk_size%(handle: opaque of topk%): count function topk_size%(handle: opaque of topk%): count
%{ %{
assert(handle); assert(handle);
probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle; probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle;
return new Val(h->getSize(), TYPE_COUNT); return new Val(h->GetSize(), TYPE_COUNT);
%} %}
## Get the sum of all counts of all elements in the data structure. Is equal to the number ## Get the sum of all counts of all elements in the data structure.
## of all inserted objects if the data structure never has been pruned. Do not use after ##
## calling topk_merge_prune (will throw a warning message if used afterwards) ## .. note:: This is equal to the number of all inserted objects if the data structure
## never has been pruned. Do not use after calling topk_merge_prune (will throw a
## warning message if used afterwards)
##
## handle: the TopK handle
## ##
## Returns: sum of all counts ## Returns: sum of all counts
function topk_sum%(handle: opaque of topk%): count function topk_sum%(handle: opaque of topk%): count
%{ %{
assert(handle); assert(handle);
probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle; probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle;
return new Val(h->getSum(), TYPE_COUNT); return new Val(h->GetSum(), TYPE_COUNT);
%} %}
## Merge the second topk data structure into the first. Does not remove any elements, the ## Merge the second topk data structure into the first.
## resulting data structure can be bigger than the maximum size given on initialization. ##
## .. note:: This does not remove any elements, the resulting data structure can
## be bigger than the maximum size given on initialization.
function topk_merge%(handle1: opaque of topk, handle2: opaque of topk%): any function topk_merge%(handle1: opaque of topk, handle2: opaque of topk%): any
%{ %{
assert(handle1); assert(handle1);
@ -103,9 +133,15 @@ function topk_merge%(handle1: opaque of topk, handle2: opaque of topk%): any
%} %}
## Merge the second topk data structure into the first and prunes the final data structure ## Merge the second topk data structure into the first and prunes the final data structure
## back to the size given on initialization. Use with care and only when being aware of the ## back to the size given on initialization.
## restrictions this imposed. Do not call topk_size or topk_add afterwards, results will ##
## probably not be what you expect. ## .. note:: Use with care and only when being aware of the restrictions this entails.
## Do not call topk_size or topk_add afterwards, results will probably not be what you
## expect.
##
## handle1: the TopK handle in which the second TopK structure is merged
##
## handle2: the TopK handle in which is merged into the first TopK structure
function topk_merge_prune%(handle1: opaque of topk, handle2: opaque of topk%): any function topk_merge_prune%(handle1: opaque of topk, handle2: opaque of topk%): any
%{ %{
assert(handle1); assert(handle1);

22
testing/btest/Baseline/bifs.topk/.stderr
View file

@ -1,11 +1,11 @@
error: getCount for element that is not in top-k error: GetCount for element that is not in top-k
error: getEpsilon for element that is not in top-k error: GetEpsilon for element that is not in top-k
error: getCount for element that is not in top-k error: GetCount for element that is not in top-k
error: getEpsilon for element that is not in top-k error: GetEpsilon for element that is not in top-k
error: getCount for element that is not in top-k error: GetCount for element that is not in top-k
error: getEpsilon for element that is not in top-k error: GetEpsilon for element that is not in top-k
error: getCount for element that is not in top-k error: GetCount for element that is not in top-k
error: getEpsilon for element that is not in top-k error: GetEpsilon for element that is not in top-k
warning: TopkVal::getSum() was used on a pruned data structure. Result values do not represent total element count warning: TopkVal::GetSum() was used on a pruned data structure. Result values do not represent total element count
error: getCount for element that is not in top-k error: GetCount for element that is not in top-k
error: getEpsilon for element that is not in top-k error: GetEpsilon for element that is not in top-k