Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-08 09:38:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

rework cardinality interface to use opaque.

Browse source 
I like it better...
This commit is contained in:
Bernhard Amann 2013-04-07 23:05:14 +02:00
parent fd51db1c89
commit 53d6f3aae7
4 changed files with 147 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/OpaqueVal.cc
View file

@ -2,6 +2,28 @@
#include "Reporter.h" #include "Reporter.h"
#include "Serializer.h" #include "Serializer.h"
CardinalityVal::CardinalityVal()
{
valid = false;
}
CardinalityVal::~CardinalityVal()
{
if ( valid && c )
delete c;
}
bool CardinalityVal::Init(CardinalityCounter* arg_c)
{
if ( valid )
return false;
valid = true;
c = arg_c;
return valid;
}
bool HashVal::IsValid() const bool HashVal::IsValid() const
{ {
return valid; return valid;

18
src/OpaqueVal.h
View file

@ -7,6 +7,24 @@
#include "Val.h" #include "Val.h"
#include "digest.h" #include "digest.h"
class CardinalityCounter;
class CardinalityVal: public OpaqueVal {
public:
CardinalityVal();
~CardinalityVal();
bool Init(CardinalityCounter*);
bool IsValid() const { return valid; };
CardinalityCounter* Get() { return c; };
private:
bool valid;
CardinalityCounter* c;
// DECLARE_SERIAL(CardinalityVal); Fixme?
};
class HashVal : public OpaqueVal { class HashVal : public OpaqueVal {
public: public:
virtual bool IsValid() const; virtual bool IsValid() const;

207
src/bro.bif
View file

@ -5650,44 +5650,36 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr
%%{ %%{
#include "HyperLogLog.h" #include "HyperLogLog.h"
static map<BroString, CardinalityCounter*> hll_counters;
BroString* convert_index_to_string(Val* index)
{
ODesc d;
index->Describe(&d);
BroString* s = new BroString(1, d.TakeBytes(), d.Len());
s->SetUseFreeToDelete(1);
return s;
}
%%} %%}
## Initializes the hash for the HyperLogLog cardinality counting algorithm. ## Initializes the hash for the HyperLogLog cardinality counting algorithm.
## It returns true if it was successful in creating a structure and false ## It returns true if it was successful in creating a structure and false
## if it wasn't. ## if it wasn't.
function hll_cardinality_init%(err: double,index: any%): bool function hll_cardinality_init%(err: double%): opaque of cardinality
%{ %{
BroString* s = convert_index_to_string(index); CardinalityCounter* c = new CardinalityCounter(err);
int status = 0; CardinalityVal* cv = new CardinalityVal();
if ( hll_counters.count(*s) < 1 ) if ( !c )
{ reporter->Error("Failed initialize Cardinality counter");
hll_counters[*s] = new CardinalityCounter(err); else
status = 1; cv->Init(c);
}
delete s; return cv;
return new Val(status, TYPE_BOOL);
%} %}
## Adds an element to the HyperLogLog data structure located at index. ## Adds an element to the HyperLogLog data structure located at index.
##elem->Type() to get the type of elem. ##elem->Type() to get the type of elem.
function hll_cardinality_add%(elem: any, index: any%): bool function hll_cardinality_add%(handle: opaque of cardinality, elem: any%): bool
%{ %{
BroString* s = convert_index_to_string(index); if ( !((CardinalityVal*) handle)->IsValid() ) {
reporter->Error("Need valid handle");
return new Val(0, TYPE_BOOL);
}
int status = 0; int status = 0;
uint64_t a = 123456; uint64_t a = 123456;
@ -5696,19 +5688,13 @@ function hll_cardinality_add%(elem: any, index: any%): bool
CompositeHash* hll_hash = new CompositeHash(tl); CompositeHash* hll_hash = new CompositeHash(tl);
Unref(tl); Unref(tl);
if( hll_counters.count(*s) > 0 ) CardinalityCounter* h = ((CardinalityVal*) handle)->Get();
{ HashKey* key = hll_hash->ComputeHash(elem, 1);
CardinalityCounter* h = hll_counters[*s]; a = key->Hash();
HashKey* key = hll_hash->ComputeHash(elem, 1); h->addElement(a);
a = key->Hash();
h->addElement(a);
status = 1;
delete key;
}
delete hll_hash; delete hll_hash;
delete s; return new Val(1, TYPE_BOOL);
return new Val(status, TYPE_BOOL);
%} %}
## The data structure at index1 will contain the combined count for the ## The data structure at index1 will contain the combined count for the
@ -5716,135 +5702,68 @@ function hll_cardinality_add%(elem: any, index: any%): bool
## It returns true if it either cloned the value at index2 into index1 ## It returns true if it either cloned the value at index2 into index1
## or if it merged the two data structures together. ## or if it merged the two data structures together.
function hll_cardinality_merge_into%(index1: any, index2: any%): bool function hll_cardinality_merge_into%(handle1: opaque of cardinality, handle2: opaque of cardinality%): bool
%{ %{
BroString* s1 = convert_index_to_string(index1); CardinalityVal* v1 = (CardinalityVal*) handle1;
BroString* s2 = convert_index_to_string(index2); CardinalityVal* v2 = (CardinalityVal*) handle2;
int status = 0;
if(hll_counters.count(*s1) < 1) if ( !v1->IsValid() || !v2->IsValid() ) {
{ reporter->Error("need valid handles");
if(hll_counters.count(*s2) < 1) return new Val(0, TYPE_BOOL);
{ }
status = 0;
}
else
{
uint64_t m = (*hll_counters[*s2]).getM();
CardinalityCounter* newInst = new CardinalityCounter(m);
hll_counters[*s1] = newInst;
(*newInst).merge(hll_counters[*s2]);
status = 1;
}
}
else
{
if(hll_counters.count(*s2) < 1)
{
status = 0;
}
else
{
if((*hll_counters[*s2]).getM() == (*hll_counters[*s1]).getM())
{
status = 1;
(*hll_counters[*s1]).merge(hll_counters[*s2]);
}
}
}
delete s1; CardinalityCounter* h1 = v1->Get();
delete s2; CardinalityCounter* h2 = v2->Get();
return new Val(status, TYPE_BOOL);
h1->merge(h2);
return new Val(1, TYPE_BOOL);
%} %}
## Returns true if it destroyed something. False if it didn't. ## Returns true if it destroyed something. False if it didn't.
function hll_cardinality_destroy%(index: any%): bool #function hll_cardinality_destroy%(handle: opaque of cardinality%): bool
%{ # %{
BroString* s = convert_index_to_string(index); # if ( !((CardinalityVal*) handle)->IsValid() ) {
int status = 0; # reporter->Error("Need valid handle");
# return new Val(0, TYPE_BOOL);
if(hll_counters.count(*s) > 0) # }
{ # CardinalityCounter* h = ((CardinalityVal*) handle)->Get();
hll_counters.erase(*s); # delete h;
status = 1; # h = 0;
} # return new Val(1, TYPE_BOOL);
# %}
delete s;
return new Val(status, TYPE_BOOL);
%}
## Returns the cardinality estimate. Returns -1.0 if there is nothing in that index. ## Returns the cardinality estimate. Returns -1.0 if there is nothing in that index.
function hll_cardinality_estimate%(index: any%): double function hll_cardinality_estimate%(handle: opaque of cardinality%): double
%{ %{
BroString* s = convert_index_to_string(index); if ( !((CardinalityVal*) handle)->IsValid() ) {
double estimate = -1.0; reporter->Error("Need valid handle");
return new Val(0, TYPE_BOOL);
}
CardinalityCounter* h = ((CardinalityVal*) handle)->Get();
if(hll_counters.count(*s) > 0) double estimate = h->size();
{
estimate = (*hll_counters[*s]).size();
}
delete s;
return new Val(estimate, TYPE_DOUBLE); return new Val(estimate, TYPE_DOUBLE);
%} %}
##I'm really not sure about the notation of this function...
function hll_cardinality_keys%(%): string_set
%{
TableVal* a = new TableVal(string_set);
map<BroString, CardinalityCounter*>::iterator it;
int i = 0;
for(it = hll_counters.begin() ; it != hll_counters.end(); it++)
{
BroString* s = (BroString*) &(it->first);
a->Assign(new StringVal(s), 0);
}
return a;
%}
## Stores the data structure at index2 into index1. Deletes the data structure at index1 ## Stores the data structure at index2 into index1. Deletes the data structure at index1
## if there was any. Returns True if the data structure at index1 was changed in any way. ## if there was any. Returns True if the data structure at index1 was changed in any way.
function hll_cardinality_clone%(index1: any, index2: any%): bool function hll_cardinality_clone%(handle: opaque of cardinality%): opaque of cardinality
%{ %{
BroString* s1 = convert_index_to_string(index1); if ( !((CardinalityVal*) handle)->IsValid() ) {
BroString* s2 = convert_index_to_string(index2); reporter->Error("Need valid handle");
int status = 0; return new Val(0, TYPE_BOOL);
}
CardinalityCounter* h = ((CardinalityVal*) handle)->Get();
if(hll_counters.count(*s2) < 1)
{ uint64_t m = h->getM();
if(hll_counters.count(*s1) < 1) CardinalityCounter* h2 = new CardinalityCounter(m);
{ int i = 0;
status = 0; h2->merge(h);
} CardinalityVal* cv = new CardinalityVal();
else cv->Init(h2);
{ return cv;
delete hll_counters[*s1];
status = 1;
}
}
else
{
uint64_t m = (*hll_counters[*s2]).getM();
CardinalityCounter* newInst = new CardinalityCounter(m);
int i = 0;
(*newInst).merge(hll_counters[*s2]);
if(hll_counters.count(*s1) < 1)
{
hll_counters[*s1] = newInst;
}
else
{
delete hll_counters[*s1];
hll_counters[*s1] = newInst;
}
status = 1;
}
delete s1;
delete s2;
return new Val(status, TYPE_BOOL);
%} %}

145
testing/btest/bifs/hll_cardinality.bro
View file

@ -4,135 +4,78 @@
event bro_init() event bro_init()
{ {
local m1 = "measurement1"; local c1 = hll_cardinality_init(0.01);
local m2 = "measurement2"; local c2 = hll_cardinality_init(0.01);
print "This value should be true:";
print hll_cardinality_init(0.01, m1);
hll_cardinality_init(0.01, m2);
print "This value should be false:";
print hll_cardinality_init(0.02, "measurement1");
local add1 = "hey"; local add1 = "hey";
local add2 = "hi"; local add2 = "hi";
local add3 = 123; local add3 = 123;
hll_cardinality_add(add1, m1); hll_cardinality_add(c1, add1);
hll_cardinality_add(add2, m1); hll_cardinality_add(c1, add2);
hll_cardinality_add(add3, m1); hll_cardinality_add(c1, add3);
hll_cardinality_add("a", m1); hll_cardinality_add(c1, "a");
hll_cardinality_add("b", m1); hll_cardinality_add(c1, "b");
hll_cardinality_add("c", m1); hll_cardinality_add(c1, "c");
hll_cardinality_add("d", m1); hll_cardinality_add(c1, "d");
hll_cardinality_add("e", m1); hll_cardinality_add(c1, "e");
hll_cardinality_add("f", m1); hll_cardinality_add(c1, "f");
hll_cardinality_add("g", m1); hll_cardinality_add(c1, "g");
hll_cardinality_add("h", m1); hll_cardinality_add(c1, "h");
hll_cardinality_add("i", m1); hll_cardinality_add(c1, "i");
hll_cardinality_add(c1, "j");
print "This value should be true:"; hll_cardinality_add(c2, add1);
print hll_cardinality_add("j", m1); hll_cardinality_add(c2, add2);
hll_cardinality_add(c2, add3);
print "This value should be false:"; hll_cardinality_add(c2, 1);
print hll_cardinality_add("asdf", "something"); hll_cardinality_add(c2, "b");
hll_cardinality_add(c2, 2);
hll_cardinality_add(c2, 3);
hll_cardinality_add(add1, m2); hll_cardinality_add(c2, 4);
hll_cardinality_add(add2, m2); hll_cardinality_add(c2, 5);
hll_cardinality_add(add3, m2); hll_cardinality_add(c2, 6);
hll_cardinality_add(1, m2); hll_cardinality_add(c2, 7);
hll_cardinality_add("b", m2); hll_cardinality_add(c2, 8);
hll_cardinality_add(2, m2);
hll_cardinality_add(3, m2);
hll_cardinality_add(4, m2);
hll_cardinality_add(5, m2);
hll_cardinality_add(6, m2);
hll_cardinality_add(7, m2);
hll_cardinality_add(8, m2);
print "This value should be around 13:"; print "This value should be around 13:";
print hll_cardinality_estimate("measurement1"); print hll_cardinality_estimate(c1);
print "This value should be -1.0:"; print "This value should be about 12:";
print hll_cardinality_estimate("m2"); print hll_cardinality_estimate(c2);
hll_cardinality_init(0.02, "m2"); local m2 = hll_cardinality_init(0.02);
print "This value should be around 0:"; print "This value should be around 0:";
print hll_cardinality_estimate("m2"); print hll_cardinality_estimate(m2);
print "This value should be true:"; local c3 = hll_cardinality_clone(c1);
print hll_cardinality_destroy("m2");
print "This value should be false:";
print hll_cardinality_destroy("m2");
print "This value should be -1.0:";
print hll_cardinality_estimate("m2");
print "This next thing should be false:";
print hll_cardinality_clone("m3", "m2");
print "This next thing should be true:";
print hll_cardinality_clone("measurement3", "measurement1");
print "This value should be around 13:"; print "This value should be around 13:";
print hll_cardinality_estimate("measurement3"); print hll_cardinality_estimate(c3);
hll_cardinality_destroy("measurement3"); c3 = hll_cardinality_init(0.01);
print "This value should be 0:";
print "This next thing should be equal to -1.0:"; print hll_cardinality_estimate(c3);
print hll_cardinality_estimate("measurement3");
print "This value should be around 13:";
print hll_cardinality_estimate("measurement1");
print "This value should be true:"; print "This value should be true:";
print hll_cardinality_merge_into("measurement3", "measurement2"); print hll_cardinality_merge_into(c3, c2);
print "This value should be false:";
print hll_cardinality_merge_into("measurement4", "measurement6");
print "This value should be about 12:"; print "This value should be about 12:";
print hll_cardinality_estimate("measurement3"); print hll_cardinality_estimate(c2);
print hll_cardinality_estimate(c3);
print "This value should be false:";
print hll_cardinality_merge_into("measurement3", "measurement15");
print "This value should be about 12:";
print hll_cardinality_estimate("measurement3");
print "This value should be true:"; print "This value should be true:";
print hll_cardinality_merge_into("measurement2", "measurement1"); print hll_cardinality_merge_into(c2, c1);
print "This value should be about 21:"; print "This value should be about 21:";
print hll_cardinality_estimate("measurement2"); print hll_cardinality_estimate(c2);
print "This value should be about 13:"; print "This value should be about 13:";
print hll_cardinality_estimate("measurement1"); print hll_cardinality_estimate(c1);
print "This value should be about 12:"; print "This value should be about 12:";
print hll_cardinality_estimate("measurement3"); print hll_cardinality_estimate(c3);
local keys = hll_cardinality_keys();
for(key in keys)
{
print "The key is:";
print key;
print "The value is:";
print hll_cardinality_estimate(key);
}
} }
#function hll_cardinality_keys%(%): bool
# %{
#// TableVal* a = new TableVal(string_set);
#// map<BroString, CardinalityCounter*>::iterator it;
#
#// for(it = hll_counters.begin() ; it != hll_counters.end(); it++)
#// {
#// a->Assign((*it).first);
#// }
#// return a;
# return new Val(1, TYPE_BOOL);