Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-08 01:28:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

rework cardinality interface to use opaque.

Browse source 
I like it better...
This commit is contained in:
Bernhard Amann 2013-04-07 23:05:14 +02:00
parent fd51db1c89
commit 53d6f3aae7
4 changed files with 147 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/OpaqueVal.cc
View file

@ -2,6 +2,28 @@
#include "Reporter.h"
#include "Serializer.h"
CardinalityVal::CardinalityVal()
{
valid = false;
}
CardinalityVal::~CardinalityVal()
{
if ( valid && c )
delete c;
}
bool CardinalityVal::Init(CardinalityCounter* arg_c)
{
if ( valid )
return false;
valid = true;
c = arg_c;
return valid;
}
bool HashVal::IsValid() const
{
return valid;

18
src/OpaqueVal.h
View file

@ -7,6 +7,24 @@
#include "Val.h"
#include "digest.h"
class CardinalityCounter;
class CardinalityVal: public OpaqueVal {
public:
CardinalityVal();
~CardinalityVal();
bool Init(CardinalityCounter*);
bool IsValid() const { return valid; };
CardinalityCounter* Get() { return c; };
private:
bool valid;
CardinalityCounter* c;
// DECLARE_SERIAL(CardinalityVal); Fixme?
};
class HashVal : public OpaqueVal {
public:
virtual bool IsValid() const;

197
src/bro.bif
View file

@ -5650,44 +5650,36 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr
%%{
#include "HyperLogLog.h"
static map<BroString, CardinalityCounter*> hll_counters;
BroString* convert_index_to_string(Val* index)
{
ODesc d;
index->Describe(&d);
BroString* s = new BroString(1, d.TakeBytes(), d.Len());
s->SetUseFreeToDelete(1);
return s;
}
%%}
## Initializes the hash for the HyperLogLog cardinality counting algorithm.
## It returns true if it was successful in creating a structure and false
## if it wasn't.
function hll_cardinality_init%(err: double,index: any%): bool
function hll_cardinality_init%(err: double%): opaque of cardinality
%{
BroString* s = convert_index_to_string(index);
int status = 0;
CardinalityCounter* c = new CardinalityCounter(err);
CardinalityVal* cv = new CardinalityVal();
if ( hll_counters.count(*s) < 1 )
{
hll_counters[*s] = new CardinalityCounter(err);
status = 1;
}
if ( !c )
reporter->Error("Failed initialize Cardinality counter");
else
cv->Init(c);
delete s;
return new Val(status, TYPE_BOOL);
return cv;
%}
## Adds an element to the HyperLogLog data structure located at index.
##elem->Type() to get the type of elem.
function hll_cardinality_add%(elem: any, index: any%): bool
function hll_cardinality_add%(handle: opaque of cardinality, elem: any%): bool
%{
BroString* s = convert_index_to_string(index);
if ( !((CardinalityVal*) handle)->IsValid() ) {
reporter->Error("Need valid handle");
return new Val(0, TYPE_BOOL);
}
int status = 0;
uint64_t a = 123456;
@ -5696,19 +5688,13 @@ function hll_cardinality_add%(elem: any, index: any%): bool
CompositeHash* hll_hash = new CompositeHash(tl);
Unref(tl);
if( hll_counters.count(*s) > 0 )
{
CardinalityCounter* h = hll_counters[*s];
CardinalityCounter* h = ((CardinalityVal*) handle)->Get();
HashKey* key = hll_hash->ComputeHash(elem, 1);
a = key->Hash();
h->addElement(a);
status = 1;
delete key;
}
delete hll_hash;
delete s;
return new Val(status, TYPE_BOOL);
return new Val(1, TYPE_BOOL);
%}
## The data structure at index1 will contain the combined count for the
@ -5716,135 +5702,68 @@ function hll_cardinality_add%(elem: any, index: any%): bool
## It returns true if it either cloned the value at index2 into index1
## or if it merged the two data structures together.
function hll_cardinality_merge_into%(index1: any, index2: any%): bool
function hll_cardinality_merge_into%(handle1: opaque of cardinality, handle2: opaque of cardinality%): bool
%{
BroString* s1 = convert_index_to_string(index1);
BroString* s2 = convert_index_to_string(index2);
int status = 0;
CardinalityVal* v1 = (CardinalityVal*) handle1;
CardinalityVal* v2 = (CardinalityVal*) handle2;
if(hll_counters.count(*s1) < 1)
{
if(hll_counters.count(*s2) < 1)
{
status = 0;
}
else
{
uint64_t m = (*hll_counters[*s2]).getM();
CardinalityCounter* newInst = new CardinalityCounter(m);
hll_counters[*s1] = newInst;
(*newInst).merge(hll_counters[*s2]);
status = 1;
}
}
else
{
if(hll_counters.count(*s2) < 1)
{
status = 0;
}
else
{
if((*hll_counters[*s2]).getM() == (*hll_counters[*s1]).getM())
{
status = 1;
(*hll_counters[*s1]).merge(hll_counters[*s2]);
}
}
if ( !v1->IsValid() || !v2->IsValid() ) {
reporter->Error("need valid handles");
return new Val(0, TYPE_BOOL);
}
delete s1;
delete s2;
return new Val(status, TYPE_BOOL);
CardinalityCounter* h1 = v1->Get();
CardinalityCounter* h2 = v2->Get();
h1->merge(h2);
return new Val(1, TYPE_BOOL);
%}
## Returns true if it destroyed something. False if it didn't.
function hll_cardinality_destroy%(index: any%): bool
%{
BroString* s = convert_index_to_string(index);
int status = 0;
if(hll_counters.count(*s) > 0)
{
hll_counters.erase(*s);
status = 1;
}
delete s;
return new Val(status, TYPE_BOOL);
%}
#function hll_cardinality_destroy%(handle: opaque of cardinality%): bool
# %{
# if ( !((CardinalityVal*) handle)->IsValid() ) {
# reporter->Error("Need valid handle");
# return new Val(0, TYPE_BOOL);
# }
# CardinalityCounter* h = ((CardinalityVal*) handle)->Get();
# delete h;
# h = 0;
# return new Val(1, TYPE_BOOL);
# %}
## Returns the cardinality estimate. Returns -1.0 if there is nothing in that index.
function hll_cardinality_estimate%(index: any%): double
function hll_cardinality_estimate%(handle: opaque of cardinality%): double
%{
BroString* s = convert_index_to_string(index);
double estimate = -1.0;
if(hll_counters.count(*s) > 0)
{
estimate = (*hll_counters[*s]).size();
if ( !((CardinalityVal*) handle)->IsValid() ) {
reporter->Error("Need valid handle");
return new Val(0, TYPE_BOOL);
}
CardinalityCounter* h = ((CardinalityVal*) handle)->Get();
double estimate = h->size();
delete s;
return new Val(estimate, TYPE_DOUBLE);
%}
##I'm really not sure about the notation of this function...
function hll_cardinality_keys%(%): string_set
%{
TableVal* a = new TableVal(string_set);
map<BroString, CardinalityCounter*>::iterator it;
int i = 0;
for(it = hll_counters.begin() ; it != hll_counters.end(); it++)
{
BroString* s = (BroString*) &(it->first);
a->Assign(new StringVal(s), 0);
}
return a;
%}
## Stores the data structure at index2 into index1. Deletes the data structure at index1
## if there was any. Returns True if the data structure at index1 was changed in any way.
function hll_cardinality_clone%(index1: any, index2: any%): bool
function hll_cardinality_clone%(handle: opaque of cardinality%): opaque of cardinality
%{
BroString* s1 = convert_index_to_string(index1);
BroString* s2 = convert_index_to_string(index2);
int status = 0;
if ( !((CardinalityVal*) handle)->IsValid() ) {
reporter->Error("Need valid handle");
return new Val(0, TYPE_BOOL);
}
CardinalityCounter* h = ((CardinalityVal*) handle)->Get();
if(hll_counters.count(*s2) < 1)
{
if(hll_counters.count(*s1) < 1)
{
status = 0;
}
else
{
delete hll_counters[*s1];
status = 1;
}
}
else
{
uint64_t m = (*hll_counters[*s2]).getM();
CardinalityCounter* newInst = new CardinalityCounter(m);
uint64_t m = h->getM();
CardinalityCounter* h2 = new CardinalityCounter(m);
int i = 0;
(*newInst).merge(hll_counters[*s2]);
if(hll_counters.count(*s1) < 1)
{
hll_counters[*s1] = newInst;
}
else
{
delete hll_counters[*s1];
hll_counters[*s1] = newInst;
}
status = 1;
}
delete s1;
delete s2;
return new Val(status, TYPE_BOOL);
h2->merge(h);
CardinalityVal* cv = new CardinalityVal();
cv->Init(h2);
return cv;
%}

145
testing/btest/bifs/hll_cardinality.bro
View file

@ -4,135 +4,78 @@
event bro_init()
{
local m1 = "measurement1";
local m2 = "measurement2";
print "This value should be true:";
print hll_cardinality_init(0.01, m1);
hll_cardinality_init(0.01, m2);
print "This value should be false:";
print hll_cardinality_init(0.02, "measurement1");
local c1 = hll_cardinality_init(0.01);
local c2 = hll_cardinality_init(0.01);
local add1 = "hey";
local add2 = "hi";
local add3 = 123;
hll_cardinality_add(add1, m1);
hll_cardinality_add(add2, m1);
hll_cardinality_add(add3, m1);
hll_cardinality_add("a", m1);
hll_cardinality_add("b", m1);
hll_cardinality_add("c", m1);
hll_cardinality_add("d", m1);
hll_cardinality_add("e", m1);
hll_cardinality_add("f", m1);
hll_cardinality_add("g", m1);
hll_cardinality_add("h", m1);
hll_cardinality_add("i", m1);
hll_cardinality_add(c1, add1);
hll_cardinality_add(c1, add2);
hll_cardinality_add(c1, add3);
hll_cardinality_add(c1, "a");
hll_cardinality_add(c1, "b");
hll_cardinality_add(c1, "c");
hll_cardinality_add(c1, "d");
hll_cardinality_add(c1, "e");
hll_cardinality_add(c1, "f");
hll_cardinality_add(c1, "g");
hll_cardinality_add(c1, "h");
hll_cardinality_add(c1, "i");
hll_cardinality_add(c1, "j");
print "This value should be true:";
print hll_cardinality_add("j", m1);
print "This value should be false:";
print hll_cardinality_add("asdf", "something");
hll_cardinality_add(add1, m2);
hll_cardinality_add(add2, m2);
hll_cardinality_add(add3, m2);
hll_cardinality_add(1, m2);
hll_cardinality_add("b", m2);
hll_cardinality_add(2, m2);
hll_cardinality_add(3, m2);
hll_cardinality_add(4, m2);
hll_cardinality_add(5, m2);
hll_cardinality_add(6, m2);
hll_cardinality_add(7, m2);
hll_cardinality_add(8, m2);
hll_cardinality_add(c2, add1);
hll_cardinality_add(c2, add2);
hll_cardinality_add(c2, add3);
hll_cardinality_add(c2, 1);
hll_cardinality_add(c2, "b");
hll_cardinality_add(c2, 2);
hll_cardinality_add(c2, 3);
hll_cardinality_add(c2, 4);
hll_cardinality_add(c2, 5);
hll_cardinality_add(c2, 6);
hll_cardinality_add(c2, 7);
hll_cardinality_add(c2, 8);
print "This value should be around 13:";
print hll_cardinality_estimate("measurement1");
print hll_cardinality_estimate(c1);
print "This value should be -1.0:";
print hll_cardinality_estimate("m2");
print "This value should be about 12:";
print hll_cardinality_estimate(c2);
hll_cardinality_init(0.02, "m2");
local m2 = hll_cardinality_init(0.02);
print "This value should be around 0:";
print hll_cardinality_estimate("m2");
print hll_cardinality_estimate(m2);
print "This value should be true:";
print hll_cardinality_destroy("m2");
print "This value should be false:";
print hll_cardinality_destroy("m2");
print "This value should be -1.0:";
print hll_cardinality_estimate("m2");
print "This next thing should be false:";
print hll_cardinality_clone("m3", "m2");
print "This next thing should be true:";
print hll_cardinality_clone("measurement3", "measurement1");
local c3 = hll_cardinality_clone(c1);
print "This value should be around 13:";
print hll_cardinality_estimate("measurement3");
print hll_cardinality_estimate(c3);
hll_cardinality_destroy("measurement3");
print "This next thing should be equal to -1.0:";
print hll_cardinality_estimate("measurement3");
print "This value should be around 13:";
print hll_cardinality_estimate("measurement1");
c3 = hll_cardinality_init(0.01);
print "This value should be 0:";
print hll_cardinality_estimate(c3);
print "This value should be true:";
print hll_cardinality_merge_into("measurement3", "measurement2");
print "This value should be false:";
print hll_cardinality_merge_into("measurement4", "measurement6");
print hll_cardinality_merge_into(c3, c2);
print "This value should be about 12:";
print hll_cardinality_estimate("measurement3");
print "This value should be false:";
print hll_cardinality_merge_into("measurement3", "measurement15");
print "This value should be about 12:";
print hll_cardinality_estimate("measurement3");
print hll_cardinality_estimate(c2);
print hll_cardinality_estimate(c3);
print "This value should be true:";
print hll_cardinality_merge_into("measurement2", "measurement1");
print hll_cardinality_merge_into(c2, c1);
print "This value should be about 21:";
print hll_cardinality_estimate("measurement2");
print hll_cardinality_estimate(c2);
print "This value should be about 13:";
print hll_cardinality_estimate("measurement1");
print hll_cardinality_estimate(c1);
print "This value should be about 12:";
print hll_cardinality_estimate("measurement3");
print hll_cardinality_estimate(c3);
local keys = hll_cardinality_keys();
for(key in keys)
{
print "The key is:";
print key;
print "The value is:";
print hll_cardinality_estimate(key);
}
}
#function hll_cardinality_keys%(%): bool
# %{
#// TableVal* a = new TableVal(string_set);
#// map<BroString, CardinalityCounter*>::iterator it;
#
#// for(it = hll_counters.begin() ; it != hll_counters.end(); it++)
#// {
#// a->Assign((*it).first);
#// }
#// return a;
# return new Val(1, TYPE_BOOL);