diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index efdd890f70..b70cfee086 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -578,22 +578,48 @@ size_t BloomFilterVal::Count(const Val* val) const return cnt; } +void BloomFilterVal::Clear() + { + bloom_filter->Clear(); + } + +bool BloomFilterVal::Empty() const + { + return bloom_filter->Empty(); + } + BloomFilterVal* BloomFilterVal::Merge(const BloomFilterVal* x, - const BloomFilterVal* y) + const BloomFilterVal* y) { if ( ! same_type(x->Type(), y->Type()) ) - reporter->InternalError("cannot merge Bloom filters with different types"); + { + reporter->Error("cannot merge Bloom filters with different types"); + return 0; + } - BloomFilterVal* result; + if ( typeid(*x->bloom_filter) != typeid(*y->bloom_filter) ) + { + reporter->Error("cannot merge different Bloom filter types"); + return 0; + } - if ( (result = DoMerge(x, y)) ) - return result; + probabilistic::BloomFilter* copy = x->bloom_filter->Clone(); - else if ( (result = DoMerge(x, y)) ) - return result; + if ( ! copy->Merge(y->bloom_filter) ) + { + reporter->Error("failed to merge Bloom filter"); + return 0; + } - reporter->InternalError("failed to merge Bloom filters"); - return 0; + BloomFilterVal* merged = new BloomFilterVal(copy); + + if ( ! merged->Typify(x->Type()) ) + { + reporter->Error("failed to set type on merged Bloom filter"); + return 0; + } + + return merged; } BloomFilterVal::~BloomFilterVal() diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index ea704cb70a..52c9583fc7 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -125,6 +125,8 @@ public: void Add(const Val* val); size_t Count(const Val* val) const; + void Clear(); + bool Empty() const; static BloomFilterVal* Merge(const BloomFilterVal* x, const BloomFilterVal* y); @@ -141,28 +143,6 @@ private: BloomFilterVal(const BloomFilterVal&); BloomFilterVal& operator=(const BloomFilterVal&); - template - static BloomFilterVal* DoMerge(const BloomFilterVal* x, - const BloomFilterVal* y) - { - if ( typeid(*x->bloom_filter) != typeid(*y->bloom_filter) ) - reporter->InternalError("cannot merge different Bloom filter types"); - - if ( typeid(T) != typeid(*x->bloom_filter) ) - return 0; - - const T* a = static_cast(x->bloom_filter); - const T* b = static_cast(y->bloom_filter); - - BloomFilterVal* merged = new BloomFilterVal(T::Merge(a, b)); - assert(merged); - - if ( ! merged->Typify(x->Type()) ) - reporter->InternalError("failed to set type on merged Bloom filter"); - - return merged; - } - BroType* type; CompositeHash* hash; probabilistic::BloomFilter* bloom_filter; diff --git a/src/probabilistic/BitVector.cc b/src/probabilistic/BitVector.cc index 98f008b24b..c0285eced3 100644 --- a/src/probabilistic/BitVector.cc +++ b/src/probabilistic/BitVector.cc @@ -463,6 +463,17 @@ bool BitVector::Empty() const return bits.empty(); } +bool BitVector::AllZero() const + { + for ( size_t i = 0; i < bits.size(); ++i ) + { + if ( bits[i] ) + return false; + } + + return true; + } + BitVector::size_type BitVector::FindFirst() const { return find_from(0); diff --git a/src/probabilistic/BitVector.h b/src/probabilistic/BitVector.h index 9eefe1b633..d9c55d53c6 100644 --- a/src/probabilistic/BitVector.h +++ b/src/probabilistic/BitVector.h @@ -253,6 +253,12 @@ public: */ bool Empty() const; + /** + * Checks whether all bits are 0. + * @return `true` iff all bits in all blocks are 0. + */ + bool AllZero() const; + /** * Finds the bit position of of the first 1-bit. * @return The position of the first bit that equals to one or `npos` if no diff --git a/src/probabilistic/BloomFilter.cc b/src/probabilistic/BloomFilter.cc index 5613dcce05..db768ed934 100644 --- a/src/probabilistic/BloomFilter.cc +++ b/src/probabilistic/BloomFilter.cc @@ -1,9 +1,11 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include +#include +#include + #include "BloomFilter.h" -#include -#include #include "CounterVector.h" #include "Serializer.h" @@ -74,17 +76,48 @@ size_t BasicBloomFilter::K(size_t cells, size_t capacity) return std::ceil(frac * std::log(2)); } -BasicBloomFilter* BasicBloomFilter::Merge(const BasicBloomFilter* x, - const BasicBloomFilter* y) +bool BasicBloomFilter::Empty() const { - if ( ! x->hasher->Equals(y->hasher) ) - reporter->InternalError("incompatible hashers during BasicBloomFilter merge"); + return bits->AllZero(); + } - BasicBloomFilter* result = new BasicBloomFilter(); - result->hasher = x->hasher->Clone(); - result->bits = new BitVector(*x->bits | *y->bits); +void BasicBloomFilter::Clear() + { + bits->Clear(); + } - return result; +bool BasicBloomFilter::Merge(const BloomFilter* other) + { + if ( typeid(*this) != typeid(*other) ) + return false; + + const BasicBloomFilter* o = static_cast(other); + + if ( ! hasher->Equals(o->hasher) ) + { + reporter->Error("incompatible hashers in BasicBloomFilter merge"); + return false; + } + + else if ( bits->Size() != o->bits->Size() ) + { + reporter->Error("different bitvector size in BasicBloomFilter merge"); + return false; + } + + (*bits) |= *o->bits; + + return true; + } + +BasicBloomFilter* BasicBloomFilter::Clone() const + { + BasicBloomFilter* copy = new BasicBloomFilter(); + + copy->hasher = hasher->Clone(); + copy->bits = new BitVector(*bits); + + return copy; } BasicBloomFilter::BasicBloomFilter() @@ -130,19 +163,6 @@ size_t BasicBloomFilter::CountImpl(const Hasher::digest_vector& h) const return 1; } -CountingBloomFilter* CountingBloomFilter::Merge(const CountingBloomFilter* x, - const CountingBloomFilter* y) - { - if ( ! x->hasher->Equals(y->hasher) ) - reporter->InternalError("incompatible hashers during CountingBloomFilter merge"); - - CountingBloomFilter* result = new CountingBloomFilter(); - result->hasher = x->hasher->Clone(); - result->cells = new CounterVector(*x->cells | *y->cells); - - return result; - } - CountingBloomFilter::CountingBloomFilter() { cells = 0; @@ -155,6 +175,50 @@ CountingBloomFilter::CountingBloomFilter(const Hasher* hasher, cells = new CounterVector(width, arg_cells); } +bool CountingBloomFilter::Empty() const + { + return cells->AllZero(); + } + +void CountingBloomFilter::Clear() + { + cells->Clear(); + } + +bool CountingBloomFilter::Merge(const BloomFilter* other) + { + if ( typeid(*this) != typeid(*other) ) + return false; + + const CountingBloomFilter* o = static_cast(other); + + if ( ! hasher->Equals(o->hasher) ) + { + reporter->Error("incompatible hashers in CountingBloomFilter merge"); + return false; + } + + else if ( cells->Size() != o->cells->Size() ) + { + reporter->Error("different bitvector size in CountingBloomFilter merge"); + return false; + } + + (*cells) |= *o->cells; + + return true; + } + +CountingBloomFilter* CountingBloomFilter::Clone() const + { + CountingBloomFilter* copy = new CountingBloomFilter(); + + copy->hasher = hasher->Clone(); + copy->cells = new CounterVector(*cells); + + return copy; + } + IMPLEMENT_SERIAL(CountingBloomFilter, SER_COUNTINGBLOOMFILTER) bool CountingBloomFilter::DoSerialize(SerialInfo* info) const diff --git a/src/probabilistic/BloomFilter.h b/src/probabilistic/BloomFilter.h index 4a6b01c484..b6cf18672f 100644 --- a/src/probabilistic/BloomFilter.h +++ b/src/probabilistic/BloomFilter.h @@ -47,6 +47,34 @@ public: return CountImpl((*hasher)(x)); } + /** + * Checks whether the Bloom filter is empty. + * + * @return `true` if the Bloom filter contains no elements. + */ + virtual bool Empty() const = 0; + + /** + * Removes all elements, i.e., resets all bits in the underlying bit vector. + */ + virtual void Clear() = 0; + + /** + * Merges another Bloom filter into a copy of this one. + * + * @param other The other Bloom filter. + * + * @return `true` on success. + */ + virtual bool Merge(const BloomFilter* other) = 0; + + /** + * Constructs a copy of this Bloom filter. + * + * @return A copy of `*this`. + */ + virtual BloomFilter* Clone() const = 0; + /** * Serializes the Bloom filter. * @@ -147,13 +175,11 @@ public: */ static size_t K(size_t cells, size_t capacity); - /** - * Merges two basic Bloom filters. - * - * @return The merged Bloom filter. - */ - static BasicBloomFilter* Merge(const BasicBloomFilter* x, - const BasicBloomFilter* y); + // Overridden from BloomFilter. + virtual bool Empty() const; + virtual void Clear(); + virtual bool Merge(const BloomFilter* other); + virtual BasicBloomFilter* Clone() const; protected: DECLARE_SERIAL(BasicBloomFilter); @@ -188,13 +214,11 @@ public: */ CountingBloomFilter(const Hasher* hasher, size_t cells, size_t width); - /** - * Merges two counting Bloom filters. - * - * @return The merged Bloom filter. - */ - static CountingBloomFilter* Merge(const CountingBloomFilter* x, - const CountingBloomFilter* y); + // Overridden from BloomFilter. + virtual bool Empty() const; + virtual void Clear(); + virtual bool Merge(const BloomFilter* other); + virtual CountingBloomFilter* Clone() const; protected: DECLARE_SERIAL(CountingBloomFilter); diff --git a/src/probabilistic/CMakeLists.txt b/src/probabilistic/CMakeLists.txt index 961c07fb33..f82cdfaf8e 100644 --- a/src/probabilistic/CMakeLists.txt +++ b/src/probabilistic/CMakeLists.txt @@ -15,4 +15,5 @@ set(probabilistic_SRCS bif_target(bloom-filter.bif) bro_add_subdir_library(probabilistic ${probabilistic_SRCS} ${BIF_OUTPUT_CC}) + add_dependencies(bro_probabilistic generate_outputs) diff --git a/src/probabilistic/CounterVector.cc b/src/probabilistic/CounterVector.cc index 570ed1f8ea..24c9ff3638 100644 --- a/src/probabilistic/CounterVector.cc +++ b/src/probabilistic/CounterVector.cc @@ -70,6 +70,16 @@ bool CounterVector::Decrement(size_type cell, count_type value) return carry; } +bool CounterVector::AllZero() const + { + return bits->AllZero(); + } + +void CounterVector::Clear() + { + bits->Clear(); + } + CounterVector::count_type CounterVector::Count(size_type cell) const { assert(cell < Size()); diff --git a/src/probabilistic/CounterVector.h b/src/probabilistic/CounterVector.h index 178a68e8f2..df6fc57ac2 100644 --- a/src/probabilistic/CounterVector.h +++ b/src/probabilistic/CounterVector.h @@ -77,6 +77,17 @@ public: */ count_type Count(size_type cell) const; + /** + * Checks whether all counters are 0. + * @return `true` iff all counters have the value 0. + */ + bool AllZero() const; + + /** + * Sets all counters to 0. + */ + void Clear(); + /** * Retrieves the number of cells in the storage. * diff --git a/src/probabilistic/Hasher.h b/src/probabilistic/Hasher.h index 62c5d58d1f..9f7d4ae32d 100644 --- a/src/probabilistic/Hasher.h +++ b/src/probabilistic/Hasher.h @@ -74,16 +74,25 @@ public: * * @param k The number of hash functions to apply. * - * @param name The hasher's name. + * @param name The hasher's name. Hashers with the same name should + * provide consistent results. * * @return Returns a new hasher instance. */ static Hasher* Create(size_t k, const std::string& name); protected: + /** + * Constructor. + * + * @param k the number of hash functions. + * + * @param name A name for the hasher. Hashers with the same name + * should provide consistent results. + */ Hasher(size_t k, const std::string& name); - private: +private: const size_t k; std::string name; }; diff --git a/src/probabilistic/bloom-filter.bif b/src/probabilistic/bloom-filter.bif index cbbff85d7d..52f8a27f4f 100644 --- a/src/probabilistic/bloom-filter.bif +++ b/src/probabilistic/bloom-filter.bif @@ -18,17 +18,25 @@ using namespace probabilistic; module GLOBAL; -## Creates a basic Bloom filter. -## +## Creates a basic Bloom filter. +## +## .. note:: A Bloom filter can have a name associated with it. In the future, +## Bloom filters with the same name will be compatible across indepedent Bro +## instances, i.e., it will be possible to merge them. Currently, however, that is +## not yet supported. +## ## fp: The desired false-positive rate. ## ## capacity: the maximum number of elements that guarantees a false-positive ## rate of *fp*. ## ## name: A name that uniquely identifies and seeds the Bloom filter. If empty, -## the initialization will become dependent on the initial seed. +## the filter will remain tied to the current Bro process. ## ## Returns: A Bloom filter handle. +## +## .. bro:see:: bloomfilter_counting_init bloomfilter_add bloomfilter_lookup +## bloomfilter_clear bloomfilter_merge function bloomfilter_basic_init%(fp: double, capacity: count, name: string &default=""%): opaque of bloomfilter %{ @@ -46,19 +54,29 @@ function bloomfilter_basic_init%(fp: double, capacity: count, %} ## Creates a counting Bloom filter. +## +## .. note:: A Bloom filter can have a name associated with it. In the future, +## Bloom filters with the same name will be compatible across indepedent Bro +## instances, i.e., it will be possible to merge them. Currently, however, that is +## not yet supported. ## ## k: The number of hash functions to use. ## -## cells: The number of cells of the underlying counter vector. +## cells: The number of cells of the underlying counter vector. As there's no +## single answer to what's the best parameterization for a counting Bloom filter, +## we refer to the Bloom filter literature here for choosing an appropiate value. ## ## max: The maximum counter value associated with each each element described ## by *w = ceil(log_2(max))* bits. Each bit in the underlying counter vector ## becomes a cell of size *w* bits. ## ## name: A name that uniquely identifies and seeds the Bloom filter. If empty, -## the initialization will become dependent on the initial seed. +## the filter will remain tied to the current Bro process. ## ## Returns: A Bloom filter handle. +## +## .. bro:see:: bloomfilter_basic_init bloomfilter_add bloomfilter_lookup +## bloomfilter_clear bloomfilter_merge function bloomfilter_counting_init%(k: count, cells: count, max: count, name: string &default=""%): opaque of bloomfilter %{ @@ -82,6 +100,9 @@ function bloomfilter_counting_init%(k: count, cells: count, max: count, ## bf: The Bloom filter handle. ## ## x: The element to add. +## +## .. bro:see:: bloomfilter_counting_init bloomfilter_basic_init loomfilter_lookup +## bloomfilter_clear bloomfilter_merge function bloomfilter_add%(bf: opaque of bloomfilter, x: any%): any %{ BloomFilterVal* bfv = static_cast(bf); @@ -105,10 +126,16 @@ function bloomfilter_add%(bf: opaque of bloomfilter, x: any%): any ## x: The element to count. ## ## Returns: the counter associated with *x* in *bf*. +## +## .. bro:see:: bloomfilter_counting_init bloomfilter_basic_init +## bloomfilter_add bloomfilter_clear bloomfilter_merge function bloomfilter_lookup%(bf: opaque of bloomfilter, x: any%): count %{ const BloomFilterVal* bfv = static_cast(bf); + if ( bfv->Empty() ) + return new Val(0, TYPE_COUNT); + if ( ! bfv->Type() ) reporter->Error("cannot perform lookup on untyped Bloom filter"); @@ -121,13 +148,38 @@ function bloomfilter_lookup%(bf: opaque of bloomfilter, x: any%): count return new Val(0, TYPE_COUNT); %} -## Merges two Bloom filters. +## Removes all elements from a Bloom filter. This function resets all bits in the +## underlying bitvector back to 0 but does not change the parameterization of the +## Bloom filter, such as the element type and the hasher seed. +## +## bf: The Bloom filter handle. +## +## .. bro:see:: bloomfilter_counting_init bloomfilter_basic_init +## bloomfilter_add bloomfilter_lookup bloomfilter_merge +function bloomfilter_clear%(bf: opaque of bloomfilter%): any + %{ + BloomFilterVal* bfv = static_cast(bf); + + if ( bfv->Type() ) // Untyped Bloom filters are already empty. + bfv->Clear(); + + return 0; + %} + +## Merges two Bloom filters. +## +## .. note:: Currently Bloom filters created by different Bro instances cannot +## be merged. In the future, this will be supported as long as both filters +## are created with the same name. ## ## bf1: The first Bloom filter handle. ## ## bf2: The second Bloom filter handle. ## ## Returns: The union of *bf1* and *bf2*. +## +## .. bro:see:: bloomfilter_counting_init bloomfilter_basic_init +## bloomfilter_add bloomfilter_lookup bloomfilter_clear function bloomfilter_merge%(bf1: opaque of bloomfilter, bf2: opaque of bloomfilter%): opaque of bloomfilter %{