Merge remote-tracking branch 'origin/topic/matthias/bloom-filter' into topic/robin/bloom-filter-merge

* origin/topic/matthias/bloom-filter:
  Support emptiness check on Bloom filters.
  Refactor Bloom filter merging.
  Add bloomfilter_clear() BiF.

src/OpaqueVal.cc

@@ -578,22 +578,48 @@ size_t BloomFilterVal::Count(const Val* val) const
 	return cnt;
 	}
 
+void BloomFilterVal::Clear()
+	{
+	bloom_filter->Clear();
+	}
+
+bool BloomFilterVal::Empty() const
+	{
+	return bloom_filter->Empty();
+	}
+
 BloomFilterVal* BloomFilterVal::Merge(const BloomFilterVal* x,
-                                      const BloomFilterVal* y)
+				      const BloomFilterVal* y)
 	{
 	if ( ! same_type(x->Type(), y->Type()) )
-		reporter->InternalError("cannot merge Bloom filters with different types");
+		{
+		reporter->Error("cannot merge Bloom filters with different types");
+		return 0;
+		}
 
-	BloomFilterVal* result;
+	if ( typeid(*x->bloom_filter) != typeid(*y->bloom_filter) )
+		{
+		reporter->Error("cannot merge different Bloom filter types");
+		return 0;
+		}
 
-	if ( (result = DoMerge<probabilistic::BasicBloomFilter>(x, y)) )
+	probabilistic::BloomFilter* copy = x->bloom_filter->Clone();
-		return result;
 
-	else if ( (result = DoMerge<probabilistic::CountingBloomFilter>(x, y)) )
+	if ( ! copy->Merge(y->bloom_filter) )
-		return result;
+		{
+		reporter->Error("failed to merge Bloom filter");
+		return 0;
+		}
 
-	reporter->InternalError("failed to merge Bloom filters");
+	BloomFilterVal* merged = new BloomFilterVal(copy);
-	return 0;
+
+	if ( ! merged->Typify(x->Type()) )
+		{
+		reporter->Error("failed to set type on merged Bloom filter");
+		return 0;
+		}
+
+	return merged;
 	}
 
 BloomFilterVal::~BloomFilterVal()

src/OpaqueVal.h

@@ -125,6 +125,8 @@ public:
 
 	void Add(const Val* val);
 	size_t Count(const Val* val) const;
+	void Clear();
+	bool Empty() const;
 
 	static BloomFilterVal* Merge(const BloomFilterVal* x,
 				     const BloomFilterVal* y);
@@ -141,28 +143,6 @@ private:
 	BloomFilterVal(const BloomFilterVal&);
 	BloomFilterVal& operator=(const BloomFilterVal&);
 
-	template <typename T>
-	static BloomFilterVal* DoMerge(const BloomFilterVal* x,
-				       const BloomFilterVal* y)
-		{
-		if ( typeid(*x->bloom_filter) != typeid(*y->bloom_filter) )
-			reporter->InternalError("cannot merge different Bloom filter types");
-
-		if ( typeid(T) != typeid(*x->bloom_filter) )
-			return 0;
-
-		const T* a = static_cast<const T*>(x->bloom_filter);
-		const T* b = static_cast<const T*>(y->bloom_filter);
-
-		BloomFilterVal* merged = new BloomFilterVal(T::Merge(a, b));
-		assert(merged);
-
-		if ( ! merged->Typify(x->Type()) )
-			reporter->InternalError("failed to set type on merged Bloom filter");
-
-		return merged;
-		}
-
 	BroType* type;
 	CompositeHash* hash;
 	probabilistic::BloomFilter* bloom_filter;

src/probabilistic/BitVector.cc

@@ -463,6 +463,17 @@ bool BitVector::Empty() const
 	return bits.empty();
 	}
 
+bool BitVector::AllZero() const
+	{
+	for ( size_t i = 0; i < bits.size(); ++i )
+		{
+		if ( bits[i] )
+			return false;
+		}
+
+	return true;
+	}
+
 BitVector::size_type BitVector::FindFirst() const
 	{
 	return find_from(0);

src/probabilistic/BitVector.h

@@ -253,6 +253,12 @@ public:
 	 */
 	bool Empty() const;
 
+	/**
+	 * Checks whether all bits are 0.
+	 * @return `true` iff all bits in all blocks are 0.
+	 */
+	bool AllZero() const;
+
 	/**
 	 * Finds the bit position of of the first 1-bit.
 	 * @return The position of the first bit that equals to one or `npos` if no

src/probabilistic/BloomFilter.cc

@@ -1,9 +1,11 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
+#include <typeinfo>
+#include <cmath>
+#include <limits>
+
 #include "BloomFilter.h"
 
-#include <cmath>
-#include <limits>
 #include "CounterVector.h"
 #include "Serializer.h"
 
@@ -74,17 +76,48 @@ size_t BasicBloomFilter::K(size_t cells, size_t capacity)
 	return std::ceil(frac * std::log(2));
 	}
 
-BasicBloomFilter* BasicBloomFilter::Merge(const BasicBloomFilter* x,
+bool BasicBloomFilter::Empty() const
-                                          const BasicBloomFilter* y)
 	{
-	if ( ! x->hasher->Equals(y->hasher) )
+	return bits->AllZero();
-		reporter->InternalError("incompatible hashers during BasicBloomFilter merge");
+	}
 
-	BasicBloomFilter* result = new BasicBloomFilter();
+void BasicBloomFilter::Clear()
-	result->hasher = x->hasher->Clone();
+	{
-	result->bits = new BitVector(*x->bits | *y->bits);
+	bits->Clear();
+	}
 
-	return result;
+bool BasicBloomFilter::Merge(const BloomFilter* other)
+	{
+	if ( typeid(*this) != typeid(*other) )
+		return false;
+
+	const BasicBloomFilter* o = static_cast<const BasicBloomFilter*>(other);
+
+	if ( ! hasher->Equals(o->hasher) )
+		{
+		reporter->Error("incompatible hashers in BasicBloomFilter merge");
+		return false;
+		}
+
+	else if ( bits->Size() != o->bits->Size() )
+		{
+		reporter->Error("different bitvector size in BasicBloomFilter merge");
+		return false;
+		}
+
+	(*bits) |= *o->bits;
+
+	return true;
+	}
+
+BasicBloomFilter* BasicBloomFilter::Clone() const
+	{
+	BasicBloomFilter* copy = new BasicBloomFilter();
+
+	copy->hasher = hasher->Clone();
+	copy->bits = new BitVector(*bits);
+
+	return copy;
 	}
 
 BasicBloomFilter::BasicBloomFilter()
@@ -130,19 +163,6 @@ size_t BasicBloomFilter::CountImpl(const Hasher::digest_vector& h) const
 	return 1;
 	}
 
-CountingBloomFilter* CountingBloomFilter::Merge(const CountingBloomFilter* x,
-						const CountingBloomFilter* y)
-	{
-	if ( ! x->hasher->Equals(y->hasher) )
-		reporter->InternalError("incompatible hashers during CountingBloomFilter merge");
-
-	CountingBloomFilter* result = new CountingBloomFilter();
-	result->hasher = x->hasher->Clone();
-	result->cells = new CounterVector(*x->cells | *y->cells);
-
-	return result;
-	}
-
 CountingBloomFilter::CountingBloomFilter()
 	{
 	cells = 0;
@@ -155,6 +175,50 @@ CountingBloomFilter::CountingBloomFilter(const Hasher* hasher,
 	cells = new CounterVector(width, arg_cells);
 	}
 
+bool CountingBloomFilter::Empty() const
+	{
+	return cells->AllZero();
+	}
+
+void CountingBloomFilter::Clear()
+	{
+	cells->Clear();
+	}
+
+bool CountingBloomFilter::Merge(const BloomFilter* other)
+	{
+	if ( typeid(*this) != typeid(*other) )
+		return false;
+
+	const CountingBloomFilter* o = static_cast<const CountingBloomFilter*>(other);
+
+	if ( ! hasher->Equals(o->hasher) )
+		{
+		reporter->Error("incompatible hashers in CountingBloomFilter merge");
+		return false;
+		}
+
+	else if ( cells->Size() != o->cells->Size() )
+		{
+		reporter->Error("different bitvector size in CountingBloomFilter merge");
+		return false;
+		}
+
+	(*cells) |= *o->cells;
+
+	return true;
+	}
+
+CountingBloomFilter* CountingBloomFilter::Clone() const
+	{
+	CountingBloomFilter* copy = new CountingBloomFilter();
+
+	copy->hasher = hasher->Clone();
+	copy->cells = new CounterVector(*cells);
+
+	return copy;
+	}
+
 IMPLEMENT_SERIAL(CountingBloomFilter, SER_COUNTINGBLOOMFILTER)
 
 bool CountingBloomFilter::DoSerialize(SerialInfo* info) const

src/probabilistic/BloomFilter.h

@@ -47,6 +47,34 @@ public:
 		return CountImpl((*hasher)(x));
 		}
 
+	/**
+	 * Checks whether the Bloom filter is empty.
+	 *
+	 * @return `true` if the Bloom filter contains no elements.
+	 */
+	virtual bool Empty() const = 0;
+
+	/**
+	 * Removes all elements, i.e., resets all bits in the underlying bit vector.
+	 */
+	virtual void Clear() = 0;
+
+	/**
+	 * Merges another Bloom filter into a copy of this one.
+	 *
+	 * @param other The other Bloom filter.
+	 *
+	 * @return `true` on success.
+	 */
+	virtual bool Merge(const BloomFilter* other) = 0;
+
+	/**
+	 * Constructs a copy of this Bloom filter.
+	 *
+	 * @return A copy of `*this`.
+	 */
+	virtual BloomFilter* Clone() const = 0;
+
 	/**
 	 * Serializes the Bloom filter.
 	 *
@@ -147,13 +175,11 @@ public:
 	 */
 	static size_t K(size_t cells, size_t capacity);
 
-	/**
+	// Overridden from BloomFilter.
-	 * Merges two basic Bloom filters.
+	virtual bool Empty() const;
-	 *
+	virtual void Clear();
-	 * @return The merged Bloom filter.
+	virtual bool Merge(const BloomFilter* other);
-	 */
+	virtual BasicBloomFilter* Clone() const;
-	static BasicBloomFilter* Merge(const BasicBloomFilter* x,
-				       const BasicBloomFilter* y);
 
 protected:
 	DECLARE_SERIAL(BasicBloomFilter);
@@ -188,13 +214,11 @@ public:
 	 */
 	CountingBloomFilter(const Hasher* hasher, size_t cells, size_t width);
 
-	/**
+	// Overridden from BloomFilter.
-	 * Merges two counting Bloom filters.
+	virtual bool Empty() const;
-	 *
+	virtual void Clear();
-	 * @return The merged Bloom filter.
+	virtual bool Merge(const BloomFilter* other);
-	 */
+	virtual CountingBloomFilter* Clone() const;
-	static CountingBloomFilter* Merge(const CountingBloomFilter* x,
-					  const CountingBloomFilter* y);
 
 protected:
 	DECLARE_SERIAL(CountingBloomFilter);

src/probabilistic/CMakeLists.txt

@@ -15,4 +15,5 @@ set(probabilistic_SRCS
 bif_target(bloom-filter.bif)
 
 bro_add_subdir_library(probabilistic ${probabilistic_SRCS} ${BIF_OUTPUT_CC})
+
 add_dependencies(bro_probabilistic generate_outputs)

src/probabilistic/CounterVector.cc

@@ -70,6 +70,16 @@ bool CounterVector::Decrement(size_type cell, count_type value)
 	return carry;
 	}
 
+bool CounterVector::AllZero() const
+	{
+	return bits->AllZero();
+	}
+
+void CounterVector::Clear()
+	{
+	bits->Clear();
+	}
+
 CounterVector::count_type CounterVector::Count(size_type cell) const
 	{
 	assert(cell < Size());

src/probabilistic/CounterVector.h

@@ -77,6 +77,17 @@ public:
 	 */
 	count_type Count(size_type cell) const;
 
+	/**
+	 * Checks whether all counters are 0.
+	 * @return `true` iff all counters have the value 0.
+	 */
+	bool AllZero() const;
+
+	/**
+	 * Sets all counters to 0.
+	 */
+	void Clear();
+
 	/**
 	 * Retrieves the number of cells in the storage.
 	 *

src/probabilistic/Hasher.h

@@ -74,16 +74,25 @@ public:
 	 *
 	 * @param k The number of hash functions to apply.
 	 *
-	 * @param name The hasher's name.
+	 * @param name The hasher's name. Hashers with the same name should
+	 * provide consistent results.
 	 *
 	 * @return Returns a new hasher instance.
 	 */
 	static Hasher* Create(size_t k, const std::string& name);
 
 protected:
+	/**
+	 * Constructor.
+	 *
+	 * @param k the number of hash functions.
+	 *
+	 * @param name A name for the hasher. Hashers with the same name
+	 * should provide consistent results.
+	 */
 	Hasher(size_t k, const std::string& name);
 
-	private:
+private:
 	const size_t k;
 	std::string name;
 };

src/probabilistic/bloom-filter.bif

@@ -18,17 +18,25 @@ using namespace probabilistic;
 
 module GLOBAL;
 
-## Creates a basic Bloom filter.
+## Creates a basic Bloom filter. 
-##
+## 
+## .. note:: A Bloom filter can have a name associated with it. In the future,
+##    Bloom filters with the same name will be compatible across indepedent Bro
+##    instances, i.e., it will be possible to merge them. Currently, however, that is
+##    not yet supported. 
+## 
 ## fp: The desired false-positive rate.
 ##
 ## capacity: the maximum number of elements that guarantees a false-positive
 ## rate of *fp*.
 ##
 ## name: A name that uniquely identifies and seeds the Bloom filter. If empty,
-## the initialization will become dependent on the initial seed.
+## the filter will remain tied to the current Bro process. 
 ##
 ## Returns: A Bloom filter handle.
+##
+## .. bro:see:: bloomfilter_counting_init  bloomfilter_add bloomfilter_lookup
+##    bloomfilter_clear bloomfilter_merge
 function bloomfilter_basic_init%(fp: double, capacity: count,
                                  name: string &default=""%): opaque of bloomfilter
 	%{
@@ -46,19 +54,29 @@ function bloomfilter_basic_init%(fp: double, capacity: count,
 	%}
 
 ## Creates a counting Bloom filter.
+## 
+## .. note:: A Bloom filter can have a name associated with it. In the future,
+##    Bloom filters with the same name will be compatible across indepedent Bro
+##    instances, i.e., it will be possible to merge them. Currently, however, that is
+##    not yet supported. 
 ##
 ## k: The number of hash functions to use.
 ##
-## cells: The number of cells of the underlying counter vector.
+## cells: The number of cells of the underlying counter vector. As there's no
+## single answer to what's the best parameterization for a counting Bloom filter,
+## we refer to the Bloom filter literature here for choosing an appropiate value.  
 ##
 ## max: The maximum counter value associated with each each element described
 ## by *w = ceil(log_2(max))* bits. Each bit in the underlying counter vector
 ## becomes a cell of size *w* bits.
 ##
 ## name: A name that uniquely identifies and seeds the Bloom filter. If empty,
-## the initialization will become dependent on the initial seed.
+## the filter will remain tied to the current Bro process. 
 ##
 ## Returns: A Bloom filter handle.
+##
+## .. bro:see:: bloomfilter_basic_init bloomfilter_add bloomfilter_lookup
+##    bloomfilter_clear bloomfilter_merge
 function bloomfilter_counting_init%(k: count, cells: count, max: count,
 				    name: string &default=""%): opaque of bloomfilter
 	%{
@@ -82,6 +100,9 @@ function bloomfilter_counting_init%(k: count, cells: count, max: count,
 ## bf: The Bloom filter handle.
 ##
 ## x: The element to add.
+##
+## .. bro:see:: bloomfilter_counting_init bloomfilter_basic_init loomfilter_lookup
+##    bloomfilter_clear bloomfilter_merge 
 function bloomfilter_add%(bf: opaque of bloomfilter, x: any%): any
 	%{
 	BloomFilterVal* bfv = static_cast<BloomFilterVal*>(bf);
@@ -105,10 +126,16 @@ function bloomfilter_add%(bf: opaque of bloomfilter, x: any%): any
 ## x: The element to count.
 ##
 ## Returns: the counter associated with *x* in *bf*.
+##
+## .. bro:see:: bloomfilter_counting_init bloomfilter_basic_init
+##    bloomfilter_add bloomfilter_clear bloomfilter_merge 
 function bloomfilter_lookup%(bf: opaque of bloomfilter, x: any%): count
 	%{
 	const BloomFilterVal* bfv = static_cast<const BloomFilterVal*>(bf);
 
+	if ( bfv->Empty() )
+		return new Val(0, TYPE_COUNT);
+
 	if ( ! bfv->Type() )
 		reporter->Error("cannot perform lookup on untyped Bloom filter");
 
@@ -121,13 +148,38 @@ function bloomfilter_lookup%(bf: opaque of bloomfilter, x: any%): count
 	return new Val(0, TYPE_COUNT);
 	%}
 
-## Merges two Bloom filters.
+## Removes all elements from a Bloom filter. This function resets all bits in the
+## underlying bitvector back to 0 but does not change the parameterization of the
+## Bloom filter, such as the element type and the hasher seed.
+##
+## bf: The Bloom filter handle.
+##
+## .. bro:see:: bloomfilter_counting_init bloomfilter_basic_init
+##    bloomfilter_add bloomfilter_lookup bloomfilter_merge 
+function bloomfilter_clear%(bf: opaque of bloomfilter%): any
+	%{
+	BloomFilterVal* bfv = static_cast<BloomFilterVal*>(bf);
+
+	if ( bfv->Type() ) // Untyped Bloom filters are already empty.
+		bfv->Clear();
+
+	return 0;
+	%}
+
+## Merges two Bloom filters. 
+## 
+## .. note:: Currently Bloom filters created by different Bro instances cannot
+##    be merged. In the future, this will be supported as long as both filters
+##    are created with the same name.  
 ##
 ## bf1: The first Bloom filter handle.
 ##
 ## bf2: The second Bloom filter handle.
 ##
 ## Returns: The union of *bf1* and *bf2*.
+##
+## .. bro:see:: bloomfilter_counting_init bloomfilter_basic_init
+##    bloomfilter_add bloomfilter_lookup bloomfilter_clear
 function bloomfilter_merge%(bf1: opaque of bloomfilter,
 			    bf2: opaque of bloomfilter%): opaque of bloomfilter
 	%{