Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-16 21:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add namespaces for OpenDict files

Browse source
This commit is contained in:
Tim Wojtulewicz 2020-07-06 10:00:00 -07:00
parent 8f0e8037ba
commit 10e4694f8e
2 changed files with 59 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

52
src/OpenDict.cc
View file

@ -23,6 +23,8 @@
#define ASSERT_VALID(o) #define ASSERT_VALID(o)
#endif//DEBUG #endif//DEBUG
namespace zeek {
class IterCookie { class IterCookie {
public: public:
IterCookie(Dictionary* d) : d(d) {} IterCookie(Dictionary* d) : d(d) {}
@ -35,11 +37,11 @@ public:
int next = -1; //index for next valid entry. -1 is default not started yet. int next = -1; //index for next valid entry. -1 is default not started yet.
// Tracks the new entries inserted while iterating. Only used for robust cookies. // Tracks the new entries inserted while iterating. Only used for robust cookies.
std::vector<DictEntry>* inserted = nullptr; std::vector<detail::DictEntry>* inserted = nullptr;
// Tracks the entries already visited but were moved across the next iteration // Tracks the entries already visited but were moved across the next iteration
// point due to an insertion. Only used for robust cookies. // point due to an insertion. Only used for robust cookies.
std::vector<DictEntry>* visited = nullptr; std::vector<detail::DictEntry>* visited = nullptr;
void MakeRobust() void MakeRobust()
{ {
@ -48,8 +50,8 @@ public:
ASSERT(d && d->cookies); ASSERT(d && d->cookies);
robust = true; robust = true;
inserted = new std::vector<DictEntry>(); inserted = new std::vector<detail::DictEntry>();
visited = new std::vector<DictEntry>(); visited = new std::vector<detail::DictEntry>();
d->cookies->push_back(this); d->cookies->push_back(this);
} }
@ -71,6 +73,8 @@ public:
} }
}; };
// namespace detail
TEST_SUITE_BEGIN("Dict"); TEST_SUITE_BEGIN("Dict");
TEST_CASE("dict construction") TEST_CASE("dict construction")
@ -402,14 +406,14 @@ size_t Dictionary::MemoryAllocation() const
size_t size = padded_sizeof(*this); size_t size = padded_sizeof(*this);
if ( table ) if ( table )
{ {
size += pad_size(Capacity() * sizeof(DictEntry)); size += pad_size(Capacity() * sizeof(detail::DictEntry));
for ( int i = Capacity()-1; i>=0; i-- ) for ( int i = Capacity()-1; i>=0; i-- )
if ( ! table[i].Empty() && table[i].key_size > 8 ) if ( ! table[i].Empty() && table[i].key_size > 8 )
size += pad_size(table[i].key_size); size += pad_size(table[i].key_size);
} }
if ( order ) if ( order )
size += padded_sizeof(std::vector<DictEntry>) + pad_size(sizeof(DictEntry) * order->capacity()); size += padded_sizeof(std::vector<detail::DictEntry>) + pad_size(sizeof(detail::DictEntry) * order->capacity());
return size; return size;
} }
@ -528,7 +532,7 @@ void Dictionary::Dump(int level) const
////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////
//Initialization. //Initialization.
//////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////
Dictionary::Dictionary(dict_order ordering, int initial_size) Dictionary::Dictionary(DictOrder ordering, int initial_size)
{ {
if ( initial_size > 0 ) if ( initial_size > 0 )
{ {
@ -539,7 +543,7 @@ Dictionary::Dictionary(dict_order ordering, int initial_size)
} }
if ( ordering == ORDERED ) if ( ordering == ORDERED )
order = new std::vector<DictEntry>; order = new std::vector<detail::DictEntry>;
} }
Dictionary::~Dictionary() Dictionary::~Dictionary()
@ -584,7 +588,7 @@ void Dictionary::Clear()
void Dictionary::Init() void Dictionary::Init()
{ {
ASSERT(! table); ASSERT(! table);
table = (DictEntry*)malloc(sizeof(DictEntry)*Capacity(true)); table = (detail::DictEntry*)malloc(sizeof(detail::DictEntry)*Capacity(true));
for (int i = Capacity()-1; i >= 0; i--) for (int i = Capacity()-1; i >= 0; i--)
table[i].SetEmpty(); table[i].SetEmpty();
} }
@ -746,7 +750,7 @@ void* Dictionary::Insert(void* key, int key_size, hash_t hash, void* val, bool c
else else
{ {
// Allocate memory for key if necesary. Key is updated to reflect internal key if necessary. // Allocate memory for key if necesary. Key is updated to reflect internal key if necessary.
DictEntry entry(key, key_size, hash, val, insert_distance, copy_key); detail::DictEntry entry(key, key_size, hash, val, insert_distance, copy_key);
InsertRelocateAndAdjust(entry, insert_position); InsertRelocateAndAdjust(entry, insert_position);
if ( order ) if ( order )
order->push_back(entry); order->push_back(entry);
@ -769,7 +773,7 @@ void* Dictionary::Insert(void* key, int key_size, hash_t hash, void* val, bool c
} }
///e.distance is adjusted to be the one at insert_position. ///e.distance is adjusted to be the one at insert_position.
void Dictionary::InsertRelocateAndAdjust(DictEntry& entry, int insert_position) void Dictionary::InsertRelocateAndAdjust(detail::DictEntry& entry, int insert_position)
{ {
#ifdef DEBUG #ifdef DEBUG
entry.bucket = BucketByHash(entry.hash,log2_buckets); entry.bucket = BucketByHash(entry.hash,log2_buckets);
@ -790,7 +794,7 @@ void Dictionary::InsertRelocateAndAdjust(DictEntry& entry, int insert_position)
} }
/// insert entry into position, relocate other entries when necessary. /// insert entry into position, relocate other entries when necessary.
void Dictionary::InsertAndRelocate(DictEntry& entry, int insert_position, int* last_affected_position) void Dictionary::InsertAndRelocate(detail::DictEntry& entry, int insert_position, int* last_affected_position)
{///take out the head of cluster and append to the end of the cluster. {///take out the head of cluster and append to the end of the cluster.
while ( true ) while ( true )
{ {
@ -824,7 +828,7 @@ void Dictionary::InsertAndRelocate(DictEntry& entry, int insert_position, int* l
} }
/// Adjust Cookies on Insert. /// Adjust Cookies on Insert.
void Dictionary::AdjustOnInsert(IterCookie* c, const DictEntry& entry, int insert_position, int last_affected_position) void Dictionary::AdjustOnInsert(IterCookie* c, const detail::DictEntry& entry, int insert_position, int last_affected_position)
{ {
ASSERT(c); ASSERT(c);
ASSERT_VALID(c); ASSERT_VALID(c);
@ -843,7 +847,7 @@ void Dictionary::SizeUp()
int prev_capacity = Capacity(); int prev_capacity = Capacity();
log2_buckets++; log2_buckets++;
int capacity = Capacity(); int capacity = Capacity();
table = (DictEntry*)realloc(table, capacity*sizeof(DictEntry)); table = (detail::DictEntry*)realloc(table, capacity*sizeof(detail::DictEntry));
for ( int i = prev_capacity; i < capacity; i++ ) for ( int i = prev_capacity; i < capacity; i++ )
table[i].SetEmpty(); table[i].SetEmpty();
@ -872,7 +876,7 @@ void* Dictionary::Remove(const void* key, int key_size, hash_t hash, bool dont_d
if ( position < 0 ) if ( position < 0 )
return nullptr; return nullptr;
DictEntry entry = RemoveRelocateAndAdjust(position); detail::DictEntry entry = RemoveRelocateAndAdjust(position);
num_entries--; num_entries--;
ASSERT(num_entries >= 0); ASSERT(num_entries >= 0);
//e is about to be invalid. remove it from all references. //e is about to be invalid. remove it from all references.
@ -885,10 +889,10 @@ void* Dictionary::Remove(const void* key, int key_size, hash_t hash, bool dont_d
return v; return v;
} }
DictEntry Dictionary::RemoveRelocateAndAdjust(int position) detail::DictEntry Dictionary::RemoveRelocateAndAdjust(int position)
{ {
int last_affected_position = position; int last_affected_position = position;
DictEntry entry = RemoveAndRelocate(position, &last_affected_position); detail::DictEntry entry = RemoveAndRelocate(position, &last_affected_position);
#ifdef DEBUG #ifdef DEBUG
//validation: index to i-1 should be continuous without empty spaces. //validation: index to i-1 should be continuous without empty spaces.
@ -903,12 +907,12 @@ DictEntry Dictionary::RemoveRelocateAndAdjust(int position)
return entry; return entry;
} }
DictEntry Dictionary::RemoveAndRelocate(int position, int* last_affected_position) detail::DictEntry Dictionary::RemoveAndRelocate(int position, int* last_affected_position)
{ {
//fill the empty position with the tail of the cluster of position+1. //fill the empty position with the tail of the cluster of position+1.
ASSERT(position >= 0 && position < Capacity() && ! table[position].Empty()); ASSERT(position >= 0 && position < Capacity() && ! table[position].Empty());
DictEntry entry = table[position]; detail::DictEntry entry = table[position];
while ( true ) while ( true )
{ {
if ( position == Capacity() - 1 || table[position+1].Empty() || table[position+1].distance == 0 ) if ( position == Capacity() - 1 || table[position+1].Empty() || table[position+1].distance == 0 )
@ -928,7 +932,7 @@ DictEntry Dictionary::RemoveAndRelocate(int position, int* last_affected_positio
return entry; return entry;
} }
void Dictionary::AdjustOnRemove(IterCookie* c, const DictEntry& entry, int position, int last_affected_position) void Dictionary::AdjustOnRemove(IterCookie* c, const detail::DictEntry& entry, int position, int last_affected_position)
{ {
ASSERT_VALID(c); ASSERT_VALID(c);
c->inserted->erase(std::remove(c->inserted->begin(), c->inserted->end(), entry), c->inserted->end()); c->inserted->erase(std::remove(c->inserted->begin(), c->inserted->end(), entry), c->inserted->end());
@ -981,7 +985,7 @@ bool Dictionary::Remap(int position, int* new_position)
//equal because 1: it's a new item, 2: it's an old item, but new bucket is the same as old. 50% of old items act this way due to fibhash. //equal because 1: it's a new item, 2: it's an old item, but new bucket is the same as old. 50% of old items act this way due to fibhash.
if ( current == expected ) if ( current == expected )
return false; return false;
DictEntry entry = RemoveAndRelocate(position); // no iteration cookies to adjust, no need for last_affected_position. detail::DictEntry entry = RemoveAndRelocate(position); // no iteration cookies to adjust, no need for last_affected_position.
#ifdef DEBUG #ifdef DEBUG
entry.bucket = expected; entry.bucket = expected;
#endif//DEBUG #endif//DEBUG
@ -1004,7 +1008,7 @@ void* Dictionary::NthEntry(int n, const void*& key, int& key_size) const
{ {
if ( ! order || n < 0 || n >= Length() ) if ( ! order || n < 0 || n >= Length() )
return nullptr; return nullptr;
DictEntry entry = (*order)[n]; detail::DictEntry entry = (*order)[n];
key = entry.GetKey(); key = entry.GetKey();
key_size = entry.key_size; key_size = entry.key_size;
return entry.value; return entry.value;
@ -1052,7 +1056,7 @@ void* Dictionary::NextEntryNonConst(HashKey*& h, IterCookie*& c, bool return_has
{ {
// Return the last one. Order doesn't matter, // Return the last one. Order doesn't matter,
// and removing from the tail is cheaper. // and removing from the tail is cheaper.
DictEntry e = c->inserted->back(); detail::DictEntry e = c->inserted->back();
if ( return_hash ) if ( return_hash )
h = new HashKey(e.GetKey(), e.key_size, e.hash); h = new HashKey(e.GetKey(), e.key_size, e.hash);
void* v = e.value; void* v = e.value;
@ -1126,3 +1130,5 @@ void Dictionary::StopIteration(IterCookie* cookie) const
Dictionary* dp = const_cast<Dictionary*>(this); Dictionary* dp = const_cast<Dictionary*>(this);
dp->StopIterationNonConst(cookie); dp->StopIterationNonConst(cookie);
} }
} // namespace zeek

47
src/OpenDict.h
View file

@ -7,8 +7,13 @@
#include "Hash.h" #include "Hash.h"
class Dictionary; ZEEK_FORWARD_DECLARE_NAMESPACED(IterCookie, zeek);
class IterCookie; ZEEK_FORWARD_DECLARE_NAMESPACED(DictEntry, zeek::detail);
// Type for function to be called when deleting elements.
typedef void (*dict_delete_func)(void*);
namespace zeek {
// Default number of hash buckets in dictionary. The dictionary will increase the size // Default number of hash buckets in dictionary. The dictionary will increase the size
// of the hash table as needed. // of the hash table as needed.
@ -44,14 +49,13 @@ constexpr uint8_t DICT_THRESHOLD_BITS = 3;
// bucket at which to start looking for the next value to return. // bucket at which to start looking for the next value to return.
constexpr uint16_t TOO_FAR_TO_REACH = 0xFFFF; constexpr uint16_t TOO_FAR_TO_REACH = 0xFFFF;
enum dict_order { ORDERED, UNORDERED }; enum DictOrder { ORDERED, UNORDERED };
// Type for function to be called when deleting elements.
typedef void (*dict_delete_func)(void*);
// A dict_delete_func that just calls delete. // A dict_delete_func that just calls delete.
extern void generic_delete_func(void*); extern void generic_delete_func(void*);
namespace detail {
/** /**
* An entry stored in the dictionary. * An entry stored in the dictionary.
*/ */
@ -140,6 +144,8 @@ public:
} }
}; };
} // namespace detail
/** /**
* A dictionary type that uses clustered hashing, a variation of Robinhood/Open Addressing * A dictionary type that uses clustered hashing, a variation of Robinhood/Open Addressing
* hashing. The following posts help to understand the implementation: * hashing. The following posts help to understand the implementation:
@ -154,7 +160,7 @@ public:
*/ */
class Dictionary{ class Dictionary{
public: public:
explicit Dictionary(dict_order ordering = UNORDERED, int initial_size = DEFAULT_DICT_SIZE); explicit Dictionary(DictOrder ordering = UNORDERED, int initial_size = DEFAULT_DICT_SIZE);
~Dictionary(); ~Dictionary();
// Member functions for looking up a key, inserting/changing its // Member functions for looking up a key, inserting/changing its
@ -254,7 +260,7 @@ public:
void DumpKeys() const; void DumpKeys() const;
private: private:
friend IterCookie; friend zeek::IterCookie;
/// Buckets of the table, not including overflow size. /// Buckets of the table, not including overflow size.
int Buckets(bool expected=false) const; int Buckets(bool expected=false) const;
@ -310,22 +316,22 @@ private:
int* insert_position = nullptr, int* insert_distance = nullptr); int* insert_position = nullptr, int* insert_distance = nullptr);
/// Insert entry, Adjust cookies when necessary. /// Insert entry, Adjust cookies when necessary.
void InsertRelocateAndAdjust(DictEntry& entry, int insert_position); void InsertRelocateAndAdjust(detail::DictEntry& entry, int insert_position);
/// insert entry into position, relocate other entries when necessary. /// insert entry into position, relocate other entries when necessary.
void InsertAndRelocate(DictEntry& entry, int insert_position, int* last_affected_position = nullptr); void InsertAndRelocate(detail::DictEntry& entry, int insert_position, int* last_affected_position = nullptr);
/// Adjust Cookies on Insert. /// Adjust Cookies on Insert.
void AdjustOnInsert(IterCookie* c, const DictEntry& entry, int insert_position, int last_affected_position); void AdjustOnInsert(IterCookie* c, const detail::DictEntry& entry, int insert_position, int last_affected_position);
///Remove, Relocate & Adjust cookies. ///Remove, Relocate & Adjust cookies.
DictEntry RemoveRelocateAndAdjust(int position); detail::DictEntry RemoveRelocateAndAdjust(int position);
///Remove & Relocate ///Remove & Relocate
DictEntry RemoveAndRelocate(int position, int* last_affected_position = nullptr); detail::DictEntry RemoveAndRelocate(int position, int* last_affected_position = nullptr);
///Adjust safe cookies after Removal of entry at position. ///Adjust safe cookies after Removal of entry at position.
void AdjustOnRemove(IterCookie* c, const DictEntry& entry, int position, int last_affected_position); void AdjustOnRemove(IterCookie* c, const detail::DictEntry& entry, int position, int last_affected_position);
bool Remapping() const { return remap_end >= 0;} //remap in reverse order. bool Remapping() const { return remap_end >= 0;} //remap in reverse order.
@ -360,11 +366,11 @@ private:
uint64_t cum_entries = 0; uint64_t cum_entries = 0;
dict_delete_func delete_func = nullptr; dict_delete_func delete_func = nullptr;
DictEntry* table = nullptr; detail::DictEntry* table = nullptr;
std::vector<IterCookie*>* cookies = nullptr; std::vector<IterCookie*>* cookies = nullptr;
// Order means the order of insertion. means no deletion until exit. will be inefficient. // Order means the order of insertion. means no deletion until exit. will be inefficient.
std::vector<DictEntry>* order = nullptr; std::vector<detail::DictEntry>* order = nullptr;
}; };
/* /*
@ -373,7 +379,7 @@ private:
template<typename T> template<typename T>
class PDict : public Dictionary { class PDict : public Dictionary {
public: public:
explicit PDict(dict_order ordering = UNORDERED, int initial_size = 0) : explicit PDict(DictOrder ordering = UNORDERED, int initial_size = 0) :
Dictionary(ordering, initial_size) {} Dictionary(ordering, initial_size) {}
T* Lookup(const char* key) const T* Lookup(const char* key) const
{ {
@ -405,4 +411,11 @@ public:
{ return (T*) Dictionary::NextEntry(h, cookie, true); } { return (T*) Dictionary::NextEntry(h, cookie, true); }
T* RemoveEntry(const HashKey* key) T* RemoveEntry(const HashKey* key)
{ return (T*) Remove(key->Key(), key->Size(), key->Hash()); } { return (T*) Remove(key->Key(), key->Size(), key->Hash()); }
T* RemoveEntry(const HashKey& key)
{ return (T*) Remove(key.Key(), key.Size(), key.Hash()); }
}; };
} // namespace zeek
using Dictionary [[deprecated("Remove in v4.1. Use zeek::Dictionary instead.")]] = zeek::Dictionary;
template<typename T> using PDict [[deprecated("Remove in v4.1. Use zeek::PDict instead.")]] = zeek::PDict<T>;