mirror of
https://github.com/zeek/zeek.git
synced 2025-10-05 16:18:19 +00:00
Merge remote-tracking branch 'origin/topic/johanna/table-changes'
* origin/topic/johanna/table-changes: (26 commits) TableSync: try to make test more robust & add debug output Increase timeouts to see if FreeBSD will be happy with this. Try to make FreeBSD test happy with larger timeout. TableSync: refactor common functionality into function TableSync: don't raise &on_change, smaller fixes TableSync: rename auto_store -> table_store SyncTables: address feedback part 1 - naming (broker and zeek) BrokerStore <-> Zeek Tables: cleanup and bug workaround Zeek Table<->Brokerstore: cleanup, documentation, small fixes BrokerStore<->Zeek table: adopt to recent Zeek API changes BrokerStore<->Zeek Tables Fix a few small test failures. BrokerStore<->Zeek tables: allow setting storage location & tests BrokerStore<->Zeek tables: &backend works for in-memory stores. BrokerStore<->Zeek table - introdude &backend attribute BrokerStore<->Zeek tables: test for clones synchronizing to a master BrokerStore<->Zeek tables: load persistent tables on startup. Brokerstore<->Tables: attribute conflicts Zeek/Brokerstore updates: expiration Zeek/Brokerstore updates: add test that includes updates from clones Zeek/Brokerstore updates: first working end-to-end test ...
This commit is contained in:
Robin Sommer
commit
c3f4971eb2
50 changed files with 2290 additions and 78 deletions
186
src/Val.cc
186
src/Val.cc
|
|
@ -37,6 +37,8 @@
|
|||
#include "ID.h"
|
||||
|
||||
#include "broker/Data.h"
|
||||
#include "broker/Store.h"
|
||||
#include "broker/Manager.h"
|
||||
|
||||
#include "threading/formatters/JSON.h"
|
||||
|
||||
|
|
@ -1511,6 +1513,16 @@ void TableVal::SetAttrs(detail::AttributesPtr a)
|
|||
|
||||
if ( cf )
|
||||
change_func = cf->GetExpr();
|
||||
|
||||
auto bs = attrs->Find(zeek::detail::ATTR_BROKER_STORE);
|
||||
if ( bs && broker_store.empty() )
|
||||
{
|
||||
IntrusivePtr<Val> c = bs->GetExpr()->Eval(nullptr);
|
||||
assert(c);
|
||||
assert(c->GetType()->Tag() == zeek::TYPE_STRING);
|
||||
broker_store = c->AsStringVal()->AsString()->CheckString();
|
||||
broker_mgr->AddForwardedStore(broker_store, {NewRef{}, this});
|
||||
}
|
||||
}
|
||||
|
||||
void TableVal::CheckExpireAttr(detail::AttrTag at)
|
||||
|
|
@ -1539,7 +1551,7 @@ void TableVal::CheckExpireAttr(detail::AttrTag at)
|
|||
}
|
||||
}
|
||||
|
||||
bool TableVal::Assign(ValPtr index, ValPtr new_val)
|
||||
bool TableVal::Assign(ValPtr index, ValPtr new_val, bool broker_forward)
|
||||
{
|
||||
auto k = MakeHashKey(*index);
|
||||
|
||||
|
|
@ -1549,7 +1561,7 @@ bool TableVal::Assign(ValPtr index, ValPtr new_val)
|
|||
return false;
|
||||
}
|
||||
|
||||
return Assign(std::move(index), std::move(k), std::move(new_val));
|
||||
return Assign(std::move(index), std::move(k), std::move(new_val), broker_forward);
|
||||
}
|
||||
|
||||
bool TableVal::Assign(Val* index, Val* new_val)
|
||||
|
|
@ -1558,7 +1570,7 @@ bool TableVal::Assign(Val* index, Val* new_val)
|
|||
}
|
||||
|
||||
bool TableVal::Assign(ValPtr index, std::unique_ptr<HashKey> k,
|
||||
ValPtr new_val)
|
||||
ValPtr new_val, bool broker_forward)
|
||||
{
|
||||
bool is_set = table_type->IsSet();
|
||||
|
||||
|
|
@ -1591,11 +1603,19 @@ bool TableVal::Assign(ValPtr index, std::unique_ptr<HashKey> k,
|
|||
|
||||
Modified();
|
||||
|
||||
if ( change_func )
|
||||
if ( change_func || ( broker_forward && ! broker_store.empty() ) )
|
||||
{
|
||||
auto change_index = index ? std::move(index) : RecreateIndex(k_copy);
|
||||
const auto& v = old_entry_val ? old_entry_val->GetVal() : new_entry_val->GetVal();
|
||||
CallChangeFunc(change_index.get(), v, old_entry_val ? ELEMENT_CHANGED : ELEMENT_NEW);
|
||||
auto change_index = index ? std::move(index)
|
||||
: RecreateIndex(k_copy);
|
||||
|
||||
if ( broker_forward && ! broker_store.empty() )
|
||||
SendToStore(change_index.get(), new_entry_val, old_entry_val ? ELEMENT_CHANGED : ELEMENT_NEW);
|
||||
|
||||
if ( change_func )
|
||||
{
|
||||
const auto& v = old_entry_val ? old_entry_val->GetVal() : new_entry_val->GetVal();
|
||||
CallChangeFunc(change_index, v, old_entry_val ? ELEMENT_CHANGED : ELEMENT_NEW);
|
||||
}
|
||||
}
|
||||
|
||||
delete old_entry_val;
|
||||
|
|
@ -2061,7 +2081,7 @@ ListValPtr TableVal::RecreateIndex(const HashKey& k) const
|
|||
return table_hash->RecoverVals(k);
|
||||
}
|
||||
|
||||
void TableVal::CallChangeFunc(const Val* index,
|
||||
void TableVal::CallChangeFunc(const ValPtr& index,
|
||||
const ValPtr& old_value,
|
||||
OnChangeType tpe)
|
||||
{
|
||||
|
|
@ -2076,9 +2096,7 @@ void TableVal::CallChangeFunc(const Val* index,
|
|||
auto thefunc = change_func->Eval(nullptr);
|
||||
|
||||
if ( ! thefunc )
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if ( thefunc->GetType()->Tag() != TYPE_FUNC )
|
||||
{
|
||||
|
|
@ -2087,10 +2105,14 @@ void TableVal::CallChangeFunc(const Val* index,
|
|||
}
|
||||
|
||||
const zeek::Func* f = thefunc->AsFunc();
|
||||
auto lv = index->AsListVal();
|
||||
zeek::Args vl;
|
||||
|
||||
// we either get passed the raw index_val - or a ListVal with exactly one element.
|
||||
if ( index->GetType()->Tag() == zeek::TYPE_LIST )
|
||||
vl.reserve(2 + index->AsListVal()->Length() + table_type->IsTable());
|
||||
else
|
||||
vl.reserve(3 + table_type->IsTable());
|
||||
|
||||
Args vl;
|
||||
vl.reserve(2 + lv->Length() + table_type->IsTable());
|
||||
vl.emplace_back(NewRef{}, this);
|
||||
|
||||
switch ( tpe )
|
||||
|
|
@ -2108,8 +2130,14 @@ void TableVal::CallChangeFunc(const Val* index,
|
|||
vl.emplace_back(BifType::Enum::TableChange->GetEnumVal(BifEnum::TableChange::TABLE_ELEMENT_EXPIRED));
|
||||
}
|
||||
|
||||
for ( const auto& v : lv->Vals() )
|
||||
vl.emplace_back(v);
|
||||
|
||||
if ( index->GetType()->Tag() == zeek::TYPE_LIST )
|
||||
{
|
||||
for ( const auto& v : index->AsListVal()->Vals() )
|
||||
vl.emplace_back(v);
|
||||
}
|
||||
else
|
||||
vl.emplace_back(index);
|
||||
|
||||
if ( table_type->IsTable() )
|
||||
vl.emplace_back(old_value);
|
||||
|
|
@ -2124,9 +2152,115 @@ void TableVal::CallChangeFunc(const Val* index,
|
|||
in_change_func = false;
|
||||
}
|
||||
|
||||
ValPtr TableVal::Remove(const Val& index)
|
||||
void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val, OnChangeType tpe)
|
||||
{
|
||||
if ( broker_store.empty() || ! index )
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
auto handle = broker_mgr->LookupStore(broker_store);
|
||||
|
||||
if ( ! handle )
|
||||
return;
|
||||
|
||||
// we either get passed the raw index_val - or a ListVal with exactly one element.
|
||||
// Since Broker does not support ListVals, we have to unoll this in the second case.
|
||||
const Val* index_val;
|
||||
if ( index->GetType()->Tag() == zeek::TYPE_LIST )
|
||||
{
|
||||
if ( index->AsListVal()->Length() != 1 )
|
||||
{
|
||||
zeek::emit_builtin_error("table with complex index not supported for &broker_store");
|
||||
return;
|
||||
}
|
||||
|
||||
index_val = index->AsListVal()->Idx(0).get();
|
||||
}
|
||||
else
|
||||
{
|
||||
index_val = index;
|
||||
}
|
||||
|
||||
auto broker_index = bro_broker::val_to_data(index_val);
|
||||
|
||||
if ( ! broker_index )
|
||||
{
|
||||
zeek::emit_builtin_error("invalid Broker data conversation for table index");
|
||||
return;
|
||||
}
|
||||
|
||||
switch ( tpe )
|
||||
{
|
||||
case ELEMENT_NEW:
|
||||
case ELEMENT_CHANGED:
|
||||
{
|
||||
broker::optional<broker::timespan> expiry;
|
||||
|
||||
auto expire_time = GetExpireTime();
|
||||
if ( expire_time == 0 )
|
||||
// Entry is set to immediately expire. Let's not forward it.
|
||||
break;
|
||||
|
||||
if ( expire_time > 0 )
|
||||
{
|
||||
if ( attrs->Find(zeek::detail::ATTR_EXPIRE_CREATE) )
|
||||
{
|
||||
// for create expiry, we have to substract the already elapsed time from the expiry.
|
||||
auto e = expire_time - (network_time - new_entry_val->ExpireAccessTime());
|
||||
if ( e <= 0 )
|
||||
// element already expired? Let's not insert it.
|
||||
break;
|
||||
|
||||
expiry = bro_broker::convert_expiry(e);
|
||||
}
|
||||
else
|
||||
expiry = bro_broker::convert_expiry(expire_time);
|
||||
}
|
||||
|
||||
if ( table_type->IsSet() )
|
||||
handle->store.put(std::move(*broker_index), broker::data(), expiry);
|
||||
else
|
||||
{
|
||||
if ( ! new_entry_val )
|
||||
{
|
||||
zeek::emit_builtin_error("did not receive new value for Broker datastore send operation");
|
||||
return;
|
||||
}
|
||||
|
||||
auto new_value = new_entry_val->GetVal().get();
|
||||
auto broker_val = bro_broker::val_to_data(new_value);
|
||||
if ( ! broker_val )
|
||||
{
|
||||
zeek::emit_builtin_error("invalid Broker data conversation for table value");
|
||||
return;
|
||||
}
|
||||
|
||||
handle->store.put(std::move(*broker_index), std::move(*broker_val), expiry);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case ELEMENT_REMOVED:
|
||||
handle->store.erase(std::move(*broker_index));
|
||||
break;
|
||||
|
||||
case ELEMENT_EXPIRED:
|
||||
// we do nothing here. The Broker store does its own expiration - so the element
|
||||
// should expire at about the same time.
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch ( InterpreterException& e )
|
||||
{
|
||||
zeek::emit_builtin_error("The previous error was encountered while trying to resolve the &broker_store attribute of the set/table. Potentially the Broker::Store has not been initialized before being used.");
|
||||
}
|
||||
}
|
||||
|
||||
ValPtr TableVal::Remove(const Val& index, bool broker_forward)
|
||||
{
|
||||
auto k = MakeHashKey(index);
|
||||
|
||||
TableEntryVal* v = k ? AsNonConstTable()->RemoveEntry(k.get()) : nullptr;
|
||||
ValPtr va;
|
||||
|
||||
|
|
@ -2140,8 +2274,15 @@ ValPtr TableVal::Remove(const Val& index)
|
|||
|
||||
Modified();
|
||||
|
||||
if ( broker_forward && ! broker_store.empty() )
|
||||
SendToStore(&index, nullptr, ELEMENT_REMOVED);
|
||||
|
||||
if ( change_func )
|
||||
CallChangeFunc(&index, va, ELEMENT_REMOVED);
|
||||
{
|
||||
// this is totally cheating around the fact that we need a Intrusive pointer.
|
||||
IntrusivePtr<Val> changefunc_val = RecreateIndex(*(k.get()));
|
||||
CallChangeFunc(changefunc_val, va, ELEMENT_REMOVED);
|
||||
}
|
||||
|
||||
return va;
|
||||
}
|
||||
|
|
@ -2166,10 +2307,14 @@ ValPtr TableVal::Remove(const HashKey& k)
|
|||
|
||||
Modified();
|
||||
|
||||
if ( change_func && va )
|
||||
if ( va && ( change_func || ! broker_store.empty() ) )
|
||||
{
|
||||
auto index = table_hash->RecoverVals(k);
|
||||
CallChangeFunc(index.get(), va, ELEMENT_REMOVED);
|
||||
if ( ! broker_store.empty() )
|
||||
SendToStore(index.get(), nullptr, ELEMENT_REMOVED);
|
||||
|
||||
if ( change_func && va )
|
||||
CallChangeFunc(index, va, ELEMENT_REMOVED);
|
||||
}
|
||||
|
||||
return va;
|
||||
|
|
@ -2473,7 +2618,8 @@ void TableVal::DoExpire(double t)
|
|||
{
|
||||
if ( ! idx )
|
||||
idx = RecreateIndex(*k);
|
||||
CallChangeFunc(idx.get(), v->GetVal(), ELEMENT_EXPIRED);
|
||||
|
||||
CallChangeFunc(idx, v->GetVal(), ELEMENT_EXPIRED);
|
||||
}
|
||||
|
||||
delete v;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue