Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

BrokerStore<->Zeek tables: load persistent tables on startup.

Browse source 
This currently only handles the most basic case, and is not thoroughly
tested.

When initializing a master store, we now check if there already is data
in it. If yes, we load it directly into the zeek table when the store is
created. We assume that this is happening at Zeek startup - and are
supremely evil and just load it synchronously. Which could block
execution for a bit for larger stores.

That being said - this might sidestep other issues that would arise when
doing this async (like scripts already inserting data).

Next step: check if this approach also works for clones.
This commit is contained in:
Johanna Amann 2020-06-26 16:10:15 -07:00
parent 38a3d67643
commit b9fe79c697
5 changed files with 180 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

61
src/broker/Manager.cc
View file

@ -1608,15 +1608,66 @@ StoreHandleVal* Manager::MakeMaster(const string& name, broker::backend type,
iosource_mgr->RegisterFd(handle->proxy.mailbox().descriptor(), this);
CheckForwarding(name);
if ( bstate->endpoint.use_real_time() )
return handle;
if ( ! bstate->endpoint.use_real_time() )
// Wait for master to become available/responsive.
// Possibly avoids timeouts in scripts during unit tests.
handle->store.exists("");
BrokerStoreToZeekTable(name, handle);
// Wait for master to become available/responsive.
// Possibly avoids timeouts in scripts during unit tests.
handle->store.exists("");
return handle;
}
void Manager::BrokerStoreToZeekTable(const std::string& name, const StoreHandleVal* handle)
{
// consider if it might be wise to disable &on_change while filling the table
if ( ! handle->forward_to )
return;
auto keys = handle->store.keys();
if ( ! keys )
return;
auto set = caf::get_if<broker::set>(&(keys->get_data()));
auto table = handle->forward_to;
const auto& its = table->GetType()->AsTableType()->IndexTypes();
bool is_set = table->GetType()->IsSet();
assert( its.size() == 1 );
for ( const auto key : *set )
{
auto zeek_key = data_to_val(key, its[0].get());
if ( ! zeek_key )
{
reporter->Error("Failed to convert key %s while importing broker store to table for store %s. Aborting import.", to_string(key).c_str(), name.c_str());
// just abort - this probably means the types are incompatible
return;
}
if ( is_set )
{
table->Assign(zeek_key, nullptr, false);
continue;
}
auto value = handle->store.get(key);
if ( ! value )
{
reporter->Error("Failed to load value for key %s while importing broker store %s to table", to_string(key).c_str(), name.c_str());
continue;
}
auto zeek_value = data_to_val(*value, table->GetType()->Yield().get());
if ( ! zeek_value )
{
reporter->Error("Could not convert %s to table value while trying to import broker store %s. Aborting import.", to_string(value).c_str(), name.c_str());
return;
}
table->Assign(zeek_key, zeek_value, false);
}
return;
}
StoreHandleVal* Manager::MakeClone(const string& name, double resync_interval,
double stale_interval,
double mutation_buffer_interval)