Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

SyncTables: address feedback part 1 - naming (broker and zeek)

Browse source 
This commit fixes capitalization issues.
This commit is contained in:
Johanna Amann 2020-07-17 10:56:28 -07:00
parent 1888d6acae
commit 6d2aa84952
11 changed files with 35 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

2
NEWS
View file

@ -57,7 +57,7 @@ New Functionality
- Broker Store table synchronization (Experimental).
Zeek now supports synchronizing tables/sets across clusters using a backing broker
Zeek now supports synchronizing tables/sets across clusters using a backing Broker
store. The same feature also allows persistent storage of data in tables/sets
over Zeek restarts. This feature is implemented using the new ```&backend``` attribute.

10
scripts/base/frameworks/broker/store.zeek
View file

@ -25,14 +25,14 @@ export {
## A negative/zero value indicates to never buffer commands.
const default_clone_mutation_buffer_interval = 2min &redef;
## If set to true, the current node is the master node for broker stores
## backing zeek tables. By default this value will be automatically set to
## If set to true, the current node is the master node for Broker stores
## backing Zeek tables. By default this value will be automatically set to
## true in standalone mode, or on the manager node of a cluster. This value
## should not typically be changed manually.
const auto_store_master = T &redef;
## The directory used for storing persistent database files when using brokerstore
## backed zeek tables.
## The directory used for storing persistent database files when using Broker store
## backed Zeek tables.
const auto_store_db_directory = "." &redef;
## Whether a data store query could be completed or not.
@ -393,7 +393,7 @@ export {
## d: the communication data.
##
## Returns: The data type associated with the communication data.
## Note that broker represents records in the same way as
## Note that Broker represents records in the same way as
## vectors, so there is no "record" type.
global data_type: function(d: Broker::Data): Broker::DataType;

8
scripts/base/frameworks/cluster/broker-stores.zeek
View file

@ -1,9 +1,9 @@
##! This script deals with the cluster parts of broker backed zeek tables.
##! This script deals with the cluster parts of Broker backed Zeek tables.
##! It makes sure that the master store is set correctly and that clones
##! are automatically created on the non-manager nodes.
# Note - this script should become unnecessary in the future, when we just can
# speculatively attach clones. This should be possible once the new ALM broker
# speculatively attach clones. This should be possible once the new ALM Broker
# transport becomes available.
@load ./main
@ -11,8 +11,8 @@
module Broker;
export {
## Event that is used by the manager to announce the master stores for zeek backed
## tables that is uses.
## Event that is used by the manager to announce the master stores for Broker backed
## tables.
global announce_masters: event(masters: set[string]);
}

2
scripts/base/init-bare.zeek
View file

@ -747,7 +747,7 @@ type script_id: record {
enum_constant: bool; ##< True if the identifier is an enum value.
option_value: bool; ##< True if the identifier is an option.
redefinable: bool; ##< True if the identifier is declared with the :zeek:attr:`&redef` attribute.
broker_backend: bool; ##< True if the identifier has a broker backend defined using the :zeek:attr:`&backend` attribute.
broker_backend: bool; ##< True if the identifier has a Broker backend defined using the :zeek:attr:`&backend` attribute.
value: any &optional; ##< The current value of the identifier.
};

6
src/Attr.h
View file

@ -42,9 +42,9 @@ enum AttrTag {
ATTR_TYPE_COLUMN, // for input framework
ATTR_TRACKED, // hidden attribute, tracked by NotifierRegistry
ATTR_ON_CHANGE, // for table change tracking
ATTR_BROKER_STORE, // for broker-store backed tables
ATTR_BROKER_STORE_ALLOW_COMPLEX, // for broker-store backed tables
ATTR_BACKEND, // for broker-store backed tabled
ATTR_BROKER_STORE, // for Broker store backed tables
ATTR_BROKER_STORE_ALLOW_COMPLEX, // for Broker store backed tables
ATTR_BACKEND, // for Broker store backed tabled
ATTR_DEPRECATED,
NUM_ATTRS // this item should always be last
};

6
src/Val.cc
View file

@ -2164,7 +2164,7 @@ void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val,
return;
// we either get passed the raw index_val - or a ListVal with exactly one element.
// Since broker does not support ListVals, we have to unoll this in the second case.
// Since Broker does not support ListVals, we have to unoll this in the second case.
const Val* index_val;
if ( index->GetType()->Tag() == zeek::TYPE_LIST )
{
@ -2221,7 +2221,7 @@ void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val,
{
if ( ! new_entry_val )
{
zeek::emit_builtin_error("did not receive new value for broker-store send operation");
zeek::emit_builtin_error("did not receive new value for Broker datastore send operation");
return;
}
auto new_value = new_entry_val->GetVal().get();
@ -2239,7 +2239,7 @@ void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val,
handle->store.erase(std::move(*broker_index));
break;
case ELEMENT_EXPIRED:
// we do nothing here. The broker store does its own expiration - so the element
// we do nothing here. The Broker store does its own expiration - so the element
// should expire at about the same time.
break;
}

8
src/Val.h
View file

@ -785,7 +785,7 @@ public:
* @param new_val The value to assign at the index. For a set, this
* must be nullptr.
* @param broker_forward Controls if the value will be forwarded to attached
* broker stores.
* Broker stores.
* @return True if the assignment type-checked.
*/
bool Assign(ValPtr index, ValPtr new_val, bool broker_forward = true);
@ -799,7 +799,7 @@ public:
* @param new_val The value to assign at the index. For a set, this
* must be nullptr.
* @param broker_forward Controls if the value will be forwarded to attached
* broker stores.
* Broker stores.
* @return True if the assignment type-checked.
*/
bool Assign(ValPtr index, std::unique_ptr<HashKey> k,
@ -926,7 +926,7 @@ public:
* Remove an element from the table and return it.
* @param index The index to remove.
* @param broker_forward Controls if the remove operation will be forwarded to attached
* broker stores.
* Broker stores.
* @return The value associated with the index if it exists, else nullptr.
* For a sets that don't really contain associated values, a placeholder
* value is returned to differentiate it from non-existent index (nullptr),
@ -1024,7 +1024,7 @@ public:
static void DoneParsing();
/**
* Sets the name of the broker store that is backing this table.
* Sets the name of the Broker store that is backing this table.
* @param store store that is backing this table.
*/
void SetBrokerStore(const std::string& store) { broker_store = store; }

7
src/broker/Manager.cc
View file

@ -235,7 +235,7 @@ void Manager::InitializeBrokerStoreForwarding()
id->GetVal()->AsTableVal()->SetBrokerStore(storename);
AddForwardedStore(storename, {zeek::NewRef{}, id->GetVal()->AsTableVal()});
// we only create masters here. For clones, we do all the work of setting up
// We only create masters here. For clones, we do all the work of setting up
// the forwarding - but we do not try to initialize the clone. We can only initialize
// the clone, once a node has a connection to a master. This is currently done in scriptland
// in scripts/base/frameworks/cluster/broker-stores.zeek. Once the ALM transport is ready
@ -921,7 +921,6 @@ void Manager::Process()
{
// Ensure that time gets update before processing broker messages, or events
// based on them might get scheduled wrong.
// Fixme: unclear if final solution - see https://github.com/zeek/broker/issues/135
if ( use_real_time )
net_update_time(current_time());
@ -1700,14 +1699,14 @@ void Manager::BrokerStoreToZeekTable(const std::string& name, const StoreHandleV
auto value = handle->store.get(key);
if ( ! value )
{
reporter->Error("Failed to load value for key %s while importing broker store %s to table", to_string(key).c_str(), name.c_str());
reporter->Error("Failed to load value for key %s while importing Broker store %s to table", to_string(key).c_str(), name.c_str());
continue;
}
auto zeek_value = data_to_val(*value, table->GetType()->Yield().get());
if ( ! zeek_value )
{
reporter->Error("Could not convert %s to table value while trying to import broker store %s. Aborting import.", to_string(value).c_str(), name.c_str());
reporter->Error("Could not convert %s to table value while trying to import Broker store %s. Aborting import.", to_string(value).c_str(), name.c_str());
return;
}

18
src/broker/Manager.h
View file

@ -304,12 +304,12 @@ public:
StoreHandleVal* LookupStore(const std::string& name);
/**
* Register a zeek table that is associated with a broker store that is backing it. This
* causes all changes that happen to the brokerstore in the future to be applied to the zeek
* Register a Zeek table that is associated with a Broker store that is backing it. This
* causes all changes that happen to the Broker store in the future to be applied to theZzeek
* table.
* A single broker store can only be forwarded to a single table.
* @param name name of the broker store
* @param table pointer to the table/set that is being backed
* A single Broker store can only be forwarded to a single table.
* @param name name of the Broker store.
* @param table pointer to the table/set that is being backed.
* @return true on success, false if the named store is already being forwarded.
*/
bool AddForwardedStore(const std::string& name, zeek::IntrusivePtr<zeek::TableVal> table);
@ -359,7 +359,7 @@ public:
private:
void DispatchMessage(const broker::topic& topic, broker::data msg);
// Process events used for broker store backed zeek tables
// Process events used for Broker store backed zeek tables
void ProcessStoreEvent(broker::data msg);
void ProcessEvent(const broker::topic& topic, broker::zeek::Event ev);
bool ProcessLogCreate(broker::zeek::LogCreate lc);
@ -369,11 +369,11 @@ private:
void ProcessError(broker::error err);
void ProcessStoreResponse(StoreHandleVal*, broker::store::response response);
void FlushPendingQueries();
// Initializes the masters for broker backed zeek tables when using the &backend attribute
// Initializes the masters for Broker backed Zeek tables when using the &backend attribute
void InitializeBrokerStoreForwarding();
// Check if a broker store is associated to a table on the Zeek side.
// Check if a Broker store is associated to a table on the Zeek side.
void CheckForwarding(const std::string& name);
// Send the content of a broker store to the backing table. This is typically used
// Send the content of a Broker store to the backing table. This is typically used
// when a master/clone is created.
void BrokerStoreToZeekTable(const std::string& name, const StoreHandleVal* handle);

2
src/broker/Store.h
View file

@ -49,7 +49,7 @@ inline zeek::RecordValPtr query_result(zeek::RecordValPtr data)
/**
* Convert an expiry from a double (used to Zeek) to the format required by Broker
* @param e: expire interval as double; 0 if no expiry
* @return expire interval in broker format
* @return expire interval in Broker format
*/
static broker::optional<broker::timespan> convert_expiry(double e)
{

2
testing/btest/broker/store/brokerstore-attr-expire.zeek
View file

@ -1,5 +1,5 @@
# So - this test currently is not really that great. The goal was to test expiration after
# syncing values with broker. However, it turns out that the delays introduced by broker seem
# syncing values with Broker. However, it turns out that the delays introduced by Broker seem
# a bit random - and too high to really test this without the test taking forever.
#
# so - instead we just check that expiries do indeed happen - however the ordering is not as