Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

SyncTables: address feedback part 1 - naming (broker and zeek)

Browse source 
This commit fixes capitalization issues.
This commit is contained in:
Johanna Amann 2020-07-17 10:56:28 -07:00
parent 1888d6acae
commit 6d2aa84952
11 changed files with 35 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

2
NEWS
View file

@ -57,7 +57,7 @@ New Functionality
- Broker Store table synchronization (Experimental). - Broker Store table synchronization (Experimental).
Zeek now supports synchronizing tables/sets across clusters using a backing broker Zeek now supports synchronizing tables/sets across clusters using a backing Broker
store. The same feature also allows persistent storage of data in tables/sets store. The same feature also allows persistent storage of data in tables/sets
over Zeek restarts. This feature is implemented using the new ```&backend``` attribute. over Zeek restarts. This feature is implemented using the new ```&backend``` attribute.

10
scripts/base/frameworks/broker/store.zeek
View file

@ -25,14 +25,14 @@ export {
## A negative/zero value indicates to never buffer commands. ## A negative/zero value indicates to never buffer commands.
const default_clone_mutation_buffer_interval = 2min &redef; const default_clone_mutation_buffer_interval = 2min &redef;
## If set to true, the current node is the master node for broker stores ## If set to true, the current node is the master node for Broker stores
## backing zeek tables. By default this value will be automatically set to ## backing Zeek tables. By default this value will be automatically set to
## true in standalone mode, or on the manager node of a cluster. This value ## true in standalone mode, or on the manager node of a cluster. This value
## should not typically be changed manually. ## should not typically be changed manually.
const auto_store_master = T &redef; const auto_store_master = T &redef;
## The directory used for storing persistent database files when using brokerstore ## The directory used for storing persistent database files when using Broker store
## backed zeek tables. ## backed Zeek tables.
const auto_store_db_directory = "." &redef; const auto_store_db_directory = "." &redef;
## Whether a data store query could be completed or not. ## Whether a data store query could be completed or not.
@ -393,7 +393,7 @@ export {
## d: the communication data. ## d: the communication data.
## ##
## Returns: The data type associated with the communication data. ## Returns: The data type associated with the communication data.
## Note that broker represents records in the same way as ## Note that Broker represents records in the same way as
## vectors, so there is no "record" type. ## vectors, so there is no "record" type.
global data_type: function(d: Broker::Data): Broker::DataType; global data_type: function(d: Broker::Data): Broker::DataType;

8
scripts/base/frameworks/cluster/broker-stores.zeek
View file

@ -1,9 +1,9 @@
##! This script deals with the cluster parts of broker backed zeek tables. ##! This script deals with the cluster parts of Broker backed Zeek tables.
##! It makes sure that the master store is set correctly and that clones ##! It makes sure that the master store is set correctly and that clones
##! are automatically created on the non-manager nodes. ##! are automatically created on the non-manager nodes.
# Note - this script should become unnecessary in the future, when we just can # Note - this script should become unnecessary in the future, when we just can
# speculatively attach clones. This should be possible once the new ALM broker # speculatively attach clones. This should be possible once the new ALM Broker
# transport becomes available. # transport becomes available.
@load ./main @load ./main
@ -11,8 +11,8 @@
module Broker; module Broker;
export { export {
## Event that is used by the manager to announce the master stores for zeek backed ## Event that is used by the manager to announce the master stores for Broker backed
## tables that is uses. ## tables.
global announce_masters: event(masters: set[string]); global announce_masters: event(masters: set[string]);
} }

2
scripts/base/init-bare.zeek
View file

@ -747,7 +747,7 @@ type script_id: record {
enum_constant: bool; ##< True if the identifier is an enum value. enum_constant: bool; ##< True if the identifier is an enum value.
option_value: bool; ##< True if the identifier is an option. option_value: bool; ##< True if the identifier is an option.
redefinable: bool; ##< True if the identifier is declared with the :zeek:attr:`&redef` attribute. redefinable: bool; ##< True if the identifier is declared with the :zeek:attr:`&redef` attribute.
broker_backend: bool; ##< True if the identifier has a broker backend defined using the :zeek:attr:`&backend` attribute. broker_backend: bool; ##< True if the identifier has a Broker backend defined using the :zeek:attr:`&backend` attribute.
value: any &optional; ##< The current value of the identifier. value: any &optional; ##< The current value of the identifier.
}; };

6
src/Attr.h
View file

@ -42,9 +42,9 @@ enum AttrTag {
ATTR_TYPE_COLUMN, // for input framework ATTR_TYPE_COLUMN, // for input framework
ATTR_TRACKED, // hidden attribute, tracked by NotifierRegistry ATTR_TRACKED, // hidden attribute, tracked by NotifierRegistry
ATTR_ON_CHANGE, // for table change tracking ATTR_ON_CHANGE, // for table change tracking
ATTR_BROKER_STORE, // for broker-store backed tables ATTR_BROKER_STORE, // for Broker store backed tables
ATTR_BROKER_STORE_ALLOW_COMPLEX, // for broker-store backed tables ATTR_BROKER_STORE_ALLOW_COMPLEX, // for Broker store backed tables
ATTR_BACKEND, // for broker-store backed tabled ATTR_BACKEND, // for Broker store backed tabled
ATTR_DEPRECATED, ATTR_DEPRECATED,
NUM_ATTRS // this item should always be last NUM_ATTRS // this item should always be last
}; };

6
src/Val.cc
View file

@ -2164,7 +2164,7 @@ void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val,
return; return;
// we either get passed the raw index_val - or a ListVal with exactly one element. // we either get passed the raw index_val - or a ListVal with exactly one element.
// Since broker does not support ListVals, we have to unoll this in the second case. // Since Broker does not support ListVals, we have to unoll this in the second case.
const Val* index_val; const Val* index_val;
if ( index->GetType()->Tag() == zeek::TYPE_LIST ) if ( index->GetType()->Tag() == zeek::TYPE_LIST )
{ {
@ -2221,7 +2221,7 @@ void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val,
{ {
if ( ! new_entry_val ) if ( ! new_entry_val )
{ {
zeek::emit_builtin_error("did not receive new value for broker-store send operation"); zeek::emit_builtin_error("did not receive new value for Broker datastore send operation");
return; return;
} }
auto new_value = new_entry_val->GetVal().get(); auto new_value = new_entry_val->GetVal().get();
@ -2239,7 +2239,7 @@ void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val,
handle->store.erase(std::move(*broker_index)); handle->store.erase(std::move(*broker_index));
break; break;
case ELEMENT_EXPIRED: case ELEMENT_EXPIRED:
// we do nothing here. The broker store does its own expiration - so the element // we do nothing here. The Broker store does its own expiration - so the element
// should expire at about the same time. // should expire at about the same time.
break; break;
} }

8
src/Val.h
View file

@ -785,7 +785,7 @@ public:
* @param new_val The value to assign at the index. For a set, this * @param new_val The value to assign at the index. For a set, this
* must be nullptr. * must be nullptr.
* @param broker_forward Controls if the value will be forwarded to attached * @param broker_forward Controls if the value will be forwarded to attached
* broker stores. * Broker stores.
* @return True if the assignment type-checked. * @return True if the assignment type-checked.
*/ */
bool Assign(ValPtr index, ValPtr new_val, bool broker_forward = true); bool Assign(ValPtr index, ValPtr new_val, bool broker_forward = true);
@ -799,7 +799,7 @@ public:
* @param new_val The value to assign at the index. For a set, this * @param new_val The value to assign at the index. For a set, this
* must be nullptr. * must be nullptr.
* @param broker_forward Controls if the value will be forwarded to attached * @param broker_forward Controls if the value will be forwarded to attached
* broker stores. * Broker stores.
* @return True if the assignment type-checked. * @return True if the assignment type-checked.
*/ */
bool Assign(ValPtr index, std::unique_ptr<HashKey> k, bool Assign(ValPtr index, std::unique_ptr<HashKey> k,
@ -926,7 +926,7 @@ public:
* Remove an element from the table and return it. * Remove an element from the table and return it.
* @param index The index to remove. * @param index The index to remove.
* @param broker_forward Controls if the remove operation will be forwarded to attached * @param broker_forward Controls if the remove operation will be forwarded to attached
* broker stores. * Broker stores.
* @return The value associated with the index if it exists, else nullptr. * @return The value associated with the index if it exists, else nullptr.
* For a sets that don't really contain associated values, a placeholder * For a sets that don't really contain associated values, a placeholder
* value is returned to differentiate it from non-existent index (nullptr), * value is returned to differentiate it from non-existent index (nullptr),
@ -1024,7 +1024,7 @@ public:
static void DoneParsing(); static void DoneParsing();
/** /**
* Sets the name of the broker store that is backing this table. * Sets the name of the Broker store that is backing this table.
* @param store store that is backing this table. * @param store store that is backing this table.
*/ */
void SetBrokerStore(const std::string& store) { broker_store = store; } void SetBrokerStore(const std::string& store) { broker_store = store; }

7
src/broker/Manager.cc
View file

@ -235,7 +235,7 @@ void Manager::InitializeBrokerStoreForwarding()
id->GetVal()->AsTableVal()->SetBrokerStore(storename); id->GetVal()->AsTableVal()->SetBrokerStore(storename);
AddForwardedStore(storename, {zeek::NewRef{}, id->GetVal()->AsTableVal()}); AddForwardedStore(storename, {zeek::NewRef{}, id->GetVal()->AsTableVal()});
// we only create masters here. For clones, we do all the work of setting up // We only create masters here. For clones, we do all the work of setting up
// the forwarding - but we do not try to initialize the clone. We can only initialize // the forwarding - but we do not try to initialize the clone. We can only initialize
// the clone, once a node has a connection to a master. This is currently done in scriptland // the clone, once a node has a connection to a master. This is currently done in scriptland
// in scripts/base/frameworks/cluster/broker-stores.zeek. Once the ALM transport is ready // in scripts/base/frameworks/cluster/broker-stores.zeek. Once the ALM transport is ready
@ -921,7 +921,6 @@ void Manager::Process()
{ {
// Ensure that time gets update before processing broker messages, or events // Ensure that time gets update before processing broker messages, or events
// based on them might get scheduled wrong. // based on them might get scheduled wrong.
// Fixme: unclear if final solution - see https://github.com/zeek/broker/issues/135
if ( use_real_time ) if ( use_real_time )
net_update_time(current_time()); net_update_time(current_time());
@ -1700,14 +1699,14 @@ void Manager::BrokerStoreToZeekTable(const std::string& name, const StoreHandleV
auto value = handle->store.get(key); auto value = handle->store.get(key);
if ( ! value ) if ( ! value )
{ {
reporter->Error("Failed to load value for key %s while importing broker store %s to table", to_string(key).c_str(), name.c_str()); reporter->Error("Failed to load value for key %s while importing Broker store %s to table", to_string(key).c_str(), name.c_str());
continue; continue;
} }
auto zeek_value = data_to_val(*value, table->GetType()->Yield().get()); auto zeek_value = data_to_val(*value, table->GetType()->Yield().get());
if ( ! zeek_value ) if ( ! zeek_value )
{ {
reporter->Error("Could not convert %s to table value while trying to import broker store %s. Aborting import.", to_string(value).c_str(), name.c_str()); reporter->Error("Could not convert %s to table value while trying to import Broker store %s. Aborting import.", to_string(value).c_str(), name.c_str());
return; return;
} }

18
src/broker/Manager.h
View file

@ -304,12 +304,12 @@ public:
StoreHandleVal* LookupStore(const std::string& name); StoreHandleVal* LookupStore(const std::string& name);
/** /**
* Register a zeek table that is associated with a broker store that is backing it. This * Register a Zeek table that is associated with a Broker store that is backing it. This
* causes all changes that happen to the brokerstore in the future to be applied to the zeek * causes all changes that happen to the Broker store in the future to be applied to theZzeek
* table. * table.
* A single broker store can only be forwarded to a single table. * A single Broker store can only be forwarded to a single table.
* @param name name of the broker store * @param name name of the Broker store.
* @param table pointer to the table/set that is being backed * @param table pointer to the table/set that is being backed.
* @return true on success, false if the named store is already being forwarded. * @return true on success, false if the named store is already being forwarded.
*/ */
bool AddForwardedStore(const std::string& name, zeek::IntrusivePtr<zeek::TableVal> table); bool AddForwardedStore(const std::string& name, zeek::IntrusivePtr<zeek::TableVal> table);
@ -359,7 +359,7 @@ public:
private: private:
void DispatchMessage(const broker::topic& topic, broker::data msg); void DispatchMessage(const broker::topic& topic, broker::data msg);
// Process events used for broker store backed zeek tables // Process events used for Broker store backed zeek tables
void ProcessStoreEvent(broker::data msg); void ProcessStoreEvent(broker::data msg);
void ProcessEvent(const broker::topic& topic, broker::zeek::Event ev); void ProcessEvent(const broker::topic& topic, broker::zeek::Event ev);
bool ProcessLogCreate(broker::zeek::LogCreate lc); bool ProcessLogCreate(broker::zeek::LogCreate lc);
@ -369,11 +369,11 @@ private:
void ProcessError(broker::error err); void ProcessError(broker::error err);
void ProcessStoreResponse(StoreHandleVal*, broker::store::response response); void ProcessStoreResponse(StoreHandleVal*, broker::store::response response);
void FlushPendingQueries(); void FlushPendingQueries();
// Initializes the masters for broker backed zeek tables when using the &backend attribute // Initializes the masters for Broker backed Zeek tables when using the &backend attribute
void InitializeBrokerStoreForwarding(); void InitializeBrokerStoreForwarding();
// Check if a broker store is associated to a table on the Zeek side. // Check if a Broker store is associated to a table on the Zeek side.
void CheckForwarding(const std::string& name); void CheckForwarding(const std::string& name);
// Send the content of a broker store to the backing table. This is typically used // Send the content of a Broker store to the backing table. This is typically used
// when a master/clone is created. // when a master/clone is created.
void BrokerStoreToZeekTable(const std::string& name, const StoreHandleVal* handle); void BrokerStoreToZeekTable(const std::string& name, const StoreHandleVal* handle);

2
src/broker/Store.h
View file

@ -49,7 +49,7 @@ inline zeek::RecordValPtr query_result(zeek::RecordValPtr data)
/** /**
* Convert an expiry from a double (used to Zeek) to the format required by Broker * Convert an expiry from a double (used to Zeek) to the format required by Broker
* @param e: expire interval as double; 0 if no expiry * @param e: expire interval as double; 0 if no expiry
* @return expire interval in broker format * @return expire interval in Broker format
*/ */
static broker::optional<broker::timespan> convert_expiry(double e) static broker::optional<broker::timespan> convert_expiry(double e)
{ {

2
testing/btest/broker/store/brokerstore-attr-expire.zeek
View file

@ -1,5 +1,5 @@
# So - this test currently is not really that great. The goal was to test expiration after # So - this test currently is not really that great. The goal was to test expiration after
# syncing values with broker. However, it turns out that the delays introduced by broker seem # syncing values with Broker. However, it turns out that the delays introduced by Broker seem
# a bit random - and too high to really test this without the test taking forever. # a bit random - and too high to really test this without the test taking forever.
# #
# so - instead we just check that expiries do indeed happen - however the ordering is not as # so - instead we just check that expiries do indeed happen - however the ordering is not as