Merge remote-tracking branch 'origin/topic/johanna/table-changes'

* origin/topic/johanna/table-changes: (26 commits)
  TableSync: try to make test more robust & add debug output
  Increase timeouts to see if FreeBSD will be happy with this.
  Try to make FreeBSD test happy with larger timeout.
  TableSync: refactor common functionality into function
  TableSync: don't raise &on_change, smaller fixes
  TableSync: rename auto_store -> table_store
  SyncTables: address feedback part 1 - naming (broker and zeek)
  BrokerStore <-> Zeek Tables: cleanup and bug workaround
  Zeek Table<->Brokerstore: cleanup, documentation, small fixes
  BrokerStore<->Zeek table: adopt to recent Zeek API changes
  BrokerStore<->Zeek Tables Fix a few small test failures.
  BrokerStore<->Zeek tables: allow setting storage location & tests
  BrokerStore<->Zeek tables: &backend works for in-memory stores.
  BrokerStore<->Zeek table - introdude &backend attribute
  BrokerStore<->Zeek tables: test for clones synchronizing to a master
  BrokerStore<->Zeek tables: load persistent tables on startup.
  Brokerstore<->Tables: attribute conflicts
  Zeek/Brokerstore updates: expiration
  Zeek/Brokerstore updates: add test that includes updates from clones
  Zeek/Brokerstore updates: first working end-to-end test
  ...

scripts/base/frameworks/broker/store.zeek

@@ -25,6 +25,16 @@ export {
 	## A negative/zero value indicates to never buffer commands.
 	const default_clone_mutation_buffer_interval = 2min &redef;
 
+	## If set to true, the current node is the master node for Broker stores
+	## backing Zeek tables. By default this value will be automatically set to
+	## true in standalone mode, and on the manager node of a cluster. This value
+	## should not typically be changed manually.
+	const table_store_master = T &redef;
+
+	## The directory used for storing persistent database files when using Broker
+        ## store backed Zeek tables.
+	const table_store_db_directory = "." &redef;
+
 	## Whether a data store query could be completed or not.
 	type QueryStatus: enum {
 		SUCCESS,
@@ -136,7 +146,7 @@ export {
 	global store_name: function(h: opaque of Broker::Store): string;
 
 	## Check if a key exists in a data store.
-	## 
+	##
 	## h: the handle of the store to query.
 	##
 	## k: the key to lookup.
@@ -178,7 +188,7 @@ export {
 	## k: the key of the container value to lookup.
 	##
 	## i: the index to retrieve from the container value.
-	## 
+	##
 	## Returns: For tables and vectors, the value at the given index, or
 	##          failure if the index doesn't exist. For sets, a boolean
 	##          indicating whether the index exists. Returns failure if the key
@@ -217,7 +227,7 @@ export {
 	## k: the key whose associated value is to be modified. The key must
 	##    already exist.
 	##
-	## a: the amount to increment the value by. 
+	## a: the amount to increment the value by.
 	##
 	## e: the new expiration interval of the modified key. If null, the
 	##    current expiration time isn't changed.
@@ -235,7 +245,7 @@ export {
 	## k: the key whose associated value is to be modified. The key must
 	##    already exist.
 	##
-	## amount: the amount to decrement the value by. 
+	## amount: the amount to decrement the value by.
 	##
 	## e: the new expiration interval of the modified key. If null, the current
 	##    expiration time isn't changed.
@@ -258,7 +268,7 @@ export {
 	##    current expiration time isn't changed.
 	##
 	## Returns: false if the store handle was not valid.
-	global append: function(h: opaque of Broker::Store, k: any, s: string, 
+	global append: function(h: opaque of Broker::Store, k: any, s: string,
 	                        e: interval &default=0sec) : bool;
 
 	## Inserts an element into an existing set.
@@ -275,7 +285,7 @@ export {
 	##
 	## Returns: false if the store handle was not valid.
 	global insert_into_set: function(h: opaque of Broker::Store,
-	                                 k: any, i: any, 
+	                                 k: any, i: any,
 	                                 e: interval &default=0sec) : bool;
 
 	## Inserts an element into an existing table.
@@ -286,7 +296,7 @@ export {
 	##    already exist.
 	##
 	## i: the index to insert into the table
-	## 
+	##
 	## v: the value to associate with the index.
 	##
 	## e: the new expiration interval of the modified key. If null, the
@@ -311,7 +321,7 @@ export {
 	##
 	## Returns: false if the store handle was not valid.
 	global remove_from: function(h: opaque of Broker::Store,
-	                             k: any, i: any, 
+	                             k: any, i: any,
 	                             e: interval &default=0sec) : bool;
 
 	## Appends an element to an existing vector.
@@ -328,7 +338,7 @@ export {
 	##
 	## Returns: false if the store handle was not valid.
 	global push: function(h: opaque of Broker::Store,
-	                      k: any, v: any, 
+	                      k: any, v: any,
 	                      e: interval &default=0sec) : bool;
 
 	## Removes the last element of an existing vector.
@@ -383,7 +393,7 @@ export {
 	## d: the communication data.
 	##
 	## Returns: The data type associated with the communication data.
-	##          Note that broker represents records in the same way as
+	##          Note that Broker represents records in the same way as
 	##          vectors, so there is no "record" type.
 	global data_type: function(d: Broker::Data): Broker::DataType;
 

scripts/base/frameworks/cluster/__load__.zeek

@@ -49,5 +49,7 @@ redef Broker::log_topic = Cluster::rr_log_topic;
 @load ./nodes/worker
 @endif
 
+@load ./broker-stores.zeek
+
 @endif
 @endif

scripts/base/frameworks/cluster/broker-stores.zeek

@@ -0,0 +1,61 @@
+##! This script deals with the cluster parts of Broker backed Zeek tables.
+##! It makes sure that the master store is set correctly and that clones
+##! are automatically created on the non-manager nodes.
+
+# Note - this script should become unnecessary in the future, when we just can
+# speculatively attach clones. This should be possible once the new ALM Broker
+# transport becomes available.
+
+@load ./main
+
+module Broker;
+
+export {
+	## Event that is used by the manager to announce the master stores for Broker backed
+	## tables.
+	global announce_masters: event(masters: set[string]);
+}
+
+# If we are not the manager, disable automatically generating masters. We will attach
+# clones instead.
+@if ( Cluster::is_enabled() && Cluster::local_node_type() != Cluster::MANAGER )
+redef Broker::table_store_master = F;
+@endif
+
+@if ( Broker::table_store_master )
+
+global broker_backed_ids: set[string];
+
+event zeek_init()
+	{
+	local globals = global_ids();
+	for ( id in globals )
+		{
+		if ( globals[id]$broker_backend )
+			add broker_backed_ids[id];
+		}
+	}
+
+# Send the auto masters we created to the newly connected node
+event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) &priority=1
+	{
+	if ( ! Cluster::is_enabled() )
+		return;
+
+	local e = Broker::make_event(Broker::announce_masters, broker_backed_ids);
+	Broker::publish(Cluster::nodeid_topic(endpoint$id), e);
+	}
+
+@else
+
+event Broker::announce_masters(masters: set[string])
+	{
+	for ( i in masters )
+		{
+		# this magic name for the store is created in broker/Manager.cc for the manager.
+		local name = "___sync_store_" + i;
+		Broker::create_clone(name);
+		}
+	}
+
+@endif