Merge remote-tracking branch 'origin/topic/dnthayer/ticket1627'

* origin/topic/dnthayer/ticket1627:
  Add a test for starting a cluster with a logger node
  Update broctl submodule
  Update broctl submodule to branch topic/dnthayer/ticket1627
  Change how logger node is detected in cluster framework
  Update test baselines for the new logger node type
  Update docs for the new logger node type
  Add a new node type for logging

scripts/base/frameworks/cluster/__load__.bro

@@ -28,6 +28,14 @@ redef Communication::listen_port = Cluster::nodes[Cluster::node]$p;
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 @load ./nodes/manager
+# If no logger is defined, then the manager receives logs.
+@if ( Cluster::manager_is_logger )
+@load ./nodes/logger
+@endif
+@endif
+
+@if ( Cluster::local_node_type() == Cluster::LOGGER )
+@load ./nodes/logger
 @endif
 
 @if ( Cluster::local_node_type() == Cluster::PROXY )

scripts/base/frameworks/cluster/main.bro

@@ -31,7 +31,9 @@ export {
 		## A node type which is allowed to view/manipulate the configuration
 		## of other nodes in the cluster.
 		CONTROL,
-		## A node type responsible for log and policy management.
+		## A node type responsible for log management.
+		LOGGER,
+		## A node type responsible for policy management.
 		MANAGER,
 		## A node type for relaying worker node communication and synchronizing
 		## worker node state.
@@ -50,12 +52,21 @@ export {
 	## Events raised by a manager and handled by proxies.
 	const manager2proxy_events = /EMPTY/ &redef;
 
+	## Events raised by a manager and handled by loggers.
+	const manager2logger_events = /EMPTY/ &redef;
+
+	## Events raised by proxies and handled by loggers.
+	const proxy2logger_events = /EMPTY/ &redef;
+
 	## Events raised by proxies and handled by a manager.
 	const proxy2manager_events = /EMPTY/ &redef;
 
 	## Events raised by proxies and handled by workers.
 	const proxy2worker_events = /EMPTY/ &redef;
 
+	## Events raised by workers and handled by loggers.
+	const worker2logger_events = /EMPTY/ &redef;
+
 	## Events raised by workers and handled by a manager.
 	const worker2manager_events = /(TimeMachine::command|Drop::.*)/ &redef;
 
@@ -86,6 +97,8 @@ export {
 		p:            port;
 		## Identifier for the interface a worker is sniffing.
 		interface:    string      &optional;
+		## Name of the logger node this node uses.  For manager, proxies and workers.
+		logger:       string      &optional;
 		## Name of the manager node this node uses.  For workers and proxies.
 		manager:      string      &optional;
 		## Name of the proxy node this node uses.  For workers and managers.
@@ -123,6 +136,12 @@ export {
 	## Note that BroControl handles all of this automatically.
 	const nodes: table[string] of Node = {} &redef;
 
+	## Indicates whether or not the manager will act as the logger and receive
+	## logs.  This value should be set in the cluster-layout.bro script (the
+	## value should be true only if no logger is specified in Cluster::nodes).
+	## Note that BroControl handles this automatically.
+	const manager_is_logger = T &redef;
+
 	## This is usually supplied on the command line for each instance
 	## of the cluster that is started up.
 	const node = getenv("CLUSTER_NODE") &redef;

scripts/base/frameworks/cluster/nodes/logger.bro

@@ -0,0 +1,29 @@
+##! This is the core Bro script to support the notion of a cluster logger.
+##!
+##! The logger is passive (other Bro instances connect to us), and once
+##! connected the logger receives logs from other Bro instances.
+##! This script will be automatically loaded if necessary based on the
+##! type of node being started.
+
+##! This is where the cluster logger sets it's specific settings for other
+##! frameworks and in the core.
+
+@prefixes += cluster-logger
+
+## Turn on local logging.
+redef Log::enable_local_logging = T;
+
+## Turn off remote logging since this is the logger and should only log here.
+redef Log::enable_remote_logging = F;
+
+## Log rotation interval.
+redef Log::default_rotation_interval = 1 hrs;
+
+## Alarm summary mail interval.
+redef Log::default_mail_alarms_interval = 24 hrs;
+
+## Use the cluster's archive logging script.
+redef Log::default_rotation_postprocessor_cmd = "archive-log";
+
+## We're processing essentially *only* remote events.
+redef max_remote_events_processed = 10000;

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -10,17 +10,17 @@
 
 @prefixes += cluster-manager
 
-## Turn off remote logging since this is the manager and should only log here.
-redef Log::enable_remote_logging = F;
+## Don't do any local logging since the logger handles writing logs.
+redef Log::enable_local_logging = F;
+
+## Turn on remote logging since the logger handles writing logs.
+redef Log::enable_remote_logging = T;
 
 ## Log rotation interval.
-redef Log::default_rotation_interval = 1 hrs;
+redef Log::default_rotation_interval = 24 hrs;
 
-## Alarm summary mail interval.
-redef Log::default_mail_alarms_interval = 24 hrs;
-
-## Use the cluster's archive logging script.
-redef Log::default_rotation_postprocessor_cmd = "archive-log";
+## Use the cluster's delete-log script.
+redef Log::default_rotation_postprocessor_cmd = "delete-log";
 
 ## We're processing essentially *only* remote events.
 redef max_remote_events_processed = 10000;

scripts/base/frameworks/cluster/nodes/worker.bro

@@ -1,6 +1,6 @@
 ##! Redefines some options common to all worker nodes within a Bro cluster.
 ##! In particular, worker nodes do not produce logs locally, instead they
-##! send them off to a manager node for processing.
+##! send them off to a logger node for processing.
 
 @prefixes += cluster-worker
 

scripts/base/frameworks/cluster/setup-connections.bro

@@ -23,17 +23,40 @@ event bro_init() &priority=9
 			                                   $connect=F, $class="control",
 			                                   $events=control_events];
 
-		if ( me$node_type == MANAGER )
+		if ( me$node_type == LOGGER )
 			{
+			if ( n$node_type == MANAGER && n$logger == node )
+				Communication::nodes[i] =
+				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
+				     $class=i, $events=manager2logger_events, $request_logs=T];
+			if ( n$node_type == PROXY && n$logger == node )
+				Communication::nodes[i] =
+				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
+				     $class=i, $events=proxy2logger_events, $request_logs=T];
+			if ( n$node_type == WORKER && n$logger == node )
+				Communication::nodes[i] =
+				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
+				     $class=i, $events=worker2logger_events, $request_logs=T];
+			}
+		else if ( me$node_type == MANAGER )
+			{
+			if ( n$node_type == LOGGER && me$logger == i )
+				Communication::nodes["logger"] =
+				    [$host=n$ip, $zone_id=n$zone_id, $p=n$p,
+				     $connect=T, $retry=retry_interval,
+				     $class=node];
+
 			if ( n$node_type == WORKER && n$manager == node )
 				Communication::nodes[i] =
 				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
-				     $class=i, $events=worker2manager_events, $request_logs=T];
+				     $class=i, $events=worker2manager_events,
+				     $request_logs=Cluster::manager_is_logger];
 
 			if ( n$node_type == PROXY && n$manager == node )
 				Communication::nodes[i] =
 				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
-				     $class=i, $events=proxy2manager_events, $request_logs=T];
+				     $class=i, $events=proxy2manager_events,
+				     $request_logs=Cluster::manager_is_logger];
 
 			if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i )
 				Communication::nodes["time-machine"] = [$host=nodes[i]$ip,
@@ -45,6 +68,12 @@ event bro_init() &priority=9
 
 		else if ( me$node_type == PROXY )
 			{
+			if ( n$node_type == LOGGER && me$logger == i )
+				Communication::nodes["logger"] =
+				    [$host=n$ip, $zone_id=n$zone_id, $p=n$p,
+				     $connect=T, $retry=retry_interval,
+				     $class=node];
+
 			if ( n$node_type == WORKER && n$proxy == node )
 				Communication::nodes[i] =
 					[$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i,
@@ -76,6 +105,12 @@ event bro_init() &priority=9
 			}
 		else if ( me$node_type == WORKER )
 			{
+			if ( n$node_type == LOGGER && me$logger == i )
+				Communication::nodes["logger"] =
+				    [$host=n$ip, $zone_id=n$zone_id, $p=n$p,
+				     $connect=T, $retry=retry_interval,
+				     $class=node];
+
 			if ( n$node_type == MANAGER && me$manager == i )
 				Communication::nodes["manager"] = [$host=nodes[i]$ip,
 				                                   $zone_id=nodes[i]$zone_id,