From f45a3e8878fdfa1e1a95e34c64c065ad39446db3 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 29 Jun 2016 17:55:49 -0500 Subject: [PATCH] Add a new node type for logging Changed the cluster framework scripts by adding a new Bro node type for doing logging (this is intended to reduce the load on the manager). If a user chooses not to specify a logger node in the cluster configuration, then the manager will write logs locally as usual. --- scripts/base/frameworks/cluster/__load__.bro | 8 ++++ scripts/base/frameworks/cluster/main.bro | 6 ++- .../base/frameworks/cluster/nodes/logger.bro | 29 +++++++++++++++ .../base/frameworks/cluster/nodes/manager.bro | 13 ++++--- .../base/frameworks/cluster/nodes/worker.bro | 2 +- .../frameworks/cluster/setup-connections.bro | 37 +++++++++++++++++-- 6 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 scripts/base/frameworks/cluster/nodes/logger.bro diff --git a/scripts/base/frameworks/cluster/__load__.bro b/scripts/base/frameworks/cluster/__load__.bro index 0f9003514d..56d51cbc3d 100644 --- a/scripts/base/frameworks/cluster/__load__.bro +++ b/scripts/base/frameworks/cluster/__load__.bro @@ -26,8 +26,16 @@ redef peer_description = Cluster::node; ## Set the port that this node is supposed to listen on. redef Communication::listen_port = Cluster::nodes[Cluster::node]$p; +@if ( Cluster::local_node_type() == Cluster::LOGGER ) +@load ./nodes/logger +@endif + @if ( Cluster::local_node_type() == Cluster::MANAGER ) @load ./nodes/manager +# If no logger is defined, then the manager writes logs. +@if ( "logger" !in Cluster::nodes ) +@load ./nodes/logger +@endif @endif @if ( Cluster::local_node_type() == Cluster::PROXY ) diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 55fc084641..cbb69df3d8 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -31,7 +31,9 @@ export { ## A node type which is allowed to view/manipulate the configuration ## of other nodes in the cluster. CONTROL, - ## A node type responsible for log and policy management. + ## A node type responsible for log management. + LOGGER, + ## A node type responsible for policy management. MANAGER, ## A node type for relaying worker node communication and synchronizing ## worker node state. @@ -86,6 +88,8 @@ export { p: port; ## Identifier for the interface a worker is sniffing. interface: string &optional; + ## Name of the logger node this node uses. For manager, proxies and workers. + logger: string &optional; ## Name of the manager node this node uses. For workers and proxies. manager: string &optional; ## Name of the proxy node this node uses. For workers and managers. diff --git a/scripts/base/frameworks/cluster/nodes/logger.bro b/scripts/base/frameworks/cluster/nodes/logger.bro new file mode 100644 index 0000000000..39dcb751df --- /dev/null +++ b/scripts/base/frameworks/cluster/nodes/logger.bro @@ -0,0 +1,29 @@ +##! This is the core Bro script to support the notion of a cluster logger. +##! +##! The logger is passive (other Bro instances connect to us), and once +##! connected the logger receives logs from other Bro instances. +##! This script will be automatically loaded if necessary based on the +##! type of node being started. + +##! This is where the cluster logger sets it's specific settings for other +##! frameworks and in the core. + +@prefixes += cluster-logger + +## Turn on local logging. +redef Log::enable_local_logging = T; + +## Turn off remote logging since this is the logger and should only log here. +redef Log::enable_remote_logging = F; + +## Log rotation interval. +redef Log::default_rotation_interval = 1 hrs; + +## Alarm summary mail interval. +redef Log::default_mail_alarms_interval = 24 hrs; + +## Use the cluster's archive logging script. +redef Log::default_rotation_postprocessor_cmd = "archive-log"; + +## We're processing essentially *only* remote events. +redef max_remote_events_processed = 10000; diff --git a/scripts/base/frameworks/cluster/nodes/manager.bro b/scripts/base/frameworks/cluster/nodes/manager.bro index 5149dbac7e..28d45c0e55 100644 --- a/scripts/base/frameworks/cluster/nodes/manager.bro +++ b/scripts/base/frameworks/cluster/nodes/manager.bro @@ -10,17 +10,20 @@ @prefixes += cluster-manager -## Turn off remote logging since this is the manager and should only log here. -redef Log::enable_remote_logging = F; +## Don't do any local logging since the logger handles writing logs. +redef Log::enable_local_logging = F; + +## Turn on remote logging since the logger handles writing logs. +redef Log::enable_remote_logging = T; ## Log rotation interval. -redef Log::default_rotation_interval = 1 hrs; +redef Log::default_rotation_interval = 24 hrs; ## Alarm summary mail interval. redef Log::default_mail_alarms_interval = 24 hrs; -## Use the cluster's archive logging script. -redef Log::default_rotation_postprocessor_cmd = "archive-log"; +## Use the cluster's delete-log script. +redef Log::default_rotation_postprocessor_cmd = "delete-log"; ## We're processing essentially *only* remote events. redef max_remote_events_processed = 10000; diff --git a/scripts/base/frameworks/cluster/nodes/worker.bro b/scripts/base/frameworks/cluster/nodes/worker.bro index f876f8592e..826df848f7 100644 --- a/scripts/base/frameworks/cluster/nodes/worker.bro +++ b/scripts/base/frameworks/cluster/nodes/worker.bro @@ -1,6 +1,6 @@ ##! Redefines some options common to all worker nodes within a Bro cluster. ##! In particular, worker nodes do not produce logs locally, instead they -##! send them off to a manager node for processing. +##! send them off to a logger node for processing. @prefixes += cluster-worker diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index 95aff64a6c..db7f2c3160 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -23,17 +23,36 @@ event bro_init() &priority=9 $connect=F, $class="control", $events=control_events]; - if ( me$node_type == MANAGER ) + if ( me$node_type == LOGGER ) { + if ( (n$node_type == MANAGER || n$node_type == PROXY || + n$node_type == WORKER) && n$logger == node ) + Communication::nodes[i] = + [$host=n$ip, $zone_id=n$zone_id, $connect=F, + $class=i, $request_logs=T]; + } + else if ( me$node_type == MANAGER ) + { + # If no logger is defined, then the manager writes logs. + local managerlogs = "logger" !in Cluster::nodes; + + if ( n$node_type == LOGGER && me$logger == i ) + Communication::nodes["logger"] = + [$host=n$ip, $zone_id=n$zone_id, $p=n$p, + $connect=T, $retry=retry_interval, + $class=node]; + if ( n$node_type == WORKER && n$manager == node ) Communication::nodes[i] = [$host=n$ip, $zone_id=n$zone_id, $connect=F, - $class=i, $events=worker2manager_events, $request_logs=T]; + $class=i, $events=worker2manager_events, + $request_logs=managerlogs]; if ( n$node_type == PROXY && n$manager == node ) Communication::nodes[i] = [$host=n$ip, $zone_id=n$zone_id, $connect=F, - $class=i, $events=proxy2manager_events, $request_logs=T]; + $class=i, $events=proxy2manager_events, + $request_logs=managerlogs]; if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i ) Communication::nodes["time-machine"] = [$host=nodes[i]$ip, @@ -45,6 +64,12 @@ event bro_init() &priority=9 else if ( me$node_type == PROXY ) { + if ( n$node_type == LOGGER && me$logger == i ) + Communication::nodes["logger"] = + [$host=n$ip, $zone_id=n$zone_id, $p=n$p, + $connect=T, $retry=retry_interval, + $class=node]; + if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = [$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i, @@ -76,6 +101,12 @@ event bro_init() &priority=9 } else if ( me$node_type == WORKER ) { + if ( n$node_type == LOGGER && me$logger == i ) + Communication::nodes["logger"] = + [$host=n$ip, $zone_id=n$zone_id, $p=n$p, + $connect=T, $retry=retry_interval, + $class=node]; + if ( n$node_type == MANAGER && me$manager == i ) Communication::nodes["manager"] = [$host=nodes[i]$ip, $zone_id=nodes[i]$zone_id,