diff --git a/CHANGES b/CHANGES index 2b9bcb8c7f..0727bb3a4e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,12 @@ +2.4-883 | 2016-08-05 12:57:26 -0400 + + * Add a new node type for logging with the cluster framework scripts by + adding a new Bro node type for doing logging (this is intended to + reduce the load on the manager). If a user chooses not to specify a + logger node in the cluster configuration, then the manager will + write logs locally as usual. (Daniel Thayer) + 2.4-874 | 2016-08-05 12:43:06 -0400 * SMB analyzer (Seth Hall, Vlad Grigorescu and many others) diff --git a/VERSION b/VERSION index a163ce8328..e6121982df 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-874 +2.4-883 diff --git a/aux/broctl b/aux/broctl index b7932813c6..21bcd850fc 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit b7932813c6bc8a711334bdf2c4beecd7c5e351ca +Subproject commit 21bcd850fca80955501c61e34c783957ee8ba20f diff --git a/doc/cluster/index.rst b/doc/cluster/index.rst index 6e426c005e..5601463144 100644 --- a/doc/cluster/index.rst +++ b/doc/cluster/index.rst @@ -39,9 +39,11 @@ Manager ******* The manager is a Bro process that has two primary jobs. It receives log messages and notices from the rest of the nodes in the cluster using the Bro -communications protocol. The result is a single log instead of many -discrete logs that you have to combine in some manner with post-processing. -The manager also takes the opportunity to de-duplicate notices, and it has the +communications protocol (note that if you are using a logger, then the +logger receives all logs instead of the manager). The result +is a single log instead of many discrete logs that you have to +combine in some manner with post-processing. The manager also takes +the opportunity to de-duplicate notices, and it has the ability to do so since it's acting as the choke point for notices and how notices might be processed into actions (e.g., emailing, paging, or blocking). @@ -51,6 +53,20 @@ connections to the rest of the cluster. Once the workers are started and connect to the manager, logs and notices will start arriving to the manager process from the workers. +Logger +****** +The logger is an optional Bro process that receives log messages from the +rest of the nodes in the cluster using the Bro communications protocol. +The purpose of having a logger receive logs instead of the manager is +to reduce the load on the manager. If no logger is needed, then the +manager will receive logs instead. + +The logger process is started first by BroControl and it only opens its +designated port and waits for connections, it doesn't initiate any +connections to the rest of the cluster. Once the rest of the cluster is +started and connect to the logger, logs will start arriving to the logger +process. + Proxy ***** The proxy is a Bro process that manages synchronized state. Variables can diff --git a/doc/configuration/index.rst b/doc/configuration/index.rst index 800d746e72..fa49c3736e 100644 --- a/doc/configuration/index.rst +++ b/doc/configuration/index.rst @@ -44,7 +44,10 @@ workers can consume a lot of CPU resources. The maximum recommended number of workers to run on a machine should be one or two less than the number of CPU cores available on that machine. Using a load-balancing method (such as PF_RING) along with CPU pinning can decrease the load on -the worker machines. +the worker machines. Also, in order to reduce the load on the manager +process, it is recommended to have a logger in your configuration. If a +logger is defined in your cluster configuration, then it will receive logs +instead of the manager process. Basic Cluster Configuration @@ -61,13 +64,17 @@ a Bro cluster (do this as the Bro user on the manager host only): :doc:`BroControl <../components/broctl/README>` documentation. - Edit the BroControl node configuration file, ``/etc/node.cfg`` - to define where manager, proxies, and workers are to run. For a cluster - configuration, you must comment-out (or remove) the standalone node + to define where logger, manager, proxies, and workers are to run. For a + cluster configuration, you must comment-out (or remove) the standalone node in that file, and either uncomment or add node entries for each node - in your cluster (manager, proxy, and workers). For example, if you wanted - to run four Bro nodes (two workers, one proxy, and a manager) on a cluster - consisting of three machines, your cluster configuration would look like - this:: + in your cluster (logger, manager, proxy, and workers). For example, if you + wanted to run five Bro nodes (two workers, one proxy, a logger, and a + manager) on a cluster consisting of three machines, your cluster + configuration would look like this:: + + [logger] + type=logger + host=10.0.0.10 [manager] type=manager @@ -94,7 +101,7 @@ a Bro cluster (do this as the Bro user on the manager host only): file lists all of the networks which the cluster should consider as local to the monitored environment. -- Install workers and proxies using BroControl:: +- Install Bro on all machines in the cluster using BroControl:: > broctl install @@ -174,7 +181,7 @@ Installing PF_RING 5. Configure BroControl to use PF_RING (explained below). 6. Run "broctl install" on the manager. This command will install Bro and - all required scripts to the other machines in your cluster. + required scripts to all machines in your cluster. Using PF_RING ^^^^^^^^^^^^^ diff --git a/scripts/base/frameworks/cluster/__load__.bro b/scripts/base/frameworks/cluster/__load__.bro index 0f9003514d..1717b833ae 100644 --- a/scripts/base/frameworks/cluster/__load__.bro +++ b/scripts/base/frameworks/cluster/__load__.bro @@ -28,6 +28,14 @@ redef Communication::listen_port = Cluster::nodes[Cluster::node]$p; @if ( Cluster::local_node_type() == Cluster::MANAGER ) @load ./nodes/manager +# If no logger is defined, then the manager receives logs. +@if ( Cluster::manager_is_logger ) +@load ./nodes/logger +@endif +@endif + +@if ( Cluster::local_node_type() == Cluster::LOGGER ) +@load ./nodes/logger @endif @if ( Cluster::local_node_type() == Cluster::PROXY ) diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 55fc084641..261f3f1026 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -31,7 +31,9 @@ export { ## A node type which is allowed to view/manipulate the configuration ## of other nodes in the cluster. CONTROL, - ## A node type responsible for log and policy management. + ## A node type responsible for log management. + LOGGER, + ## A node type responsible for policy management. MANAGER, ## A node type for relaying worker node communication and synchronizing ## worker node state. @@ -50,12 +52,21 @@ export { ## Events raised by a manager and handled by proxies. const manager2proxy_events = /EMPTY/ &redef; + ## Events raised by a manager and handled by loggers. + const manager2logger_events = /EMPTY/ &redef; + + ## Events raised by proxies and handled by loggers. + const proxy2logger_events = /EMPTY/ &redef; + ## Events raised by proxies and handled by a manager. const proxy2manager_events = /EMPTY/ &redef; ## Events raised by proxies and handled by workers. const proxy2worker_events = /EMPTY/ &redef; + ## Events raised by workers and handled by loggers. + const worker2logger_events = /EMPTY/ &redef; + ## Events raised by workers and handled by a manager. const worker2manager_events = /(TimeMachine::command|Drop::.*)/ &redef; @@ -86,6 +97,8 @@ export { p: port; ## Identifier for the interface a worker is sniffing. interface: string &optional; + ## Name of the logger node this node uses. For manager, proxies and workers. + logger: string &optional; ## Name of the manager node this node uses. For workers and proxies. manager: string &optional; ## Name of the proxy node this node uses. For workers and managers. @@ -123,6 +136,12 @@ export { ## Note that BroControl handles all of this automatically. const nodes: table[string] of Node = {} &redef; + ## Indicates whether or not the manager will act as the logger and receive + ## logs. This value should be set in the cluster-layout.bro script (the + ## value should be true only if no logger is specified in Cluster::nodes). + ## Note that BroControl handles this automatically. + const manager_is_logger = T &redef; + ## This is usually supplied on the command line for each instance ## of the cluster that is started up. const node = getenv("CLUSTER_NODE") &redef; diff --git a/scripts/base/frameworks/cluster/nodes/logger.bro b/scripts/base/frameworks/cluster/nodes/logger.bro new file mode 100644 index 0000000000..39dcb751df --- /dev/null +++ b/scripts/base/frameworks/cluster/nodes/logger.bro @@ -0,0 +1,29 @@ +##! This is the core Bro script to support the notion of a cluster logger. +##! +##! The logger is passive (other Bro instances connect to us), and once +##! connected the logger receives logs from other Bro instances. +##! This script will be automatically loaded if necessary based on the +##! type of node being started. + +##! This is where the cluster logger sets it's specific settings for other +##! frameworks and in the core. + +@prefixes += cluster-logger + +## Turn on local logging. +redef Log::enable_local_logging = T; + +## Turn off remote logging since this is the logger and should only log here. +redef Log::enable_remote_logging = F; + +## Log rotation interval. +redef Log::default_rotation_interval = 1 hrs; + +## Alarm summary mail interval. +redef Log::default_mail_alarms_interval = 24 hrs; + +## Use the cluster's archive logging script. +redef Log::default_rotation_postprocessor_cmd = "archive-log"; + +## We're processing essentially *only* remote events. +redef max_remote_events_processed = 10000; diff --git a/scripts/base/frameworks/cluster/nodes/manager.bro b/scripts/base/frameworks/cluster/nodes/manager.bro index 5149dbac7e..e54b090522 100644 --- a/scripts/base/frameworks/cluster/nodes/manager.bro +++ b/scripts/base/frameworks/cluster/nodes/manager.bro @@ -10,17 +10,17 @@ @prefixes += cluster-manager -## Turn off remote logging since this is the manager and should only log here. -redef Log::enable_remote_logging = F; +## Don't do any local logging since the logger handles writing logs. +redef Log::enable_local_logging = F; + +## Turn on remote logging since the logger handles writing logs. +redef Log::enable_remote_logging = T; ## Log rotation interval. -redef Log::default_rotation_interval = 1 hrs; +redef Log::default_rotation_interval = 24 hrs; -## Alarm summary mail interval. -redef Log::default_mail_alarms_interval = 24 hrs; - -## Use the cluster's archive logging script. -redef Log::default_rotation_postprocessor_cmd = "archive-log"; +## Use the cluster's delete-log script. +redef Log::default_rotation_postprocessor_cmd = "delete-log"; ## We're processing essentially *only* remote events. redef max_remote_events_processed = 10000; diff --git a/scripts/base/frameworks/cluster/nodes/worker.bro b/scripts/base/frameworks/cluster/nodes/worker.bro index f876f8592e..826df848f7 100644 --- a/scripts/base/frameworks/cluster/nodes/worker.bro +++ b/scripts/base/frameworks/cluster/nodes/worker.bro @@ -1,6 +1,6 @@ ##! Redefines some options common to all worker nodes within a Bro cluster. ##! In particular, worker nodes do not produce logs locally, instead they -##! send them off to a manager node for processing. +##! send them off to a logger node for processing. @prefixes += cluster-worker diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index 95aff64a6c..971a55d444 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -23,17 +23,40 @@ event bro_init() &priority=9 $connect=F, $class="control", $events=control_events]; - if ( me$node_type == MANAGER ) + if ( me$node_type == LOGGER ) { + if ( n$node_type == MANAGER && n$logger == node ) + Communication::nodes[i] = + [$host=n$ip, $zone_id=n$zone_id, $connect=F, + $class=i, $events=manager2logger_events, $request_logs=T]; + if ( n$node_type == PROXY && n$logger == node ) + Communication::nodes[i] = + [$host=n$ip, $zone_id=n$zone_id, $connect=F, + $class=i, $events=proxy2logger_events, $request_logs=T]; + if ( n$node_type == WORKER && n$logger == node ) + Communication::nodes[i] = + [$host=n$ip, $zone_id=n$zone_id, $connect=F, + $class=i, $events=worker2logger_events, $request_logs=T]; + } + else if ( me$node_type == MANAGER ) + { + if ( n$node_type == LOGGER && me$logger == i ) + Communication::nodes["logger"] = + [$host=n$ip, $zone_id=n$zone_id, $p=n$p, + $connect=T, $retry=retry_interval, + $class=node]; + if ( n$node_type == WORKER && n$manager == node ) Communication::nodes[i] = [$host=n$ip, $zone_id=n$zone_id, $connect=F, - $class=i, $events=worker2manager_events, $request_logs=T]; + $class=i, $events=worker2manager_events, + $request_logs=Cluster::manager_is_logger]; if ( n$node_type == PROXY && n$manager == node ) Communication::nodes[i] = [$host=n$ip, $zone_id=n$zone_id, $connect=F, - $class=i, $events=proxy2manager_events, $request_logs=T]; + $class=i, $events=proxy2manager_events, + $request_logs=Cluster::manager_is_logger]; if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i ) Communication::nodes["time-machine"] = [$host=nodes[i]$ip, @@ -45,6 +68,12 @@ event bro_init() &priority=9 else if ( me$node_type == PROXY ) { + if ( n$node_type == LOGGER && me$logger == i ) + Communication::nodes["logger"] = + [$host=n$ip, $zone_id=n$zone_id, $p=n$p, + $connect=T, $retry=retry_interval, + $class=node]; + if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = [$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i, @@ -76,6 +105,12 @@ event bro_init() &priority=9 } else if ( me$node_type == WORKER ) { + if ( n$node_type == LOGGER && me$logger == i ) + Communication::nodes["logger"] = + [$host=n$ip, $zone_id=n$zone_id, $p=n$p, + $connect=T, $retry=retry_interval, + $class=node]; + if ( n$node_type == MANAGER && me$manager == i ) Communication::nodes["manager"] = [$host=nodes[i]$ip, $zone_id=nodes[i]$zone_id, diff --git a/testing/btest/Baseline/coverage.init-default/missing_loads b/testing/btest/Baseline/coverage.init-default/missing_loads index 826ca4af87..50d52bd0a5 100644 --- a/testing/btest/Baseline/coverage.init-default/missing_loads +++ b/testing/btest/Baseline/coverage.init-default/missing_loads @@ -1,3 +1,4 @@ +-./frameworks/cluster/nodes/logger.bro -./frameworks/cluster/nodes/manager.bro -./frameworks/cluster/nodes/proxy.bro -./frameworks/cluster/nodes/worker.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/logger-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/logger-1..stdout new file mode 100644 index 0000000000..e10770a5cc --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/logger-1..stdout @@ -0,0 +1,5 @@ +Connected to a peer +Connected to a peer +Connected to a peer +Connected to a peer +Connected to a peer diff --git a/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/manager-1..stdout new file mode 100644 index 0000000000..e10770a5cc --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/manager-1..stdout @@ -0,0 +1,5 @@ +Connected to a peer +Connected to a peer +Connected to a peer +Connected to a peer +Connected to a peer diff --git a/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/proxy-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/proxy-1..stdout new file mode 100644 index 0000000000..c3a1950daf --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/proxy-1..stdout @@ -0,0 +1,3 @@ +Connected to a peer +Connected to a peer +Connected to a peer diff --git a/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/proxy-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/proxy-2..stdout new file mode 100644 index 0000000000..c3a1950daf --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/proxy-2..stdout @@ -0,0 +1,3 @@ +Connected to a peer +Connected to a peer +Connected to a peer diff --git a/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/worker-1..stdout new file mode 100644 index 0000000000..c3a1950daf --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/worker-1..stdout @@ -0,0 +1,3 @@ +Connected to a peer +Connected to a peer +Connected to a peer diff --git a/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/worker-2..stdout new file mode 100644 index 0000000000..c3a1950daf --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.cluster.start-it-up-logger/worker-2..stdout @@ -0,0 +1,3 @@ +Connected to a peer +Connected to a peer +Connected to a peer diff --git a/testing/btest/scripts/base/frameworks/cluster/start-it-up-logger.bro b/testing/btest/scripts/base/frameworks/cluster/start-it-up-logger.bro new file mode 100644 index 0000000000..97f3698f36 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/cluster/start-it-up-logger.bro @@ -0,0 +1,76 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run logger-1 CLUSTER_NODE=logger-1 BROPATH=$BROPATH:.. bro %INPUT +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run manager-1 CLUSTER_NODE=manager-1 BROPATH=$BROPATH:.. bro %INPUT +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run proxy-1 CLUSTER_NODE=proxy-1 BROPATH=$BROPATH:.. bro %INPUT +# @TEST-EXEC: btest-bg-run proxy-2 CLUSTER_NODE=proxy-2 BROPATH=$BROPATH:.. bro %INPUT +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run worker-1 CLUSTER_NODE=worker-1 BROPATH=$BROPATH:.. bro %INPUT +# @TEST-EXEC: btest-bg-run worker-2 CLUSTER_NODE=worker-2 BROPATH=$BROPATH:.. bro %INPUT +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: btest-diff logger-1/.stdout +# @TEST-EXEC: btest-diff manager-1/.stdout +# @TEST-EXEC: btest-diff proxy-1/.stdout +# @TEST-EXEC: btest-diff proxy-2/.stdout +# @TEST-EXEC: btest-diff worker-1/.stdout +# @TEST-EXEC: btest-diff worker-2/.stdout + +@TEST-START-FILE cluster-layout.bro +redef Cluster::manager_is_logger = F; +redef Cluster::nodes = { + ["logger-1"] = [$node_type=Cluster::LOGGER, $ip=127.0.0.1, $p=37757/tcp], + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37758/tcp, $logger="logger-1", $workers=set("worker-1")], + ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37759/tcp, $logger="logger-1", $manager="manager-1", $workers=set("worker-1")], + ["proxy-2"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37760/tcp, $logger="logger-1", $manager="manager-1", $workers=set("worker-2")], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $logger="logger-1", $manager="manager-1", $proxy="proxy-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37762/tcp, $logger="logger-1", $manager="manager-1", $proxy="proxy-2", $interface="eth1"], +}; +@TEST-END-FILE + +global fully_connected: event(); + +global peer_count = 0; + +global fully_connected_nodes = 0; + +event fully_connected() + { + ++fully_connected_nodes; + if ( Cluster::node == "logger-1" ) + { + if ( peer_count == 5 && fully_connected_nodes == 5 ) + terminate_communication(); + } + } + +redef Cluster::worker2logger_events += /fully_connected/; +redef Cluster::proxy2logger_events += /fully_connected/; +redef Cluster::manager2logger_events += /fully_connected/; + +event remote_connection_handshake_done(p: event_peer) + { + print "Connected to a peer"; + ++peer_count; + if ( Cluster::node == "logger-1" ) + { + if ( peer_count == 5 && fully_connected_nodes == 5 ) + terminate_communication(); + } + else if ( Cluster::node == "manager-1" ) + { + if ( peer_count == 5 ) + event fully_connected(); + } + else + { + if ( peer_count == 3 ) + event fully_connected(); + } + } + +event remote_connection_closed(p: event_peer) + { + terminate(); + }