mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
Clean up to cluster framework to make event handling clearer.
- Fixed a bug where notices were being passed to proxies. This was a mistake and should greatly reduce load on many clusters. - Cluster event regex variables renamed to: - Notice::manager2worker_events - Notice::manager2proxy_events - Notice::worker2manager_events - Notice::worker2proxy_events - Notice::proxy2manager_events - Notice::proxy2worker_events - The default Notice::policy set is cleared for all cluster nodes except for managers to cause all default notice processing to occur on managers. This should reduce load on workers slightly.
This commit is contained in:
Seth Hall
parent
870bdf796d
commit
aa9fdf38bb
4 changed files with 55 additions and 26 deletions
|
|
@ -19,14 +19,29 @@ export {
|
|||
TIME_MACHINE,
|
||||
};
|
||||
|
||||
## Events raised by the manager and handled by the workers.
|
||||
const manager_events = /Drop::.*/ &redef;
|
||||
## Events raised by a manager and handled by the workers.
|
||||
const manager2worker_events = /Drop::.*/ &redef;
|
||||
|
||||
## Events raised by the proxies and handled by the manager.
|
||||
const proxy_events = /EMPTY/ &redef;
|
||||
## Events raised by a manager and handled by proxies.
|
||||
const manager2proxy_events = /EMPTY/ &redef;
|
||||
|
||||
## Events raised by workers and handled by the manager.
|
||||
const worker_events = /(TimeMachine::command|Drop::.*)/ &redef;
|
||||
## Events raised by proxies and handled by a manager.
|
||||
const proxy2manager_events = /EMPTY/ &redef;
|
||||
|
||||
## Events raised by proxies and handled by workers.
|
||||
const proxy2worker_events = /EMPTY/ &redef;
|
||||
|
||||
## Events raised by workers and handled by a manager.
|
||||
const worker2manager_events = /(TimeMachine::command|Drop::.*)/ &redef;
|
||||
|
||||
## Events raised by workers and handled by proxies..
|
||||
const worker2proxy_events = /EMPTY/ &redef;
|
||||
|
||||
## Events raised by TimeMachine instances and handled by a manager.
|
||||
const tm2manager_events = /EMPTY/ &redef;
|
||||
|
||||
## Events raised by TimeMachine instances and handled by workers.
|
||||
const tm2worker_events = /EMPTY/ &redef;
|
||||
|
||||
## Events sent by the control host (i.e. BroControl) when dynamically
|
||||
## connecting to a running instance to update settings or request data.
|
||||
|
|
|
|||
|
|
@ -24,23 +24,24 @@ event bro_init() &priority=9
|
|||
if ( n$node_type == WORKER && n$manager == node )
|
||||
Communication::nodes[i] =
|
||||
[$host=n$ip, $connect=F,
|
||||
$class=i, $events=worker_events, $request_logs=T];
|
||||
$class=i, $events=worker2manager_events, $request_logs=T];
|
||||
|
||||
if ( n$node_type == PROXY && n$manager == node )
|
||||
Communication::nodes[i] =
|
||||
[$host=n$ip, $connect=F,
|
||||
$class=i, $events=proxy_events, $request_logs=T];
|
||||
$class=i, $events=proxy2manager_events, $request_logs=T];
|
||||
|
||||
if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i )
|
||||
Communication::nodes["time-machine"] = [$host=nodes[i]$ip, $p=nodes[i]$p,
|
||||
$connect=T, $retry=1min];
|
||||
$connect=T, $retry=1min,
|
||||
$events=tm2manager_events];
|
||||
}
|
||||
|
||||
else if ( me$node_type == PROXY )
|
||||
{
|
||||
if ( n$node_type == WORKER && n$proxy == node )
|
||||
Communication::nodes[i] =
|
||||
[$host=n$ip, $connect=F, $class=i, $events=worker_events];
|
||||
[$host=n$ip, $connect=F, $class=i, $events=worker2proxy_events];
|
||||
|
||||
# accepts connections from the previous one.
|
||||
# (This is not ideal for setups with many proxies)
|
||||
|
|
@ -58,25 +59,35 @@ event bro_init() &priority=9
|
|||
|
||||
# Finally the manager, to send it status updates.
|
||||
if ( n$node_type == MANAGER && me$manager == i )
|
||||
Communication::nodes["manager"] = [$host=nodes[i]$ip, $p=nodes[i]$p,
|
||||
Communication::nodes["manager"] = [$host=nodes[i]$ip,
|
||||
$p=nodes[i]$p,
|
||||
$connect=T, $retry=1mins,
|
||||
$class=node];
|
||||
$class=node,
|
||||
$events=manager2proxy_events];
|
||||
}
|
||||
else if ( me$node_type == WORKER )
|
||||
{
|
||||
if ( n$node_type == MANAGER && me$manager == i )
|
||||
Communication::nodes["manager"] = [$host=nodes[i]$ip, $p=nodes[i]$p,
|
||||
Communication::nodes["manager"] = [$host=nodes[i]$ip,
|
||||
$p=nodes[i]$p,
|
||||
$connect=T, $retry=1mins,
|
||||
$class=node, $events=manager_events];
|
||||
$class=node,
|
||||
$events=manager2worker_events];
|
||||
|
||||
if ( n$node_type == PROXY && me$proxy == i )
|
||||
Communication::nodes["proxy"] = [$host=nodes[i]$ip, $p=nodes[i]$p,
|
||||
$connect=T, $retry=1mins, $sync=T,
|
||||
$class=node];
|
||||
Communication::nodes["proxy"] = [$host=nodes[i]$ip,
|
||||
$p=nodes[i]$p,
|
||||
$connect=T, $retry=1mins,
|
||||
$sync=T, $class=node,
|
||||
$events=proxy2worker_events];
|
||||
|
||||
if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i )
|
||||
Communication::nodes["time-machine"] = [$host=nodes[i]$ip, $p=nodes[i]$p,
|
||||
$connect=T, $retry=1min];
|
||||
if ( n$node_type == TIME_MACHINE &&
|
||||
me?$time_machine && me$time_machine == i )
|
||||
Communication::nodes["time-machine"] = [$host=nodes[i]$ip,
|
||||
$p=nodes[i]$p,
|
||||
$connect=T,
|
||||
$retry=1min,
|
||||
$events=tm2worker_events];
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,8 +87,8 @@ global index_requests: table[string, ID, string, Index] of count &create_expire=
|
|||
global recent_global_view_indexes: table[ID, string, Index] of count &create_expire=5mins &default=0;
|
||||
|
||||
# Add events to the cluster framework to make this work.
|
||||
redef Cluster::manager_events += /Metrics::cluster_(filter_request|index_request)/;
|
||||
redef Cluster::worker_events += /Metrics::cluster_(filter_response|index_response|index_intermediate_response)/;
|
||||
redef Cluster::manager2worker_events += /Metrics::cluster_(filter_request|index_request)/;
|
||||
redef Cluster::worker2manager_events += /Metrics::cluster_(filter_response|index_response|index_intermediate_response)/;
|
||||
|
||||
@if ( Cluster::local_node_type() != Cluster::MANAGER )
|
||||
# This is done on all non-manager node types in the event that a metric is
|
||||
|
|
|
|||
|
|
@ -8,11 +8,14 @@ module Notice;
|
|||
# Define the event used to transport notices on the cluster.
|
||||
global cluster_notice: event(n: Notice::Info);
|
||||
|
||||
redef Cluster::manager_events += /Notice::begin_suppression/;
|
||||
redef Cluster::proxy_events += /Notice::cluster_notice/;
|
||||
redef Cluster::worker_events += /Notice::cluster_notice/;
|
||||
redef Cluster::manager2worker_events += /Notice::begin_suppression/;
|
||||
redef Cluster::worker2manager_events += /Notice::cluster_notice/;
|
||||
|
||||
@if ( Cluster::local_node_type() != Cluster::MANAGER )
|
||||
# The notice policy is completely handled by the manager and shouldn't be
|
||||
# done by workers or proxies to save time for packet processing.
|
||||
redef policy = {};
|
||||
|
||||
event Notice::begin_suppression(n: Notice::Info)
|
||||
{
|
||||
suppressing[n$note, n$identifier] = n;
|
||||
|
|
@ -24,7 +27,7 @@ event Notice::notice(n: Notice::Info)
|
|||
event Notice::cluster_notice(n);
|
||||
}
|
||||
|
||||
event bro_init() &priority=3
|
||||
event bro_init() &priority=-3
|
||||
{
|
||||
# Workers and proxies need to disable the notice streams because notice
|
||||
# events are forwarded directly instead of being logged remotely.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue