Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Update default Broker/CAF thread tuning

Browse source
This commit is contained in:
Jon Siwek 2018-09-07 17:49:34 -05:00
parent 9af0255ef7
commit 4bd6da7186
4 changed files with 57 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

4
CHANGES
View file

@ -1,4 +1,8 @@
2.5-988 | 2018-09-07 17:49:34 -0500
* Update default Broker/CAF thread tuning (Jon Siwek, Corelight)
2.5-987 | 2018-09-07 11:24:34 -0500 2.5-987 | 2018-09-07 11:24:34 -0500
* Update NEWS explaining Bro runs as 1 process instead of 2 (Daniel Thayer) * Update NEWS explaining Bro runs as 1 process instead of 2 (Daniel Thayer)

2
VERSION
View file

@ -1 +1 @@
2.5-987 2.5-988

40
scripts/base/frameworks/broker/main.bro
View file

@ -61,17 +61,37 @@ export {
## control mechanisms). ## control mechanisms).
const congestion_queue_size = 200 &redef; const congestion_queue_size = 200 &redef;
## Max number of threads to use for Broker/CAF functionality. Setting to ## Max number of threads to use for Broker/CAF functionality. The
## zero implies using the value of BRO_BROKER_MAX_THREADS environment ## BRO_BROKER_MAX_THREADS environment variable overrides this setting.
## variable, if set, or else typically defaults to 4 (actually 2 threads const max_threads = 1 &redef;
## when simply reading offline pcaps as there's not expected to be any
## communication and more threads just adds more overhead).
const max_threads = 0 &redef;
## Max number of microseconds for under-utilized Broker/CAF ## Interval of time for under-utilized Broker/CAF threads to sleep
## threads to sleep. Using zero will cause this to be automatically ## when in "moderate" mode.
## determined or just use CAF's default setting. const moderate_sleep = 16 msec &redef;
const max_sleep = 0 &redef;
## Interval of time for under-utilized Broker/CAF threads to sleep
## when in "relaxed" mode.
const relaxed_sleep = 64 msec &redef;
## Number of work-stealing polling attempts for Broker/CAF threads
## in "aggressive" mode.
const aggressive_polls = 5 &redef;
## Number of work-stealing polling attempts for Broker/CAF threads
## in "moderate" mode.
const moderate_polls = 5 &redef;
## Frequency of work-stealing polling attempts for Broker/CAF threads
## in "aggressive" mode.
const aggressive_interval = 4 &redef;
## Frequency of work-stealing polling attempts for Broker/CAF threads
## in "moderate" mode.
const moderate_interval = 2 &redef;
## Frequency of work-stealing polling attempts for Broker/CAF threads
## in "relaxed" mode.
const relaxed_interval = 1 &redef;
## Forward all received messages to subscribing peers. ## Forward all received messages to subscribing peers.
const forward_messages = F &redef; const forward_messages = F &redef;

59
src/broker/Manager.cc
View file

@ -179,47 +179,32 @@ void Manager::InitPostScript()
options.use_real_time = ! reading_pcaps; options.use_real_time = ! reading_pcaps;
BrokerConfig config{std::move(options)}; BrokerConfig config{std::move(options)};
auto max_threads = get_option("Broker::max_threads")->AsCount();
auto max_sleep = get_option("Broker::max_sleep")->AsCount();
if ( max_threads ) auto max_threads_env = getenv("BRO_BROKER_MAX_THREADS");
config.set("scheduler.max-threads", max_threads);
if ( max_threads_env )
config.set("scheduler.max-threads", atoi(max_threads_env));
else else
{ config.set("scheduler.max-threads",
auto max_threads_env = getenv("BRO_BROKER_MAX_THREADS"); get_option("Broker::max_threads")->AsCount());
if ( max_threads_env ) config.set("work-stealing.moderate-sleep-duration-us",
config.set("scheduler.max-threads", atoi(max_threads_env)); static_cast<unsigned>(get_option("Broker::moderate_sleep")->AsInterval() / Microseconds));
else
{
// On high-core-count systems, letting CAF spawn a thread per core
// can lead to significant performance problems even if most
// threads are under-utilized. Related:
// https://github.com/actor-framework/actor-framework/issues/699
if ( reading_pcaps )
config.set("scheduler.max-threads", 2u);
else
// If the goal was to map threads to actors, 4 threads seems
// like a minimal default that could make sense -- the main
// actors that should be doing work are (1) the core,
// (2) the subscriber, (3) data stores (actually made of
// a frontend + proxy actor). Number of data stores may
// actually vary, but lumped togather for simplicity. A (4)
// may be CAF's multiplexing or other internals...
// 4 is also the minimum number that CAF uses by default,
// even for systems with less than 4 cores.
config.set("scheduler.max-threads", 4u);
}
}
if ( max_sleep ) config.set("work-stealing.relaxed-sleep-duration-us",
config.set("work-stealing.relaxed-sleep-duration-us", max_sleep); static_cast<unsigned>(get_option("Broker::relaxed_sleep")->AsInterval() / Microseconds));
else
// 64ms is just an arbitrary amount derived from testing config.set("work-stealing.aggressive-poll-attempts",
// the overhead of a unused CAF actor system on a 32-core system. get_option("Broker::aggressive_polls")->AsCount());
// Performance was within 2% of baseline timings (w/o CAF) config.set("work-stealing.moderate-poll-attempts",
// when using this sleep duration. get_option("Broker::moderate_polls")->AsCount());
config.set("work-stealing.relaxed-sleep-duration-us", 64000);
config.set("work-stealing.aggressive-steal-interval",
get_option("Broker::aggressive_interval")->AsCount());
config.set("work-stealing.moderate-steal-interval",
get_option("Broker::moderate_interval")->AsCount());
config.set("work-stealing.relaxed-steal-interval",
get_option("Broker::relaxed_interval")->AsCount());
auto cqs = get_option("Broker::congestion_queue_size")->AsCount(); auto cqs = get_option("Broker::congestion_queue_size")->AsCount();
bstate = std::make_shared<BrokerState>(std::move(config), cqs); bstate = std::make_shared<BrokerState>(std::move(config), cqs);