diff --git a/CHANGES b/CHANGES index db37a0e18d..ec17130257 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-988 | 2018-09-07 17:49:34 -0500 + + * Update default Broker/CAF thread tuning (Jon Siwek, Corelight) + 2.5-987 | 2018-09-07 11:24:34 -0500 * Update NEWS explaining Bro runs as 1 process instead of 2 (Daniel Thayer) diff --git a/VERSION b/VERSION index 7e3c039944..78c217a878 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-987 +2.5-988 diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index f00edfe1ba..be4dbaf381 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -61,17 +61,37 @@ export { ## control mechanisms). const congestion_queue_size = 200 &redef; - ## Max number of threads to use for Broker/CAF functionality. Setting to - ## zero implies using the value of BRO_BROKER_MAX_THREADS environment - ## variable, if set, or else typically defaults to 4 (actually 2 threads - ## when simply reading offline pcaps as there's not expected to be any - ## communication and more threads just adds more overhead). - const max_threads = 0 &redef; + ## Max number of threads to use for Broker/CAF functionality. The + ## BRO_BROKER_MAX_THREADS environment variable overrides this setting. + const max_threads = 1 &redef; - ## Max number of microseconds for under-utilized Broker/CAF - ## threads to sleep. Using zero will cause this to be automatically - ## determined or just use CAF's default setting. - const max_sleep = 0 &redef; + ## Interval of time for under-utilized Broker/CAF threads to sleep + ## when in "moderate" mode. + const moderate_sleep = 16 msec &redef; + + ## Interval of time for under-utilized Broker/CAF threads to sleep + ## when in "relaxed" mode. + const relaxed_sleep = 64 msec &redef; + + ## Number of work-stealing polling attempts for Broker/CAF threads + ## in "aggressive" mode. + const aggressive_polls = 5 &redef; + + ## Number of work-stealing polling attempts for Broker/CAF threads + ## in "moderate" mode. + const moderate_polls = 5 &redef; + + ## Frequency of work-stealing polling attempts for Broker/CAF threads + ## in "aggressive" mode. + const aggressive_interval = 4 &redef; + + ## Frequency of work-stealing polling attempts for Broker/CAF threads + ## in "moderate" mode. + const moderate_interval = 2 &redef; + + ## Frequency of work-stealing polling attempts for Broker/CAF threads + ## in "relaxed" mode. + const relaxed_interval = 1 &redef; ## Forward all received messages to subscribing peers. const forward_messages = F &redef; diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 7861a5d420..c456954d4b 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -179,47 +179,32 @@ void Manager::InitPostScript() options.use_real_time = ! reading_pcaps; BrokerConfig config{std::move(options)}; - auto max_threads = get_option("Broker::max_threads")->AsCount(); - auto max_sleep = get_option("Broker::max_sleep")->AsCount(); - if ( max_threads ) - config.set("scheduler.max-threads", max_threads); + auto max_threads_env = getenv("BRO_BROKER_MAX_THREADS"); + + if ( max_threads_env ) + config.set("scheduler.max-threads", atoi(max_threads_env)); else - { - auto max_threads_env = getenv("BRO_BROKER_MAX_THREADS"); + config.set("scheduler.max-threads", + get_option("Broker::max_threads")->AsCount()); - if ( max_threads_env ) - config.set("scheduler.max-threads", atoi(max_threads_env)); - else - { - // On high-core-count systems, letting CAF spawn a thread per core - // can lead to significant performance problems even if most - // threads are under-utilized. Related: - // https://github.com/actor-framework/actor-framework/issues/699 - if ( reading_pcaps ) - config.set("scheduler.max-threads", 2u); - else - // If the goal was to map threads to actors, 4 threads seems - // like a minimal default that could make sense -- the main - // actors that should be doing work are (1) the core, - // (2) the subscriber, (3) data stores (actually made of - // a frontend + proxy actor). Number of data stores may - // actually vary, but lumped togather for simplicity. A (4) - // may be CAF's multiplexing or other internals... - // 4 is also the minimum number that CAF uses by default, - // even for systems with less than 4 cores. - config.set("scheduler.max-threads", 4u); - } - } + config.set("work-stealing.moderate-sleep-duration-us", + static_cast(get_option("Broker::moderate_sleep")->AsInterval() / Microseconds)); - if ( max_sleep ) - config.set("work-stealing.relaxed-sleep-duration-us", max_sleep); - else - // 64ms is just an arbitrary amount derived from testing - // the overhead of a unused CAF actor system on a 32-core system. - // Performance was within 2% of baseline timings (w/o CAF) - // when using this sleep duration. - config.set("work-stealing.relaxed-sleep-duration-us", 64000); + config.set("work-stealing.relaxed-sleep-duration-us", + static_cast(get_option("Broker::relaxed_sleep")->AsInterval() / Microseconds)); + + config.set("work-stealing.aggressive-poll-attempts", + get_option("Broker::aggressive_polls")->AsCount()); + config.set("work-stealing.moderate-poll-attempts", + get_option("Broker::moderate_polls")->AsCount()); + + config.set("work-stealing.aggressive-steal-interval", + get_option("Broker::aggressive_interval")->AsCount()); + config.set("work-stealing.moderate-steal-interval", + get_option("Broker::moderate_interval")->AsCount()); + config.set("work-stealing.relaxed-steal-interval", + get_option("Broker::relaxed_interval")->AsCount()); auto cqs = get_option("Broker::congestion_queue_size")->AsCount(); bstate = std::make_shared(std::move(config), cqs);