From 446f49e6bcde7328438bf057130373aa67cf1335 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 24 Apr 2025 16:15:56 -0700 Subject: [PATCH] Switch Broker's default backpressure policy to drop_oldest, bump buffer sizes At every site where we've dug into backpressure disconnect findings, it has been the case that the default values were too small. 8192, so 4x the old default, suffices at every site to drown out premature disconnects. With metrics now available for the send buffers regardless of backpressure overflow policy, this also switches the default from "disconnect" to "drop_oldest" (for both peers and websockets), meaning that peerings remain untouched but the oldest queued message simply gets dropped when a new message is enqueued. With this policy, the number of backpressure overflows is then simply the count of discarded messages, something that users can tune to see drop to zero in everyday use. Another benefit is that marginal overflows cause less message loss than when an entire buffer's worth (plus potentially more in-flight messages) gets thrown out with a disconnect. (cherry picked from commit 841a40ff8888fe392b7a974939cd4a39ababf791) --- scripts/base/frameworks/broker/main.zeek | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/base/frameworks/broker/main.zeek b/scripts/base/frameworks/broker/main.zeek index 38d1b5179b..91168957a2 100644 --- a/scripts/base/frameworks/broker/main.zeek +++ b/scripts/base/frameworks/broker/main.zeek @@ -88,21 +88,21 @@ export { ## Max number of items we buffer at most per peer. What action to take when ## the buffer reaches its maximum size is determined by - ## `peer_overflow_policy`. - const peer_buffer_size = 2048 &redef; + ## :zeek:see:`Broker::peer_overflow_policy`. + const peer_buffer_size = 8192 &redef; ## Configures how Broker responds to peers that cannot keep up with the ## incoming message rate. Available strategies: ## - disconnect: drop the connection to the unresponsive peer ## - drop_newest: replace the newest message in the buffer ## - drop_oldest: removed the olsted message from the buffer, then append - const peer_overflow_policy = "disconnect" &redef; + const peer_overflow_policy = "drop_oldest" &redef; - ## Same as `peer_buffer_size` but for WebSocket clients. - const web_socket_buffer_size = 512 &redef; + ## Same as :zeek:see:`Broker::peer_buffer_size` but for WebSocket clients. + const web_socket_buffer_size = 8192 &redef; - ## Same as `peer_overflow_policy` but for WebSocket clients. - const web_socket_overflow_policy = "disconnect" &redef; + ## Same as :zeek:see:`Broker::peer_overflow_policy` but for WebSocket clients. + const web_socket_overflow_policy = "drop_oldest" &redef; ## How frequently Zeek resets some peering/client buffer statistics, ## such as ``max_queued_recently`` in :zeek:see:`BrokerPeeringStats`.