Merge remote-tracking branch 'origin/topic/seth/bittorrent-fix-and-dpd-sig-breakout' into topic/seth/faf-updates

Conflicts: magic scripts/base/protocols/http/__load__.bro scripts/base/protocols/irc/__load__.bro scripts/base/protocols/smtp/__load__.bro
2025-10-02 06:38:20 +00:00 · 2013-07-10 16:28:38 -04:00 · 2013-07-10 16:28:38 -04:00 · 2e0912b543
commit 2e0912b543
parent 99d604c9b5 8322bbfd62
54 changed files with 880 additions and 381 deletions
--- a/21
+++ b/21
@ -1,4 +1,25 @@

+2.1-798 | 2013-07-08 13:05:37 -0700
+
+  * Rewrite of the packet filter framework. (Seth Hall)
+
+    This includes:
+
+    - Plugin interface for adding filtering mechanisms.
+
+    - Integrated the packet filter framework with the analyzer
+      framework to retrieve well-known ports from there.
+
+    - Support for BPF-based load balancing (IPv4 and IPv6).  This will
+      tie in with upcoming BroControl support for configuring this.
+
+    - Support for BPF-based connection sampling.
+
+    - Support for "shunting" traffic with BPF filters.
+
+    - Replaced PacketFilter::all_packets with
+      PacketFilter::enable_auto_protocol_capture_filters.
+
 2.1-784 | 2013-07-04 22:28:48 -0400

  * Add a call to lookup_connection in SSH scripts to update connval. (Seth Hall)
--- a/8
+++ b/8
@ -104,6 +104,10 @@ New Functionality

      - IRC DCC transfers: Record to disk.

+- New packet filter framework supports BPF-based load-balancing,
+  shunting, and sampling; plus plugin support to customize filters
+  dynamically.
+
 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~

@ -182,6 +186,10 @@ Changed Functionality
 - The SSH::Login notice has been superseded by an corresponding
  intelligence framework observation (SSH::SUCCESSFUL_LOGIN).

+- PacketFilter::all_packets has been replaced with
+  PacketFilter::enable_auto_protocol_capture_filters.
+
+
 Bro 2.1
 -------

--- a/2
+++ b/2
@ -1 +1 @@
-2.1-784
+2.1-798
--- a/aux/btest
+++ b/aux/btest
@ -1 +1 @@
-Subproject commit d5b8df42cb9c398142e02d4bf8ede835fd0227f4
+Subproject commit c2e73c9e1efed6bfdf2d977d716c97773c39492e
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@ -112,6 +112,7 @@ rest_target(${psd} base/frameworks/notice/non-cluster.bro)
 rest_target(${psd} base/frameworks/notice/weird.bro)
 rest_target(${psd} base/frameworks/packet-filter/main.bro)
 rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
+rest_target(${psd} base/frameworks/packet-filter/utils.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
@ -190,6 +191,7 @@ rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro)
 rest_target(${psd} policy/frameworks/intel/smtp.bro)
 rest_target(${psd} policy/frameworks/intel/ssl.bro)
 rest_target(${psd} policy/frameworks/intel/where-locations.bro)
+rest_target(${psd} policy/frameworks/packet-filter/shunt.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
@ -198,6 +200,7 @@ rest_target(${psd} policy/integration/collective-intel/main.bro)
 rest_target(${psd} policy/misc/app-metrics.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
 rest_target(${psd} policy/misc/detect-traceroute/main.bro)
+rest_target(${psd} policy/misc/load-balancing.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
 rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/scan.bro)
--- a/scripts/base/frameworks/analyzer/main.bro
+++ b/scripts/base/frameworks/analyzer/main.bro
@ -10,6 +10,8 @@
 ##! the analyzers themselves, and documented in their analyzer-specific
 ##! description along with the events that they generate.

+@load base/frameworks/packet-filter/utils
+
 module Analyzer;

 export {
@ -98,6 +100,20 @@ export {
 	global schedule_analyzer: function(orig: addr, resp: addr, resp_p: port,
 	                                   analyzer: Analyzer::Tag, tout: interval) : bool;

+	## Automatically creates a BPF filter for the specified protocol based
+	## on the data supplied for the protocol through the
+	## :bro:see:`Analyzer::register_for_ports` function.
+	##
+	## tag: The analyzer tag.
+	##
+	## Returns: BPF filter string.
+	global analyzer_to_bpf: function(tag: Analyzer::Tag): string;
+
+	## Create a BPF filter which matches all of the ports defined
+	## by the various protocol analysis scripts as "registered ports"
+	## for the protocol.
+	global get_bpf: function(): string;
+
 	## A set of analyzers to disable by default at startup. The default set
 	## contains legacy analyzers that are no longer supported.
 	global disabled_analyzers: set[Analyzer::Tag] = {
@ -177,3 +193,25 @@ function schedule_analyzer(orig: addr, resp: addr, resp_p: port,
 	return __schedule_analyzer(orig, resp, resp_p, analyzer, tout);
 	}

+function analyzer_to_bpf(tag: Analyzer::Tag): string
+	{
+	# Return an empty string if an undefined analyzer was given.
+	if ( tag !in ports )
+		return "";
+
+	local output = "";
+	for ( p in ports[tag] )
+		output = PacketFilter::combine_filters(output, "or", PacketFilter::port_to_bpf(p));
+	return output;
+	}
+
+function get_bpf(): string
+	{
+	local output = "";
+	for ( tag in ports )
+		{
+		output = PacketFilter::combine_filters(output, "or", analyzer_to_bpf(tag));
+		}
+	return output;
+	}
+
--- a/scripts/base/frameworks/communication/main.bro
+++ b/scripts/base/frameworks/communication/main.bro
@ -216,12 +216,9 @@ function setup_peer(p: event_peer, node: Node)
 		request_remote_events(p, node$events);
 		}

-	if ( node?$capture_filter )
+	if ( node?$capture_filter && node$capture_filter != "" )
 		{
 		local filter = node$capture_filter;
-		if ( filter == "" )
-			filter = PacketFilter::default_filter;
-
 		do_script_log(p, fmt("sending capture_filter: %s", filter));
 		send_capture_filter(p, filter);
 		}
--- a/scripts/base/frameworks/dpd/dpd.sig
+++ b/scripts/base/frameworks/dpd/dpd.sig
@ -1,212 +0,0 @@
-# Signatures to initiate dynamic protocol detection.
-
-signature dpd_ftp_client {
-  ip-proto == tcp
-  payload /(|.*[\n\r]) *[uU][sS][eE][rR] /
-  tcp-state originator
-}
-
-# Match for server greeting (220, 120) and for login or passwd
-# required (230, 331).
-signature dpd_ftp_server {
-  ip-proto == tcp
-  payload /[\n\r ]*(120|220)[^0-9].*[\n\r] *(230|331)[^0-9]/
-  tcp-state responder
-  requires-reverse-signature dpd_ftp_client
-  enable "ftp"
-}
-
-signature dpd_http_client {
-  ip-proto == tcp
-  payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/
-  tcp-state originator
-}
-
-signature dpd_http_server {
-  ip-proto == tcp
-  payload /^HTTP\/[0-9]/
-  tcp-state responder
-  requires-reverse-signature dpd_http_client
-  enable "http"
-}
-
-signature dpd_bittorrenttracker_client {
-  ip-proto == tcp
-  payload /^.*\/announce\?.*info_hash/
-  tcp-state originator
-}
-
-signature dpd_bittorrenttracker_server {
-  ip-proto == tcp
-  payload /^HTTP\/[0-9]/
-  tcp-state responder
-  requires-reverse-signature dpd_bittorrenttracker_client
-  enable "bittorrenttracker"
-}
-
-signature dpd_bittorrent_peer1 {
-  ip-proto == tcp
-  payload /^\x13BitTorrent protocol/
-  tcp-state originator
-}
-
-signature dpd_bittorrent_peer2 {
-  ip-proto == tcp
-  payload /^\x13BitTorrent protocol/
-  tcp-state responder
-  requires-reverse-signature dpd_bittorrent_peer1
-  enable "bittorrent"
-}
-
-signature irc_client1 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Uu][Ss][Ee][Rr] +.+[\n\r]+ *[Nn][Ii][Cc][Kk] +.*[\r\n]/
-  requires-reverse-signature irc_server_reply
-  tcp-state originator
-  enable "irc"
-}
-
-signature irc_client2 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Nn][Ii][Cc][Kk] +.+[\r\n]+ *[Uu][Ss][Ee][Rr] +.+[\r\n]/
-  requires-reverse-signature irc_server_reply
-  tcp-state originator
-  enable "irc"
-}
-
-signature irc_server_reply {
-  ip-proto == tcp
-  payload /^(|.*[\n\r])(:[^ \n\r]+ )?[0-9][0-9][0-9] /
-  tcp-state responder
-}
-
-signature irc_server_to_server1 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/
-}
-
-signature irc_server_to_server2 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/
-  requires-reverse-signature irc_server_to_server1
-  enable "irc"
-}
-
-signature dpd_smtp_client {
-  ip-proto == tcp
-  payload /(|.*[\n\r])[[:space:]]*([hH][eE][lL][oO]|[eE][hH][lL][oO])/
-  requires-reverse-signature dpd_smtp_server
-  enable "smtp"
-  tcp-state originator
-}
-
-signature dpd_smtp_server {
-  ip-proto == tcp
-  payload /^[[:space:]]*220[[:space:]-]/
-  tcp-state responder
-}
-
-signature dpd_ssh_client {
-  ip-proto == tcp
-  payload /^[sS][sS][hH]-/
-  requires-reverse-signature dpd_ssh_server
-  enable "ssh"
-  tcp-state originator
-}
-
-signature dpd_ssh_server {
-  ip-proto == tcp
-  payload /^[sS][sS][hH]-/
-  tcp-state responder
-}
-
-signature dpd_pop3_server {
-  ip-proto == tcp
-  payload /^\+OK/
-  requires-reverse-signature dpd_pop3_client
-  enable "pop3"
-  tcp-state responder
-}
-
-signature dpd_pop3_client {
-  ip-proto == tcp
-  payload /(|.*[\r\n])[[:space:]]*([uU][sS][eE][rR][[:space:]]|[aA][pP][oO][pP][[:space:]]|[cC][aA][pP][aA]|[aA][uU][tT][hH])/
-  tcp-state originator
-}
-
-signature dpd_ssl_server {
-  ip-proto == tcp
-  # Server hello.
-  payload /^(\x16\x03[\x00\x01\x02]..\x02...\x03[\x00\x01\x02]|...?\x04..\x00\x02).*/
-  requires-reverse-signature dpd_ssl_client
-  enable "ssl"
-  tcp-state responder
-}
-
-signature dpd_ssl_client {
-  ip-proto == tcp
-  # Client hello.
-  payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/
-  tcp-state originator
-}
-
-signature dpd_ayiya {
-  ip-proto = udp
-  payload /^..\x11\x29/
-  enable "ayiya"
-}
-
-signature dpd_teredo {
-  ip-proto = udp
-  payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/
-  enable "teredo"
-}
-
-signature dpd_socks4_client {
-	ip-proto == tcp
-	# '32' is a rather arbitrary max length for the user name.
-	payload /^\x04[\x01\x02].{0,32}\x00/
-	tcp-state originator
-}
-
-signature dpd_socks4_server {
-	ip-proto == tcp
-	requires-reverse-signature dpd_socks4_client
-	payload /^\x00[\x5a\x5b\x5c\x5d]/
-	tcp-state responder
-	enable "socks"
-}
-
-signature dpd_socks4_reverse_client {
-	ip-proto == tcp
-	# '32' is a rather arbitrary max length for the user name.
-	payload /^\x04[\x01\x02].{0,32}\x00/
-	tcp-state responder
-}
-
-signature dpd_socks4_reverse_server {
-	ip-proto == tcp
-	requires-reverse-signature dpd_socks4_reverse_client
-	payload /^\x00[\x5a\x5b\x5c\x5d]/
-	tcp-state originator
-	enable "socks"
-}
-
-signature dpd_socks5_client {
-	ip-proto == tcp
-	# Watch for a few authentication methods to reduce false positives.
-	payload /^\x05.[\x00\x01\x02]/
-	tcp-state originator
-}
-
-signature dpd_socks5_server {
-	ip-proto == tcp
-	requires-reverse-signature dpd_socks5_client
-	# Watch for a single authentication method to be chosen by the server or
-	# the server to indicate the no authentication is required.
-	payload /^\x05(\x00|\x01[\x00\x01\x02])/
-	tcp-state responder
-	enable "socks"
-}
-
-
--- a/scripts/base/frameworks/dpd/main.bro
+++ b/scripts/base/frameworks/dpd/main.bro
@ -3,8 +3,6 @@

 module DPD;

-@load-sigs ./dpd.sig
-
 export {
 	## Add the DPD logging stream identifier.
 	redef enum Log::ID += { LOG };
--- a/scripts/base/frameworks/packet-filter/load.bro
+++ b/scripts/base/frameworks/packet-filter/load.bro
@ -1,2 +1,3 @@
+@load ./utils
@load ./main
@load ./netstats
--- a/scripts/base/frameworks/packet-filter/main.bro
+++ b/scripts/base/frameworks/packet-filter/main.bro
@ -1,10 +1,12 @@
 ##! This script supports how Bro sets it's BPF capture filter.  By default
-##! Bro sets an unrestricted filter that allows all traffic.  If a filter
+##! Bro sets a capture filter that allows all traffic.  If a filter
 ##! is set on the command line, that filter takes precedence over the default
 ##! open filter and all filters defined in Bro scripts with the
 ##! :bro:id:`capture_filters` and :bro:id:`restrict_filters` variables.

@load base/frameworks/notice
+@load base/frameworks/analyzer
+@load ./utils

 module PacketFilter;

@ -14,11 +16,14 @@ export {

 	## Add notice types related to packet filter errors.
 	redef enum Notice::Type += {
-		## This notice is generated if a packet filter is unable to be compiled.
+		## This notice is generated if a packet filter cannot be compiled.
 		Compile_Failure,

-		## This notice is generated if a packet filter is fails to install.
+		## Generated if a packet filter is fails to install.
 		Install_Failure,
+
+		## Generated when a notice takes too long to compile.
+		Too_Long_To_Compile_Filter
 	};

 	## The record type defining columns to be logged in the packet filter
@ -42,83 +47,248 @@ export {
 		success: bool  &log &default=T;
 	};

-	## By default, Bro will examine all packets. If this is set to false,
-	## it will dynamically build a BPF filter that only select protocols
-	## for which the user has loaded a corresponding analysis script.
-	## The latter used to be default for Bro versions < 2.0. That has now
-	## changed however to enable port-independent protocol analysis.
-	const all_packets = T &redef;
+	## The BPF filter that is used by default to define what traffic should
+	## be captured.  Filters defined in :bro:id:`restrict_filters` will still
+	## be applied to reduce the captured traffic.
+	const default_capture_filter = "ip or not ip" &redef;

 	## Filter string which is unconditionally or'ed to the beginning of every
 	## dynamically built filter.
 	const unrestricted_filter = "" &redef;

+	## Filter string which is unconditionally and'ed to the beginning of every
+	## dynamically built filter.  This is mostly used when a custom filter is being
+	## used but MPLS or VLAN tags are on the traffic.
+	const restricted_filter = "" &redef;
+
+	## The maximum amount of time that you'd like to allow for BPF filters to compile.
+	## If this time is exceeded, compensation measures may be taken by the framework
+	## to reduce the filter size.  This threshold being crossed also results in
+	## the :bro:see:`PacketFilter::Too_Long_To_Compile_Filter` notice.
+	const max_filter_compile_time = 100msec &redef;
+
+	## Install a BPF filter to exclude some traffic.  The filter should positively
+	## match what is to be excluded, it will be wrapped in a "not".
+	##
+	## filter_id: An arbitrary string that can be used to identify
+	##            the filter.
+	##
+	## filter: A BPF expression of traffic that should be excluded.
+	##
+	## Returns: A boolean value to indicate if the filter was successfully
+	##          installed or not.
+	global exclude: function(filter_id: string, filter: string): bool;
+
+	## Install a temporary filter to traffic which should not be passed through
+	## the BPF filter.  The filter should match the traffic you don't want
+	## to see (it will be wrapped in a "not" condition).
+	##
+	## filter_id: An arbitrary string that can be used to identify
+	##            the filter.
+	##
+	## filter: A BPF expression of traffic that should be excluded.
+	##
+	## length: The duration for which this filter should be put in place.
+	##
+	## Returns: A boolean value to indicate if the filter was successfully
+	##          installed or not.
+	global exclude_for: function(filter_id: string, filter: string, span: interval): bool;
+
 	## Call this function to build and install a new dynamically built
 	## packet filter.
-	global install: function();
+	global install: function(): bool;
+
+	## A data structure to represent filter generating plugins.
+	type FilterPlugin: record {
+		## A function that is directly called when generating the complete filter.
+		func : function();
+	};
+
+	## API function to register a new plugin for dynamic restriction filters.
+	global register_filter_plugin: function(fp: FilterPlugin);
+
+	## Enables the old filtering approach of "only watch common ports for
+	## analyzed protocols".
+        ##
+	## Unless you know what you are doing, leave this set to F.
+	const enable_auto_protocol_capture_filters = F &redef;

 	## This is where the default packet filter is stored and it should not
 	## normally be modified by users.
-	global default_filter = "<not set yet>";
+	global current_filter = "<not set yet>";
 }

+global dynamic_restrict_filters: table[string] of string = {};
+
+# Track if a filter is currently building so functions that would ultimately
+# install a filter immediately can still be used but they won't try to build or
+# install the filter.
+global currently_building = F;
+
+# Internal tracking for if the the filter being built has possibly been changed.
+global filter_changed = F;
+
+global filter_plugins: set[FilterPlugin] = {};
+
 redef enum PcapFilterID += {
 	DefaultPcapFilter,
+	FilterTester,
 };

-function combine_filters(lfilter: string, rfilter: string, op: string): string
+function test_filter(filter: string): bool
 	{
-	if ( lfilter == "" && rfilter == "" )
-		return "";
-	else if ( lfilter == "" )
-		return rfilter;
-	else if ( rfilter == "" )
-		return lfilter;
-	else
-		return fmt("(%s) %s (%s)", lfilter, op, rfilter);
+	if ( ! precompile_pcap_filter(FilterTester, filter) )
+		{
+		# The given filter was invalid
+		# TODO: generate a notice.
+		return F;
+		}
+	return T;
 	}

-function build_default_filter(): string
+# This tracks any changes for filtering mechanisms that play along nice
+# and set filter_changed to T.
+event filter_change_tracking()
+	{
+	if ( filter_changed )
+		install();
+
+	schedule 5min { filter_change_tracking() };
+	}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(PacketFilter::LOG, [$columns=Info]);
+
+	# Preverify the capture and restrict filters to give more granular failure messages.
+	for ( id in capture_filters )
+		{
+		if ( ! test_filter(capture_filters[id]) )
+			Reporter::fatal(fmt("Invalid capture_filter named '%s' - '%s'", id, capture_filters[id]));
+		}
+
+	for ( id in restrict_filters )
+		{
+		if ( ! test_filter(restrict_filters[id]) )
+			Reporter::fatal(fmt("Invalid restrict filter named '%s' - '%s'", id, restrict_filters[id]));
+		}
+	}
+
+event bro_init() &priority=-5
+	{
+	install();
+
+	event filter_change_tracking();
+	}
+
+function register_filter_plugin(fp: FilterPlugin)
+	{
+	add filter_plugins[fp];
+	}
+
+event remove_dynamic_filter(filter_id: string)
+	{
+	if ( filter_id in dynamic_restrict_filters )
+		{
+		delete dynamic_restrict_filters[filter_id];
+		install();
+		}
+	}
+
+function exclude(filter_id: string, filter: string): bool
+	{
+	if ( ! test_filter(filter) )
+		return F;
+
+	dynamic_restrict_filters[filter_id] = filter;
+	install();
+	return T;
+	}
+
+function exclude_for(filter_id: string, filter: string, span: interval): bool
+	{
+	if ( exclude(filter_id, filter) )
+		{
+		schedule span { remove_dynamic_filter(filter_id) };
+		return T;
+		}
+	return F;
+	}
+
+function build(): string
 	{
 	if ( cmd_line_bpf_filter != "" )
 		# Return what the user specified on the command line;
 		return cmd_line_bpf_filter;

-	if ( all_packets )
-		# Return an "always true" filter.
-		return "ip or not ip";
+	currently_building = T;

-	# Build filter dynamically.
+	# Generate all of the plugin based filters.
+	for ( plugin in filter_plugins )
+		{
+		plugin$func();
+		}

-	# First the capture_filter.
 	local cfilter = "";
-	for ( id in capture_filters )
-		cfilter = combine_filters(cfilter, capture_filters[id], "or");
+	if ( |capture_filters| == 0 && ! enable_auto_protocol_capture_filters )
+		cfilter = default_capture_filter;

-	# Then the restrict_filter.
+	for ( id in capture_filters )
+		cfilter = combine_filters(cfilter, "or", capture_filters[id]);
+
+	if ( enable_auto_protocol_capture_filters )
+		cfilter = combine_filters(cfilter, "or", Analyzer::get_bpf());
+
+	# Apply the restriction filters.
 	local rfilter = "";
 	for ( id in restrict_filters )
-		rfilter = combine_filters(rfilter, restrict_filters[id], "and");
+		rfilter = combine_filters(rfilter, "and", restrict_filters[id]);
+
+	# Apply the dynamic restriction filters.
+	for ( filt in dynamic_restrict_filters )
+		rfilter = combine_filters(rfilter, "and", string_cat("not (", dynamic_restrict_filters[filt], ")"));

 	# Finally, join them into one filter.
-	local filter = combine_filters(rfilter, cfilter, "and");
-	if ( unrestricted_filter != "" )
-		filter = combine_filters(unrestricted_filter, filter, "or");
+	local filter = combine_filters(cfilter, "and", rfilter);

+	if ( unrestricted_filter != "" )
+		filter = combine_filters(unrestricted_filter, "or", filter);
+	if ( restricted_filter != "" )
+		filter = combine_filters(restricted_filter, "and", filter);
+
+	currently_building = F;
 	return filter;
 	}

-function install()
+function install(): bool
 	{
-	default_filter = build_default_filter();
+	if ( currently_building )
+		return F;

-	if ( ! precompile_pcap_filter(DefaultPcapFilter, default_filter) )
+	local tmp_filter = build();
+
+	# No need to proceed if the filter hasn't changed.
+	if ( tmp_filter == current_filter )
+		return F;
+
+	local ts = current_time();
+	if ( ! precompile_pcap_filter(DefaultPcapFilter, tmp_filter) )
 		{
 		NOTICE([$note=Compile_Failure,
 		        $msg=fmt("Compiling packet filter failed"),
-		        $sub=default_filter]);
-		Reporter::fatal(fmt("Bad pcap filter '%s'", default_filter));
+		        $sub=tmp_filter]);
+		if ( network_time() == 0.0 )
+			Reporter::fatal(fmt("Bad pcap filter '%s'", tmp_filter));
+		else
+			Reporter::warning(fmt("Bad pcap filter '%s'", tmp_filter));
 		}
+	local diff = current_time()-ts;
+	if ( diff > max_filter_compile_time )
+		NOTICE([$note=Too_Long_To_Compile_Filter,
+		        $msg=fmt("A BPF filter is taking longer than %0.1f seconds to compile", diff)]);
+
+	# Set it to the current filter if it passed precompiling
+	current_filter = tmp_filter;

 	# Do an audit log for the packet filter.
 	local info: Info;
@ -129,7 +299,7 @@ function install()
 		info$ts = current_time();
 		info$init = T;
 		}
-	info$filter = default_filter;
+	info$filter = current_filter;

 	if ( ! install_pcap_filter(DefaultPcapFilter) )
 		{
@ -137,15 +307,13 @@ function install()
 		info$success = F;
 		NOTICE([$note=Install_Failure,
 		        $msg=fmt("Installing packet filter failed"),
-		        $sub=default_filter]);
+		        $sub=current_filter]);
 		}

 	if ( reading_live_traffic() || reading_traces() )
 		Log::write(PacketFilter::LOG, info);
-	}

-event bro_init() &priority=10
-	{
-	Log::create_stream(PacketFilter::LOG, [$columns=Info]);
-	PacketFilter::install();
+	# Update the filter change tracking
+	filter_changed = F;
+	return T;
 	}
--- a/scripts/base/frameworks/packet-filter/netstats.bro
+++ b/scripts/base/frameworks/packet-filter/netstats.bro
@ -13,7 +13,7 @@ export {
 	};
 	
 	## This is the interval between individual statistics collection.
-	const stats_collection_interval = 10secs;
+	const stats_collection_interval = 5min;
 }

 event net_stats_update(last_stat: NetStats)
--- a/scripts/base/frameworks/packet-filter/utils.bro
+++ b/scripts/base/frameworks/packet-filter/utils.bro
@ -0,0 +1,58 @@
+module PacketFilter;
+
+export {
+	## Takes a :bro:type:`port` and returns a BPF expression which will
+	## match the port.
+	##
+	## p: The port.
+	##
+	## Returns: A valid BPF filter string for matching the port.
+	global port_to_bpf: function(p: port): string;
+
+	## Create a BPF filter to sample IPv4 and IPv6 traffic.
+	##
+	## num_parts: The number of parts the traffic should be split into.
+	##
+	## this_part: The part of the traffic this filter will accept.  0-based.
+	global sampling_filter: function(num_parts: count, this_part: count): string;
+
+	## Combines two valid BPF filter strings with a string based operator
+	## to form a new filter.
+	##
+	## lfilter: Filter which will go on the left side.
+	##
+	## op: Operation being applied (typically "or" or "and").
+	##
+	## rfilter: Filter which will go on the right side.
+	##
+	## Returns: A new string representing the two filters combined with
+	##          the operator.  Either filter being an empty string will
+	##          still result in a valid filter.
+	global combine_filters: function(lfilter: string, op: string, rfilter: string): string;
+}
+
+function port_to_bpf(p: port): string
+	{
+	local tp = get_port_transport_proto(p);
+	return cat(tp, " and ", fmt("port %d", p));
+	}
+
+function combine_filters(lfilter: string,  op: string, rfilter: string): string
+	{
+	if ( lfilter == "" && rfilter == "" )
+		return "";
+	else if ( lfilter == "" )
+		return rfilter;
+	else if ( rfilter == "" )
+		return lfilter;
+	else
+		return fmt("(%s) %s (%s)", lfilter, op, rfilter);
+	}
+
+function sampling_filter(num_parts: count, this_part: count): string
+	{
+	local v4_filter = fmt("ip and ((ip[14:2]+ip[18:2]) - (%d*((ip[14:2]+ip[18:2])/%d)) == %d)", num_parts, num_parts, this_part);
+	# TODO: this is probably a fairly suboptimal filter, but it should work for now.
+	local v6_filter = fmt("ip6 and ((ip6[22:2]+ip6[38:2]) - (%d*((ip6[22:2]+ip6[38:2])/%d)) == %d)", num_parts, num_parts, this_part);
+	return combine_filters(v4_filter, "or", v6_filter);
+	}
--- a/scripts/base/init-bare.bro
+++ b/scripts/base/init-bare.bro
@ -766,19 +766,6 @@ global signature_files = "" &add_func = add_signature_file;
 ## ``p0f`` fingerprint file to use. Will be searched relative to ``BROPATH``.
 const passive_fingerprint_file = "base/misc/p0f.fp" &redef;

-# todo::testing to see if I can remove these without causing problems.
-#const ftp = 21/tcp;
-#const ssh = 22/tcp;
-#const telnet = 23/tcp;
-#const smtp = 25/tcp;
-#const domain = 53/tcp;	# note, doesn't include UDP version
-#const gopher = 70/tcp;
-#const finger = 79/tcp;
-#const http = 80/tcp;
-#const ident = 113/tcp;
-#const bgp = 179/tcp;
-#const rlogin = 513/tcp;
-
 # TCP values for :bro:see:`endpoint` *state* field.
 # todo::these should go into an enum to make them autodoc'able.
 const TCP_INACTIVE = 0;	##< Endpoint is still inactive.
--- a/scripts/base/init-default.bro
+++ b/scripts/base/init-default.bro
@ -46,6 +46,7 @@
@load base/protocols/ssh
@load base/protocols/ssl
@load base/protocols/syslog
+@load base/protocols/tunnels

@load base/files/hash
@load base/files/extract
--- a/scripts/base/protocols/dns/main.bro
+++ b/scripts/base/protocols/dns/main.bro
@ -122,14 +122,6 @@ redef record connection += {
 	dns_state: State &optional;
 };

-# DPD configuration.
-redef capture_filters += {
-	["dns"] = "port 53",
-	["mdns"] = "udp and port 5353",
-	["llmns"] = "udp and port 5355",
-	["netbios-ns"] = "udp port 137",
-};
-
 const ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
 redef likely_server_ports += { ports };

--- a/scripts/base/protocols/ftp/load.bro
+++ b/scripts/base/protocols/ftp/load.bro
@ -2,3 +2,5 @@
@load ./main
@load ./files
@load ./gridftp
+
+@load-sigs ./dpd.sig
--- a/scripts/base/protocols/ftp/dpd.sig
+++ b/scripts/base/protocols/ftp/dpd.sig
@ -0,0 +1,15 @@
+signature dpd_ftp_client {
+  ip-proto == tcp
+  payload /(|.*[\n\r]) *[uU][sS][eE][rR] /
+  tcp-state originator
+}
+
+# Match for server greeting (220, 120) and for login or passwd
+# required (230, 331).
+signature dpd_ftp_server {
+  ip-proto == tcp
+  payload /[\n\r ]*(120|220)[^0-9].*[\n\r] *(230|331)[^0-9]/
+  tcp-state responder
+  requires-reverse-signature dpd_ftp_client
+  enable "ftp"
+}
--- a/scripts/base/protocols/ftp/main.bro
+++ b/scripts/base/protocols/ftp/main.bro
@ -110,21 +110,18 @@ redef record connection += {
 	ftp_data_reuse: bool &default=F;
 };

-# Configure DPD
-redef capture_filters += { ["ftp"] = "port 21 and port 2811" };
-
 const ports = { 21/tcp, 2811/tcp };
 redef likely_server_ports += { ports };

-# Establish the variable for tracking expected connections.
-global ftp_data_expected: table[addr, port] of Info &read_expire=5mins;
-
 event bro_init() &priority=5
 	{
 	Log::create_stream(FTP::LOG, [$columns=Info, $ev=log_ftp]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_FTP, ports);
 	}

+# Establish the variable for tracking expected connections.
+global ftp_data_expected: table[addr, port] of Info &read_expire=5mins;
+
 ## A set of commands where the argument can be expected to refer
 ## to a file or directory.
 const file_cmds = {
--- a/scripts/base/protocols/http/load.bro
+++ b/scripts/base/protocols/http/load.bro
@ -2,3 +2,5 @@
@load ./entities
@load ./utils
@load ./files
+
+@load-sigs ./dpd.sig
--- a/scripts/base/protocols/http/dpd.sig
+++ b/scripts/base/protocols/http/dpd.sig
@ -0,0 +1,13 @@
+signature dpd_http_client {
+  ip-proto == tcp
+  payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/
+  tcp-state originator
+}
+
+signature dpd_http_server {
+  ip-proto == tcp
+  payload /^HTTP\/[0-9]/
+  tcp-state responder
+  requires-reverse-signature dpd_http_client
+  enable "http"
+}
--- a/scripts/base/protocols/http/main.bro
+++ b/scripts/base/protocols/http/main.bro
@ -123,19 +123,12 @@ redef record connection += {
 	http_state:  State &optional;
 };

-# DPD configuration.
-redef capture_filters +=  {
-	["http"] = "tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888)"
-};
-
 const ports = {
 	80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3128/tcp,
 	8000/tcp, 8080/tcp, 8888/tcp,
 };
-
 redef likely_server_ports += { ports };

-
 # Initialize the HTTP logging stream and ports.
 event bro_init() &priority=5
 	{
--- a/scripts/base/protocols/irc/load.bro
+++ b/scripts/base/protocols/irc/load.bro
@ -1,3 +1,5 @@
@load ./main
@load ./dcc-send
@load ./files
+
+@load-sigs ./dpd.sig
--- a/scripts/base/protocols/irc/dpd.sig
+++ b/scripts/base/protocols/irc/dpd.sig
@ -0,0 +1,33 @@
+signature irc_client1 {
+  ip-proto == tcp
+  payload /(|.*[\r\n]) *[Uu][Ss][Ee][Rr] +.+[\n\r]+ *[Nn][Ii][Cc][Kk] +.*[\r\n]/
+  requires-reverse-signature irc_server_reply
+  tcp-state originator
+  enable "irc"
+}
+
+signature irc_client2 {
+  ip-proto == tcp
+  payload /(|.*[\r\n]) *[Nn][Ii][Cc][Kk] +.+[\r\n]+ *[Uu][Ss][Ee][Rr] +.+[\r\n]/
+  requires-reverse-signature irc_server_reply
+  tcp-state originator
+  enable "irc"
+}
+
+signature irc_server_reply {
+  ip-proto == tcp
+  payload /^(|.*[\n\r])(:[^ \n\r]+ )?[0-9][0-9][0-9] /
+  tcp-state responder
+}
+
+signature irc_server_to_server1 {
+  ip-proto == tcp
+  payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/
+}
+
+signature irc_server_to_server2 {
+  ip-proto == tcp
+  payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/
+  requires-reverse-signature irc_server_to_server1
+  enable "irc"
+}
--- a/scripts/base/protocols/irc/main.bro
+++ b/scripts/base/protocols/irc/main.bro
@ -38,13 +38,6 @@ redef record connection += {
 	irc:  Info &optional;
 };

-# Some common IRC ports.
-redef capture_filters += { ["irc-6666"] = "port 6666" };
-redef capture_filters += { ["irc-6667"] = "port 6667" };
-redef capture_filters += { ["irc-6668"] = "port 6668" };
-redef capture_filters += { ["irc-6669"] = "port 6669" };
-
-# DPD configuration.
 const ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp };
 redef likely_server_ports += { ports };

--- a/scripts/base/protocols/modbus/main.bro
+++ b/scripts/base/protocols/modbus/main.bro
@ -29,9 +29,6 @@ redef record connection += {
 	modbus: Info &optional;
 };

-# Configure DPD and the packet filter.
-redef capture_filters += { ["modbus"] = "tcp port 502" };
-
 const ports = { 502/tcp };
 redef likely_server_ports += { ports };

--- a/scripts/base/protocols/smtp/load.bro
+++ b/scripts/base/protocols/smtp/load.bro
@ -1,3 +1,5 @@
@load ./main
@load ./entities
@load ./files
+
+@load-sigs ./dpd.sig
--- a/scripts/base/protocols/smtp/dpd.sig
+++ b/scripts/base/protocols/smtp/dpd.sig
@ -0,0 +1,13 @@
+signature dpd_smtp_client {
+  ip-proto == tcp
+  payload /(|.*[\n\r])[[:space:]]*([hH][eE][lL][oO]|[eE][hH][lL][oO])/
+  requires-reverse-signature dpd_smtp_server
+  enable "smtp"
+  tcp-state originator
+}
+
+signature dpd_smtp_server {
+  ip-proto == tcp
+  payload /^[[:space:]]*220[[:space:]-]/
+  tcp-state responder
+}
--- a/scripts/base/protocols/smtp/main.bro
+++ b/scripts/base/protocols/smtp/main.bro
@ -81,9 +81,6 @@ redef record connection += {
 	smtp_state: State &optional;
 };

-# Configure DPD
-redef capture_filters += { ["smtp"] = "tcp port 25 or tcp port 587" };
-
 const ports = { 25/tcp, 587/tcp };
 redef likely_server_ports += { ports };

--- a/scripts/base/protocols/socks/load.bro
+++ b/scripts/base/protocols/socks/load.bro
@ -1,2 +1,4 @@
@load ./consts
@load ./main
+
+@load-sigs ./dpd.sig
--- a/scripts/base/protocols/socks/dpd.sig
+++ b/scripts/base/protocols/socks/dpd.sig
@ -0,0 +1,48 @@
+signature dpd_socks4_client {
+	ip-proto == tcp
+	# '32' is a rather arbitrary max length for the user name.
+	payload /^\x04[\x01\x02].{0,32}\x00/
+	tcp-state originator
+}
+
+signature dpd_socks4_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks4_client
+	payload /^\x00[\x5a\x5b\x5c\x5d]/
+	tcp-state responder
+	enable "socks"
+}
+
+signature dpd_socks4_reverse_client {
+	ip-proto == tcp
+	# '32' is a rather arbitrary max length for the user name.
+	payload /^\x04[\x01\x02].{0,32}\x00/
+	tcp-state responder
+}
+
+signature dpd_socks4_reverse_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks4_reverse_client
+	payload /^\x00[\x5a\x5b\x5c\x5d]/
+	tcp-state originator
+	enable "socks"
+}
+
+signature dpd_socks5_client {
+	ip-proto == tcp
+	# Watch for a few authentication methods to reduce false positives.
+	payload /^\x05.[\x00\x01\x02]/
+	tcp-state originator
+}
+
+signature dpd_socks5_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks5_client
+	# Watch for a single authentication method to be chosen by the server or
+	# the server to indicate the no authentication is required.
+	payload /^\x05(\x00|\x01[\x00\x01\x02])/
+	tcp-state responder
+	enable "socks"
+}
+
+
--- a/scripts/base/protocols/socks/main.bro
+++ b/scripts/base/protocols/socks/main.bro
@ -47,10 +47,6 @@ redef record connection += {
 	socks: SOCKS::Info &optional;
 };

-# Configure DPD
-redef capture_filters += { ["socks"] = "tcp port 1080" };
-redef likely_server_ports += { 1080/tcp };
-
 function set_session(c: connection, version: count)
 	{
 	if ( ! c?$socks )
--- a/scripts/base/protocols/ssh/load.bro
+++ b/scripts/base/protocols/ssh/load.bro
@ -1 +1,3 @@
@load ./main
+
+@load-sigs ./dpd.sig
--- a/scripts/base/protocols/ssh/dpd.sig
+++ b/scripts/base/protocols/ssh/dpd.sig
@ -0,0 +1,13 @@
+signature dpd_ssh_client {
+  ip-proto == tcp
+  payload /^[sS][sS][hH]-/
+  requires-reverse-signature dpd_ssh_server
+  enable "ssh"
+  tcp-state originator
+}
+
+signature dpd_ssh_server {
+  ip-proto == tcp
+  payload /^[sS][sS][hH]-/
+  tcp-state responder
+}
--- a/scripts/base/protocols/ssh/main.bro
+++ b/scripts/base/protocols/ssh/main.bro
@ -70,17 +70,13 @@ export {
 	global log_ssh: event(rec: Info);
 }

-# Configure DPD and the packet filter
-
-const ports = { 22/tcp };
- 
-redef capture_filters += { ["ssh"] = "tcp port 22" };
-redef likely_server_ports += { ports };
-
 redef record connection += {
 	ssh: Info &optional;
 };

+const ports = { 22/tcp };
+redef likely_server_ports += { ports };
+
 event bro_init() &priority=5
 {
 	Log::create_stream(SSH::LOG, [$columns=Info, $ev=log_ssh]);
--- a/scripts/base/protocols/ssl/load.bro
+++ b/scripts/base/protocols/ssl/load.bro
@ -1,3 +1,5 @@
@load ./consts
@load ./main
@load ./mozilla-ca-list
+
+@load-sigs ./dpd.sig
--- a/scripts/base/protocols/ssl/dpd.sig
+++ b/scripts/base/protocols/ssl/dpd.sig
@ -0,0 +1,15 @@
+signature dpd_ssl_server {
+  ip-proto == tcp
+  # Server hello.
+  payload /^(\x16\x03[\x00\x01\x02]..\x02...\x03[\x00\x01\x02]|...?\x04..\x00\x02).*/
+  requires-reverse-signature dpd_ssl_client
+  enable "ssl"
+  tcp-state responder
+}
+
+signature dpd_ssl_client {
+  ip-proto == tcp
+  # Client hello.
+  payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/
+  tcp-state originator
+}
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@ -94,26 +94,10 @@ redef record Info += {
 		delay_tokens: set[string] &optional;
 };

-redef capture_filters += {
-	["ssl"] = "tcp port 443",
-	["nntps"] = "tcp port 563",
-	["imap4-ssl"] = "tcp port 585",
-	["sshell"] = "tcp port 614",
-	["ldaps"] = "tcp port 636",
-	["ftps-data"] = "tcp port 989",
-	["ftps"] = "tcp port 990",
-	["telnets"] = "tcp port 992",
-	["imaps"] = "tcp port 993",
-	["ircs"] = "tcp port 994",
-	["pop3s"] = "tcp port 995",
-	["xmpps"] = "tcp port 5223",
-};
-
 const ports = {
 	443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp,
 	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
-} &redef;
-
+};
 redef likely_server_ports += { ports };

 event bro_init() &priority=5
@ -154,7 +138,7 @@ function log_record(info: Info)
 			{
 			log_record(info);
 			}
-		timeout max_log_delay
+		timeout SSL::max_log_delay
 			{
 			Reporter::info(fmt("SSL delay tokens not released in time (%s tokens remaining)",
 			                   |info$delay_tokens|));
--- a/scripts/base/protocols/syslog/main.bro
+++ b/scripts/base/protocols/syslog/main.bro
@ -26,15 +26,13 @@ export {
 	};
 }

-redef capture_filters += { ["syslog"] = "port 514" };
-
-const ports = { 514/udp };
-redef likely_server_ports += { ports };
-
 redef record connection += {
 	syslog: Info &optional;
 };

+const ports = { 514/udp };
+redef likely_server_ports += { ports };
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(Syslog::LOG, [$columns=Info]);
--- a/scripts/base/protocols/tunnels/load.bro
+++ b/scripts/base/protocols/tunnels/load.bro
@ -0,0 +1 @@
+@load-sigs ./dpd.sig
--- a/scripts/base/protocols/tunnels/dpd.sig
+++ b/scripts/base/protocols/tunnels/dpd.sig
@ -0,0 +1,14 @@
+# Provide DPD signatures for tunneling protocols that otherwise
+# wouldn't be detected at all.
+
+signature dpd_ayiya {
+  ip-proto = udp
+  payload /^..\x11\x29/
+  enable "ayiya"
+}
+
+signature dpd_teredo {
+  ip-proto = udp
+  payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/
+  enable "teredo"
+}
--- a/scripts/policy/frameworks/packet-filter/shunt.bro
+++ b/scripts/policy/frameworks/packet-filter/shunt.bro
@ -0,0 +1,169 @@
+@load base/frameworks/notice
+@load base/frameworks/packet-filter
+
+module PacketFilter;
+
+export {
+	## The maximum number of BPF based shunts that Bro is allowed to perform.
+	const max_bpf_shunts = 100 &redef;
+
+	## Call this function to use BPF to shunt a connection (to prevent the
+	## data packets from reaching Bro).  For TCP connections, control packets
+	## are still allowed through so that Bro can continue logging the connection
+	## and it can stop shunting once the connection ends.
+	global shunt_conn: function(id: conn_id): bool;
+
+	## This function will use a BPF expresssion to shunt traffic between
+	## the two hosts given in the `conn_id` so that the traffic is never
+	## exposed to Bro's traffic processing.
+	global shunt_host_pair: function(id: conn_id): bool;
+
+	## Remove shunting for a host pair given as a `conn_id`.  The filter
+	## is not immediately removed.  It waits for the occassional filter
+	## update done by the `PacketFilter` framework.
+	global unshunt_host_pair: function(id: conn_id): bool;
+
+	## Performs the same function as the `unshunt_host_pair` function, but
+	## it forces an immediate filter update.
+	global force_unshunt_host_pair: function(id: conn_id): bool;
+
+	## Retrieve the currently shunted connections.
+	global current_shunted_conns: function(): set[conn_id];
+
+	## Retrieve the currently shunted host pairs.
+	global current_shunted_host_pairs: function(): set[conn_id];
+
+	redef enum Notice::Type += {
+		## Indicative that :bro:id:`max_bpf_shunts` connections are already
+		## being shunted with BPF filters and no more are allowed.
+		No_More_Conn_Shunts_Available,
+
+		## Limitations in BPF make shunting some connections with BPF impossible.
+		## This notice encompasses those various cases.
+		Cannot_BPF_Shunt_Conn,
+	};
+}
+
+global shunted_conns: set[conn_id];
+global shunted_host_pairs: set[conn_id];
+
+function shunt_filters()
+	{
+	# NOTE: this could wrongly match if a connection happens with the ports reversed.
+	local tcp_filter = "";
+	local udp_filter = "";
+	for ( id in shunted_conns )
+		{
+		local prot = get_port_transport_proto(id$resp_p);
+
+		local filt = fmt("host %s and port %d and host %s and port %d", id$orig_h, id$orig_p, id$resp_h, id$resp_p);
+		if ( prot == udp )
+			udp_filter = combine_filters(udp_filter, "and", filt);
+		else if ( prot == tcp )
+			tcp_filter = combine_filters(tcp_filter, "and", filt);
+		}
+	if ( tcp_filter != "" )
+		tcp_filter = combine_filters("tcp and tcp[tcpflags] & (tcp-syn|tcp-fin|tcp-rst) == 0", "and", tcp_filter);
+	local conn_shunt_filter = combine_filters(tcp_filter, "and", udp_filter);
+
+	local hp_shunt_filter = "";
+	for ( id in shunted_host_pairs )
+		hp_shunt_filter = combine_filters(hp_shunt_filter, "and", fmt("host %s and host %s", id$orig_h, id$resp_h));
+
+	local filter = combine_filters(conn_shunt_filter, "and", hp_shunt_filter);
+	if ( filter != "" )
+		PacketFilter::exclude("shunt_filters", filter);
+}
+
+event bro_init() &priority=5
+	{
+	register_filter_plugin([
+		$func()={ return shunt_filters(); }
+		]);
+	}
+
+function current_shunted_conns(): set[conn_id]
+	{
+	return shunted_conns;
+	}
+
+function current_shunted_host_pairs(): set[conn_id]
+	{
+	return shunted_host_pairs;
+	}
+
+function reached_max_shunts(): bool
+	{
+	if ( |shunted_conns| + |shunted_host_pairs| > max_bpf_shunts )
+		{
+		NOTICE([$note=No_More_Conn_Shunts_Available,
+		        $msg=fmt("%d BPF shunts are in place and no more will be added until space clears.", max_bpf_shunts)]);
+		return T;
+		}
+	else
+		return F;
+	}
+
+function shunt_host_pair(id: conn_id): bool
+	{
+	PacketFilter::filter_changed = T;
+
+	if ( reached_max_shunts() )
+		return F;
+
+	add shunted_host_pairs[id];
+	install();
+	return T;
+	}
+
+function unshunt_host_pair(id: conn_id): bool
+	{
+	PacketFilter::filter_changed = T;
+
+	if ( id in shunted_host_pairs )
+		{
+		delete shunted_host_pairs[id];
+		return T;
+		}
+	else
+		return F;
+	}
+
+function force_unshunt_host_pair(id: conn_id): bool
+	{
+	if ( unshunt_host_pair(id) )
+		{
+		install();
+		return T;
+		}
+	else
+		return F;
+	}
+
+function shunt_conn(id: conn_id): bool
+	{
+	if ( is_v6_addr(id$orig_h) )
+		{
+		NOTICE([$note=Cannot_BPF_Shunt_Conn,
+		        $msg="IPv6 connections can't be shunted with BPF due to limitations in BPF",
+		        $sub="ipv6_conn",
+		        $id=id, $identifier=cat(id)]);
+		return F;
+		}
+
+	if ( reached_max_shunts() )
+		return F;
+
+	PacketFilter::filter_changed = T;
+	add shunted_conns[id];
+	install();
+	return T;
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	# Don't rebuild the filter right away because the packet filter framework
+	# will check every few minutes and update the filter if things have changed.
+	if ( c$id in shunted_conns )
+		delete shunted_conns[c$id];
+	}
--- a/scripts/policy/misc/load-balancing.bro
+++ b/scripts/policy/misc/load-balancing.bro
@ -0,0 +1,132 @@
+##! This script implements the "Bro side" of several load balancing
+##! approaches for Bro clusters.
+
+@load base/frameworks/cluster
+@load base/frameworks/packet-filter
+
+module LoadBalancing;
+
+export {
+
+	type Method: enum {
+		## Apply BPF filters to each worker in a way that causes them to
+		## automatically flow balance traffic between them.
+		AUTO_BPF,
+		## Load balance traffic across the workers by making each one apply
+		## a restrict filter to only listen to a single MAC address.  This
+		## is a somewhat common deployment option for sites doing network
+		## based load balancing with MAC address rewriting and passing the
+		## traffic to a single interface.  Multiple MAC addresses will show
+		## up on the same interface and need filtered to a single address.
+		#MAC_ADDR_BPF,
+	};
+
+	## Defines the method of load balancing to use.
+	const method = AUTO_BPF &redef;
+
+	# Configure the cluster framework to enable the load balancing filter configuration.
+	#global send_filter: event(for_node: string, filter: string);
+	#global confirm_filter_installation: event(success: bool);
+
+	redef record Cluster::Node += {
+		## A BPF filter for load balancing traffic sniffed on a single interface
+		## across a number of processes.  In normal uses, this will be assigned
+		## dynamically by the manager and installed by the workers.
+		lb_filter: string &optional;
+	};
+}
+
+#redef Cluster::manager2worker_events += /LoadBalancing::send_filter/;
+#redef Cluster::worker2manager_events += /LoadBalancing::confirm_filter_installation/;
+
+@if ( Cluster::is_enabled() )
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+event bro_init() &priority=5
+	{
+	if ( method != AUTO_BPF )
+		return;
+
+	local worker_ip_interface: table[addr, string] of count = table();
+	for ( n in Cluster::nodes )
+		{
+		local this_node = Cluster::nodes[n];
+
+		# Only workers!
+		if ( this_node$node_type != Cluster::WORKER ||
+		     ! this_node?$interface )
+			next;
+
+		if ( [this_node$ip, this_node$interface] !in worker_ip_interface )
+			worker_ip_interface[this_node$ip, this_node$interface] = 0;
+		++worker_ip_interface[this_node$ip, this_node$interface];
+		}
+
+	# Now that we've counted up how many processes are running on an interface
+	# let's create the filters for each worker.
+	local lb_proc_track: table[addr, string] of count = table();
+	for ( no in Cluster::nodes )
+		{
+		local that_node = Cluster::nodes[no];
+		if ( that_node$node_type == Cluster::WORKER &&
+		     that_node?$interface && [that_node$ip, that_node$interface] in worker_ip_interface )
+			{
+			if ( [that_node$ip, that_node$interface] !in lb_proc_track )
+				lb_proc_track[that_node$ip, that_node$interface] = 0;
+
+			local this_lb_proc = lb_proc_track[that_node$ip, that_node$interface];
+			local total_lb_procs = worker_ip_interface[that_node$ip, that_node$interface];
+
+			++lb_proc_track[that_node$ip, that_node$interface];
+			if ( total_lb_procs > 1 )
+				{
+				that_node$lb_filter = PacketFilter::sample_filter(total_lb_procs, this_lb_proc);
+				Communication::nodes[no]$capture_filter = that_node$lb_filter;
+				}
+			}
+		}
+	}
+
+#event remote_connection_established(p: event_peer) &priority=-5
+#	{
+#	if ( is_remote_event() )
+#		return;
+#
+#	local for_node = p$descr;
+#	# Send the filter to the peer.
+#	if ( for_node in Cluster::nodes &&
+#	     Cluster::nodes[for_node]?$lb_filter )
+#		{
+#		local filter = Cluster::nodes[for_node]$lb_filter;
+#		event LoadBalancing::send_filter(for_node, filter);
+#		}
+#	}
+
+#event LoadBalancing::confirm_filter_installation(success: bool)
+#	{
+#	# This doesn't really matter yet since we aren't getting back a meaningful success response.
+#	}
+
+@endif
+
+
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+#event LoadBalancing::send_filter(for_node: string, filter: string)
+event remote_capture_filter(p: event_peer, filter: string)
+	{
+	#if ( for_node !in Cluster::nodes )
+	#	return;
+	#
+	#if ( Cluster::node == for_node )
+	#	{
+		restrict_filters["lb_filter"] = filter;
+		PacketFilter::install();
+		#event LoadBalancing::confirm_filter_installation(T);
+	#	}
+	}
+
+@endif
+
+@endif
--- a/scripts/policy/tuning/defaults/packet-fragments.bro
+++ b/scripts/policy/tuning/defaults/packet-fragments.bro
@ -3,7 +3,9 @@

 ## This normally isn't used because of the default open packet filter 
 ## but we set it anyway in case the user is using a packet filter.
-redef capture_filters += { ["frag"] = "(ip[6:2] & 0x3fff != 0) and tcp" };
+## Note: This was removed because the default model now is to have a wide
+##       open packet filter.
+#redef capture_filters += { ["frag"] = "(ip[6:2] & 0x3fff != 0) and tcp" };

 ## Shorten the fragment timeout from never expiring to expiring fragments after
 ## five minutes.
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@ -24,6 +24,7 @@
@load frameworks/intel/smtp.bro
@load frameworks/intel/ssl.bro
@load frameworks/intel/where-locations.bro
+@load frameworks/packet-filter/shunt.bro
@load frameworks/software/version-changes.bro
@load frameworks/software/vulnerable.bro
@load integration/barnyard2/__load__.bro
@ -35,6 +36,7 @@
@load misc/capture-loss.bro
@load misc/detect-traceroute/__load__.bro
@load misc/detect-traceroute/main.bro
+@load misc/load-balancing.bro
@load misc/loaded-scripts.bro
@load misc/profiling.bro
@load misc/scan.bro
--- a/src/analyzer/protocol/bittorrent/BitTorrentTracker.cc
+++ b/src/analyzer/protocol/bittorrent/BitTorrentTracker.cc
@ -22,7 +22,7 @@ static RecordType* bittorrent_benc_value;
 static TableType* bittorrent_benc_dir;

 BitTorrentTracker_Analyzer::BitTorrentTracker_Analyzer(Connection* c)
-: tcp::TCP_ApplicationAnalyzer("BITTORRENT", c)
+: tcp::TCP_ApplicationAnalyzer("BITTORRENTTRACKER", c)
 	{
 	if ( ! bt_tracker_headers )
 		{
--- a/src/analyzer/protocol/bittorrent/Plugin.cc
+++ b/src/analyzer/protocol/bittorrent/Plugin.cc
@ -7,6 +7,6 @@
 BRO_PLUGIN_BEGIN(Bro, BitTorrent)
 	BRO_PLUGIN_DESCRIPTION("BitTorrent Analyzer");
 	BRO_PLUGIN_ANALYZER("BitTorrent", bittorrent::BitTorrent_Analyzer);
-	BRO_PLUGIN_ANALYZER("BitTorrentTracker", bittorrent::BitTorrent_Analyzer);
+	BRO_PLUGIN_ANALYZER("BitTorrentTracker", bittorrent::BitTorrentTracker_Analyzer);
 	BRO_PLUGIN_BIF_FILE(events);
 BRO_PLUGIN_END
--- a/src/main.cc
+++ b/src/main.cc
@ -1114,6 +1114,7 @@ int main(int argc, char** argv)

 	reporter->ReportViaEvents(true);

+	// Drain the event queue here to support the protocols framework configuring DPM
 	mgr.Drain();

 	analyzer_mgr->DumpDebug();
--- a/testing/btest/Baseline/core.print-bpf-filters/conn.log
+++ b/testing/btest/Baseline/core.print-bpf-filters/conn.log
@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#open	2005-10-07-23-23-57
+#open	2013-07-08-20-05-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
-1128727435.450898	UWkUyAuUGXf	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945	(empty)
-#close	2005-10-07-23-23-57
+1278600802.069419	UWkUyAuUGXf	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.004152	9	3429	SF	-	0	ShADadfF	7	381	7	3801	(empty)
+#close	2013-07-08-20-05-18
--- a/testing/btest/Baseline/core.print-bpf-filters/output
+++ b/testing/btest/Baseline/core.print-bpf-filters/output
@ -3,38 +3,38 @@
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-11-06-00-53-09
+#open	2013-07-08-20-05-17
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1352163189.729807	-	ip or not ip	T	T
-#close	2012-11-06-00-53-09
+1373313917.926565	-	ip or not ip	T	T
+#close	2013-07-08-20-05-17
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-11-06-00-53-10
+#open	2013-07-08-20-05-18
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1352163190.114261	-	((((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 502)) or (tcp port 995)) or (tcp port 22)) or (port 21 and port 2811)) or (tcp port 25 or tcp port 587)) or (tcp port 614)) or (tcp port 990)) or (port 6667)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
-#close	2012-11-06-00-53-10
+1373313918.205206	-	port 42	T	T
+#close	2013-07-08-20-05-18
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-11-06-00-53-10
+#open	2013-07-08-20-05-18
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1352163190.484506	-	port 42	T	T
-#close	2012-11-06-00-53-10
+1373313918.491383	-	(vlan) and (ip or not ip)	T	T
+#close	2013-07-08-20-05-18
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-11-06-00-53-10
+#open	2013-07-08-20-05-18
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1352163190.855090	-	port 56730	T	T
-#close	2012-11-06-00-53-10
+1373313918.795264	-	((((((((((((udp and port 3544) or (udp and port 514)) or ((tcp and port 2811) or (tcp and port 21))) or (tcp and port 502)) or ((((tcp and port 6669) or (tcp and port 6666)) or (tcp and port 6668)) or (tcp and port 6667))) or (tcp and port 1080)) or ((udp and port 2152) or (udp and port 2123))) or ((((((((tcp and port 631) or (tcp and port 8888)) or (tcp and port 3128)) or (tcp and port 80)) or (tcp and port 1080)) or (tcp and port 8000)) or (tcp and port 81)) or (tcp and port 8080))) or (udp and port 5072)) or ((tcp and port 25) or (tcp and port 587))) or (((((((((((tcp and port 5223) or (tcp and port 585)) or (tcp and port 614)) or (tcp and port 993)) or (tcp and port 636)) or (tcp and port 989)) or (tcp and port 995)) or (tcp and port 443)) or (tcp and port 563)) or (tcp and port 990)) or (tcp and port 992))) or (((((udp and port 5355) or (tcp and port 53)) or (udp and port 5353)) or (udp and port 137)) or (udp and port 53))) or (tcp and port 22)	T	T
+#close	2013-07-08-20-05-18
--- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-06-10-19-50-56
+#open	2013-07-05-05-20-50
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@ -82,10 +82,11 @@ scripts/base/init-bare.bro
    scripts/base/frameworks/input/readers/sqlite.bro
  scripts/base/frameworks/analyzer/__load__.bro
    scripts/base/frameworks/analyzer/main.bro
+      scripts/base/frameworks/packet-filter/utils.bro
      build/scripts/base/bif/analyzer.bif.bro
  scripts/base/frameworks/file-analysis/__load__.bro
    scripts/base/frameworks/file-analysis/main.bro
      build/scripts/base/bif/file_analysis.bif.bro
 scripts/policy/misc/loaded-scripts.bro
  scripts/base/utils/paths.bro
-#close	2013-06-10-19-50-56
+#close	2013-07-05-05-20-50
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-06-10-19-50-57
+#open	2013-07-10-03-19-58
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@ -82,6 +82,7 @@ scripts/base/init-bare.bro
    scripts/base/frameworks/input/readers/sqlite.bro
  scripts/base/frameworks/analyzer/__load__.bro
    scripts/base/frameworks/analyzer/main.bro
+      scripts/base/frameworks/packet-filter/utils.bro
      build/scripts/base/bif/analyzer.bif.bro
  scripts/base/frameworks/file-analysis/__load__.bro
    scripts/base/frameworks/file-analysis/main.bro
@ -190,6 +191,7 @@ scripts/base/init-default.bro
  scripts/base/protocols/syslog/__load__.bro
    scripts/base/protocols/syslog/consts.bro
    scripts/base/protocols/syslog/main.bro
+  scripts/base/protocols/tunnels/__load__.bro
  scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-06-10-19-50-57
+#close	2013-07-10-03-19-58
--- a/testing/btest/core/print-bpf-filters.bro
+++ b/testing/btest/core/print-bpf-filters.bro
@ -1,10 +1,10 @@
-# @TEST-EXEC: bro -r $TRACES/empty.trace -e '' >output
+# @TEST-EXEC: bro -r $TRACES/empty.trace >output
 # @TEST-EXEC: cat packet_filter.log >>output
-# @TEST-EXEC: bro -r $TRACES/empty.trace PacketFilter::all_packets=F >>output
+# @TEST-EXEC: bro -r $TRACES/empty.trace -f "port 42" >>output
 # @TEST-EXEC: cat packet_filter.log >>output
-# @TEST-EXEC: bro -r $TRACES/empty.trace -f "port 42" -e '' >>output
+# @TEST-EXEC: bro -r $TRACES/mixed-vlan-mpls.trace PacketFilter::restricted_filter="vlan" >>output
 # @TEST-EXEC: cat packet_filter.log >>output
-# @TEST-EXEC: bro -r $TRACES/empty.trace -C -f "port 56730" -r $TRACES/mixed-vlan-mpls.trace >>output
+# @TEST-EXEC: bro -r $TRACES/empty.trace PacketFilter::enable_auto_protocol_capture_filters=T >>output
 # @TEST-EXEC: cat packet_filter.log >>output
 # @TEST-EXEC: btest-diff output
 # @TEST-EXEC: btest-diff conn.log
--- a/testing/btest/core/tunnels/teredo-known-services.test
+++ b/testing/btest/core/tunnels/teredo-known-services.test
@ -1,6 +1,6 @@
-# @TEST-EXEC: bro -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd protocols/conn/known-services Tunnel::delay_teredo_confirmation=T "Site::local_nets+={192.168.1.0/24}"
+# @TEST-EXEC: bro -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd base/protocols/tunnels protocols/conn/known-services Tunnel::delay_teredo_confirmation=T "Site::local_nets+={192.168.1.0/24}"
 # @TEST-EXEC: test ! -e known_services.log
-# @TEST-EXEC: bro -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd protocols/conn/known-services Tunnel::delay_teredo_confirmation=F "Site::local_nets+={192.168.1.0/24}"
+# @TEST-EXEC: bro -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd base/protocols/tunnels protocols/conn/known-services Tunnel::delay_teredo_confirmation=F "Site::local_nets+={192.168.1.0/24}"
 # @TEST-EXEC: btest-diff known_services.log

 # The first case using Tunnel::delay_teredo_confirmation=T doesn't produce
 @ -1 +1 @@
 .1-784
 .1-798