From 430cd9b1460e579c65d782ad4d1b5a08bf82b6a7 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 16 Feb 2012 11:14:57 -0500
Subject: [PATCH 1/8] Initial rework of packet filter framework.

- Large rework on packet filter framework to make many things easier.
   - Removed the PacketFilter::all_packets variable because it was confusing.
   - New variable (PacketFilter::enable_auto_protocol_capture_filters) to re-enable the old filtering model of only sniffing ports for analyzed protocols.
   - In progress plugin model for adding filtering mechanisms.
   - New default single item for capture_filters = { ["default"] = PacketFilter::default_capture_filter };
   - Mechanism and helper functions to "shunt" traffic with filters.

- Created the Protocols framework to assist with reworking how base protocol scripts are registered with DPD and other things.
   - Protocols framework creates BPF filters for registered analyzers. (if using PacketFilter framework in that mode).
---
 .../base/frameworks/communication/main.bro    |   5 +-
 .../frameworks/packet-filter/__load__.bro     |   2 +
 .../base/frameworks/packet-filter/main.bro    | 235 ++++++++++++++----
 .../base/frameworks/packet-filter/shunt.bro   |  74 ++++++
 .../base/frameworks/packet-filter/utils.bro   |  43 ++++
 .../base/frameworks/protocols/__load__.bro    |   1 +
 scripts/base/frameworks/protocols/main.bro    |  49 ++++
 scripts/base/init-bare.bro                    |  13 -
 scripts/base/init-default.bro                 |   1 +
 scripts/base/protocols/dns/main.bro           |  23 +-
 scripts/base/protocols/ftp/main.bro           |  12 +-
 scripts/base/protocols/http/main.bro          |  21 +-
 scripts/base/protocols/irc/main.bro           |  17 +-
 scripts/base/protocols/smtp/main.bro          |   9 +-
 scripts/base/protocols/ssh/main.bro           |  10 +-
 scripts/base/protocols/ssl/main.bro           |  35 +--
 scripts/base/protocols/syslog/main.bro        |  10 +-
 .../tuning/defaults/packet-fragments.bro      |   4 +-
 18 files changed, 403 insertions(+), 161 deletions(-)
 create mode 100644 scripts/base/frameworks/packet-filter/shunt.bro
 create mode 100644 scripts/base/frameworks/packet-filter/utils.bro
 create mode 100644 scripts/base/frameworks/protocols/__load__.bro
 create mode 100644 scripts/base/frameworks/protocols/main.bro

diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro
index 04772f57aa..9a14b2f49f 100644
--- a/scripts/base/frameworks/communication/main.bro
+++ b/scripts/base/frameworks/communication/main.bro
@@ -196,12 +196,9 @@ function setup_peer(p: event_peer, node: Node)
 		request_remote_events(p, node$events);
 		}
 
-	if ( node?$capture_filter )
+	if ( node?$capture_filter && node$capture_filter != "" )
 		{
 		local filter = node$capture_filter;
-		if ( filter == "" )
-			filter = PacketFilter::default_filter;
-
 		do_script_log(p, fmt("sending capture_filter: %s", filter));
 		send_capture_filter(p, filter);
 		}
diff --git a/scripts/base/frameworks/packet-filter/__load__.bro b/scripts/base/frameworks/packet-filter/__load__.bro
index 1d72e1ebe0..45c2488c00 100644
--- a/scripts/base/frameworks/packet-filter/__load__.bro
+++ b/scripts/base/frameworks/packet-filter/__load__.bro
@@ -1,2 +1,4 @@
+@load ./utils
 @load ./main
+@load ./shunt
 @load ./netstats
diff --git a/scripts/base/frameworks/packet-filter/main.bro b/scripts/base/frameworks/packet-filter/main.bro
index 16e3ff9789..7706c2669f 100644
--- a/scripts/base/frameworks/packet-filter/main.bro
+++ b/scripts/base/frameworks/packet-filter/main.bro
@@ -1,10 +1,11 @@
 ##! This script supports how Bro sets it's BPF capture filter.  By default
-##! Bro sets an unrestricted filter that allows all traffic.  If a filter
+##! Bro sets a capture filter that allows all traffic.  If a filter
 ##! is set on the command line, that filter takes precedence over the default
 ##! open filter and all filters defined in Bro scripts with the
 ##! :bro:id:`capture_filters` and :bro:id:`restrict_filters` variables.
 
 @load base/frameworks/notice
+@load base/frameworks/protocols
 
 module PacketFilter;
 
@@ -19,6 +20,9 @@ export {
 	
 		## This notice is generated if a packet filter is fails to install.
 		Install_Failure,
+		
+		## Generated when a notice takes too long to compile.
+		Too_Long_To_Compile_Filter
 	};
 
 	## The record type defining columns to be logged in the packet filter
@@ -41,94 +45,220 @@ export {
 		## Indicate if the filter was applied successfully.
 		success: bool  &log &default=T;
 	};
-
-	## By default, Bro will examine all packets. If this is set to false,
-	## it will dynamically build a BPF filter that only select protocols
-	## for which the user has loaded a corresponding analysis script.
-	## The latter used to be default for Bro versions < 2.0. That has now
-	## changed however to enable port-independent protocol analysis.
-	const all_packets = T &redef;
+	
+	## The BPF filter that is used by default to define what traffic should
+	## be captured.  Filters defined in :bro:id:`restrict_filters` will still
+	## be applied to reduce the captured traffic.
+	const default_capture_filter = "ip or not ip" &redef;
 	
 	## Filter string which is unconditionally or'ed to the beginning of every 
 	## dynamically built filter.
 	const unrestricted_filter = "" &redef;
 	
+	## The maximum amount of time that you'd like to allow for filters to compile.
+	## If this time is exceeded, compensation measures may be taken by the framework
+	## to reduce the filter size.  This threshold being crossed also results in
+	## the :bro:enum:`PacketFilter::Too_Long_To_Compile_Filter` notice.
+	const max_filter_compile_time = 100msec &redef;
+	
+	## Install a BPF filter to exclude some traffic.  The filter should positively
+	## match what is to be excluded, it will be wrapped in a "not".
+	##
+	## filter_id: A somewhat arbitrary string that can be used to identify 
+	##            the filter.
+	##
+	## filter: A BPF expression of traffic that should be excluded.
+	##
+	## Returns: A boolean value to indicate if the fitler was successfully
+	##          installed or not.
+	global exclude: function(filter_id: string, filter: string): bool;
+	
+	## Install a temporary filter to traffic which should not be passed through
+	## the BPF filter.  The filter should match the traffic you don't want
+	## to see (it will be wrapped in a "not" condition).
+	##
+	## filter_id: A somewhat arbitrary string that can be used to identify 
+	##            the filter.
+	##
+	## filter: A BPF expression of traffic that should be excluded.
+	##
+	## length: The duration for which this filter should be put in place.
+	##
+	## Returns: A boolean value to indicate if the filter was successfully 
+	##          installed or not.
+	global exclude_for: function(filter_id: string, filter: string, span: interval): bool;
+	
 	## Call this function to build and install a new dynamically built
 	## packet filter.
 	global install: function();
 	
+	## A data structure to represent filter generating factories.
+	type FilterFactory: record {
+		## A function that is directly called when generating the complete filter.
+		func : function();
+	};
+	
+	## API function to register a new factory for dynamic restriction filters.
+	global register_filter_factory: function(ff: FilterFactory);
+	
+	## Enables the old filtering approach of "only watch common ports for 
+	## analyzed protocols".
+	## Unless you know what you are doing, leave this set to F.
+	const enable_auto_protocol_capture_filters = F &redef;
+	
 	## This is where the default packet filter is stored and it should not 
 	## normally be modified by users.
-	global default_filter = "<not set yet>";
+	global current_filter = "<not set yet>";
 }
 
+global dynamic_restrict_filters: table[string] of string = {};
+
+# Set the default capture filter.
+redef capture_filters += { ["default"] = default_capture_filter };
+
+# Track if a filter is currenlty building so functions that would ultimately 
+# install a filter immediately can still be used buy they won't try to build or
+# install the filter.
+global currently_building = F;
+
+global filter_factories: set[FilterFactory] = {};
+
 redef enum PcapFilterID += {
 	DefaultPcapFilter,
+	FilterTester,
 };
 
-function combine_filters(lfilter: string, rfilter: string, op: string): string
+function test_filter(filter: string): bool
 	{
-	if ( lfilter == "" && rfilter == "" )
-		return "";
-	else if ( lfilter == "" )
-		return rfilter;
-	else if ( rfilter == "" )
-		return lfilter;
-	else
-		return fmt("(%s) %s (%s)", lfilter, op, rfilter);
+	if ( ! precompile_pcap_filter(FilterTester, filter) )
+		{
+		# The given filter was invalid
+		# TODO: generate a notice.
+		return F;
+		}
+	return T;
 	}
 
-function build_default_filter(): string
+event bro_init() &priority=6
+	{
+	Log::create_stream(PacketFilter::LOG, [$columns=Info]);
+	
+	# Preverify the capture and restrict filters to give more granular failure messages.
+	for ( id in capture_filters )
+		{
+		if ( ! test_filter(capture_filters[id]) )
+			Reporter::fatal(fmt("Invalid capture_filter named '%s' - '%s'", id, capture_filters[id]));
+		}
+		
+	for ( id in restrict_filters )
+		{
+		if ( ! test_filter(restrict_filters[id]) )
+			Reporter::fatal(fmt("Invalid restrict filter named '%s' - '%s'", id, restrict_filters[id]));
+		}
+	
+	install();
+	}
+	
+function register_filter_factory(ff: FilterFactory)
+	{
+	add filter_factories[ff];
+	}
+
+event remove_dynamic_filter(filter_id: string)
+	{
+	if ( filter_id in dynamic_restrict_filters )
+		{
+		delete dynamic_restrict_filters[filter_id];
+		install();
+		}
+	}
+
+function exclude(filter_id: string, filter: string): bool
+	{
+	if ( ! test_filter(filter) )
+		return F;
+		
+	dynamic_restrict_filters[filter_id] = filter;
+	install();
+	return T;
+	}
+
+function exclude_for(filter_id: string, filter: string, span: interval): bool
+	{
+	if ( exclude(filter_id, filter) )
+		{
+		schedule span { remove_dynamic_filter(filter_id) };
+		return T;
+		}
+	return F;
+	}
+
+function build(): string
 	{
 	if ( cmd_line_bpf_filter != "" )
 		# Return what the user specified on the command line;
 		return cmd_line_bpf_filter;
-
-	if ( all_packets )
-		{
-		# Return an "always true" filter.
-		if ( bro_has_ipv6() )
-			return "ip or not ip";
-		else
-			return "not ip6";
-		}
-
-	# Build filter dynamically.
 	
-	# First the capture_filter.
+	currently_building = T;
+	
+	# Install the default capture filter.
 	local cfilter = "";
-	for ( id in capture_filters )
-		cfilter = combine_filters(cfilter, capture_filters[id], "or");
+	
+	if ( |capture_filters| == 0 && ! enable_auto_protocol_capture_filters )
+		cfilter = default_capture_filter;
 		
-	# Then the restrict_filter.
+	for ( id in capture_filters )
+		cfilter = combine_filters(cfilter, "or", capture_filters[id]);
+	
+	if ( enable_auto_protocol_capture_filters )
+		cfilter = combine_filters(cfilter, "or", Protocols::to_bpf());
+	
+	# Apply the restriction filters.
 	local rfilter = "";
 	for ( id in restrict_filters )
-		rfilter = combine_filters(rfilter, restrict_filters[id], "and");
-		
+		rfilter = combine_filters(rfilter, "and", restrict_filters[id]);
+	
+	# Apply the dynamic restriction filters.
+	for ( filt in dynamic_restrict_filters )
+		rfilter = combine_filters(rfilter, "and", string_cat("not (", dynamic_restrict_filters[filt], ")"));
+
+	# Generate all of the plugin factory based filters.
+	for ( factory in filter_factories )
+		{
+		factory$func();
+		}
+	
 	# Finally, join them into one filter.
-	local filter = combine_filters(rfilter, cfilter, "and");
+	local filter = combine_filters(cfilter, "and", rfilter);
+	
 	if ( unrestricted_filter != "" )
-		filter = combine_filters(unrestricted_filter, filter, "or");
-	
-	# Exclude IPv6 if we don't support it.
-	if ( ! bro_has_ipv6() )
-		filter = combine_filters(filter, "not ip6", "and");
+		filter = combine_filters(unrestricted_filter, "or", filter);
 	
+	currently_building = F;
 	return filter;
 	}
 
 function install()
 	{
-	default_filter = build_default_filter();
-
-	if ( ! precompile_pcap_filter(DefaultPcapFilter, default_filter) )
+	if ( currently_building )
+		return;
+	
+	current_filter = build();
+	
+	#local ts = current_time();
+	if ( ! precompile_pcap_filter(DefaultPcapFilter, current_filter) )
 		{
 		NOTICE([$note=Compile_Failure, 
 		        $msg=fmt("Compiling packet filter failed"),
-		        $sub=default_filter]);
-		Reporter::fatal(fmt("Bad pcap filter '%s'", default_filter));
+		        $sub=current_filter]);
+		Reporter::fatal(fmt("Bad pcap filter '%s'", current_filter));
 		}
 	
+	#local diff = current_time()-ts;
+	#if ( diff > max_filter_compile_time )
+	#	NOTICE([$note=Too_Long_To_Compile_Filter,
+	#	        $msg=fmt("A BPF filter is taking longer than %0.6f seconds to compile", diff)]);
+	
 	# Do an audit log for the packet filter.
 	local info: Info;
 	info$ts = network_time();
@@ -138,7 +268,7 @@ function install()
 		info$ts = current_time();
 		info$init = T;
 		}
-	info$filter = default_filter;
+	info$filter = current_filter;
 	
 	if ( ! install_pcap_filter(DefaultPcapFilter) )
 		{
@@ -146,15 +276,10 @@ function install()
 		info$success = F;
 		NOTICE([$note=Install_Failure, 
 		        $msg=fmt("Installing packet filter failed"),
-		        $sub=default_filter]);
+		        $sub=current_filter]);
 		}
+		
 	
 	if ( reading_live_traffic() || reading_traces() )
 		Log::write(PacketFilter::LOG, info);
 	}
-
-event bro_init() &priority=10
-	{
-	Log::create_stream(PacketFilter::LOG, [$columns=Info]);
-	PacketFilter::install();
-	}
diff --git a/scripts/base/frameworks/packet-filter/shunt.bro b/scripts/base/frameworks/packet-filter/shunt.bro
new file mode 100644
index 0000000000..b001da0640
--- /dev/null
+++ b/scripts/base/frameworks/packet-filter/shunt.bro
@@ -0,0 +1,74 @@
+@load base/frameworks/notice
+
+module PacketFilter;
+
+export {
+	const max_bpf_shunts = 100 &redef;
+
+	global shunt_conn: function(id: conn_id): bool;
+	
+	redef enum Notice::Type += {
+		## Indicative that :bro:id:`max_bpf_shunts` connections are already
+		## being shunted with BPF filters and no more are allowed.
+		No_More_Conn_Shunts_Available,
+	};
+}
+
+global shunted_conns: set[conn_id];
+global shunted_conns_non_flag_tracking: set[conn_id];
+
+function conn_shunt_filters()
+	{
+	# TODO: this could wrongly match if a connection happens with the ports reversed.
+	local filter = "";
+	local ipv4_tcp_filter = "";
+	for ( id in shunted_conns )
+		{
+		local prot = get_port_transport_proto(id$resp_p);
+		
+		# TODO: add ipv6
+		#if ( prot == udp ) #|| is_ipv6_addr(id$orig_h) )
+		#	{
+		#	next;
+		#	shunt_for()
+		#	}
+		
+		if ( prot == tcp )
+			ipv4_tcp_filter = combine_filters(ipv4_tcp_filter, "and", fmt("host %s and port %d and host %s and port %d and %s", id$orig_h, id$orig_p, id$resp_h, id$resp_p, prot));
+		}
+	
+	ipv4_tcp_filter = combine_filters(ipv4_tcp_filter, "and", "tcp[tcpflags] & (tcp-syn|tcp-fin|tcp-rst) == 0");
+
+	if ( ipv4_tcp_filter == "" )
+		return;
+	PacketFilter::exclude("conn_shunt_filters", ipv4_tcp_filter);
+	}
+
+event bro_init() &priority=5
+	{
+	register_filter_factory([
+		$func()={ return conn_shunt_filters(); }
+		]);
+	}
+
+function shunt_conn(id: conn_id): bool
+	{
+	if ( |shunted_conns| + |shunted_conns_non_flag_tracking| > max_bpf_shunts )
+		{
+		NOTICE([$note=No_More_Conn_Shunts_Available,
+		        $msg=fmt("%d BPF shunts are in place and no more will be added until space clears.", max_bpf_shunts)]);
+		return F;
+		}
+	
+	add shunted_conns[id];
+	install();
+	return T;
+	}
+	
+event connection_state_remove(c: connection) &priority=-5
+	{
+	# Don't rebuild the filter right away because the packet filter framework will check every few minutes
+	# and update the filter if things have changed.
+	if ( c$id in shunted_conns )
+		delete shunted_conns[c$id];
+	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/packet-filter/utils.bro b/scripts/base/frameworks/packet-filter/utils.bro
new file mode 100644
index 0000000000..6ee2993050
--- /dev/null
+++ b/scripts/base/frameworks/packet-filter/utils.bro
@@ -0,0 +1,43 @@
+module PacketFilter;
+
+export {
+	## Takes a :bro:type:`port` and returns a BPF expression which will
+	## match the port.
+	##
+	## p: The port.
+	##
+	## Returns: A valid BPF filter string for matching the port.
+	global port_to_bpf: function(p: port): string;
+	
+	## Combines two valid BPF filter strings with a string based operator
+	## to form a new filter.
+	##
+	## lfilter: Filter which will go on the left side.
+	##
+	## op: Operation being applied (typically "or" or "and").
+	##
+	## rfilter: Filter which will go on the right side.
+	## 
+	## Returns: A new string representing the two filters combined with
+	##          the operator.  Either filter being an empty string will
+	##          still result in a valid filter.
+	global combine_filters: function(lfilter: string, op: string, rfilter: string): string;
+}
+
+function port_to_bpf(p: port): string
+	{
+	local tp = get_port_transport_proto(p);
+	return cat(tp, " and ", fmt("port %d", p));
+	}
+
+function combine_filters(lfilter: string,  op: string, rfilter: string): string
+	{
+	if ( lfilter == "" && rfilter == "" )
+		return "";
+	else if ( lfilter == "" )
+		return rfilter;
+	else if ( rfilter == "" )
+		return lfilter;
+	else
+		return fmt("(%s) %s (%s)", lfilter, op, rfilter);
+	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/protocols/__load__.bro b/scripts/base/frameworks/protocols/__load__.bro
new file mode 100644
index 0000000000..d551be57d3
--- /dev/null
+++ b/scripts/base/frameworks/protocols/__load__.bro
@@ -0,0 +1 @@
+@load ./main
\ No newline at end of file
diff --git a/scripts/base/frameworks/protocols/main.bro b/scripts/base/frameworks/protocols/main.bro
new file mode 100644
index 0000000000..8924c4c259
--- /dev/null
+++ b/scripts/base/frameworks/protocols/main.bro
@@ -0,0 +1,49 @@
+
+@load base/frameworks/packet-filter
+
+module Protocols;
+
+export {
+	const common_ports: table[string] of set[port] = {} &redef;
+
+	## Automatically creates a BPF filter for the specified protocol based
+	## on the data supplied for the protocol in the :bro:id:`common_ports`
+	## variable.
+	##
+	## protocol: A string representation for a protocol, e.g. "HTTP"
+	##
+	## Returns: BPF filter string.
+	global protocol_to_bpf: function(protocol: string): string;
+	
+	global to_bpf: function(): string;
+	
+	## Maps between human readable protocol identifiers (like "HTTP")
+	## and the internal Bro representation for an analyzer (like ANALYZER_HTTP).
+	## This is typically fully populated by the base protocol analyzer scripts.
+	const analyzer_map: table[string] of set[count] = {} &redef;
+}
+
+function protocol_to_bpf(protocol: string): string
+	{
+	# Return an empty string if an undefined protocol was given.
+	if ( protocol !in common_ports )
+		return "";
+	
+	local output = "";
+	for ( one_port in common_ports[protocol] )
+		output = PacketFilter::combine_filters(output, "or", PacketFilter::port_to_bpf(one_port));
+	return output;
+	}
+	
+function to_bpf(): string
+	{
+	local output = "";
+	for ( p in common_ports )
+		output = PacketFilter::combine_filters(output, "or", protocol_to_bpf(p));
+	return output;
+	}
+	
+	
+	
+	
+	
\ No newline at end of file
diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro
index 200947938d..81ebf27cc8 100644
--- a/scripts/base/init-bare.bro
+++ b/scripts/base/init-bare.bro
@@ -607,19 +607,6 @@ global signature_files = "" &add_func = add_signature_file;
 ## ``p0f`` fingerprint file to use. Will be searched relative to ``BRO_PATH``.
 const passive_fingerprint_file = "base/misc/p0f.fp" &redef;
 
-# todo::testing to see if I can remove these without causing problems.
-#const ftp = 21/tcp;
-#const ssh = 22/tcp;
-#const telnet = 23/tcp;
-#const smtp = 25/tcp;
-#const domain = 53/tcp;	# note, doesn't include UDP version
-#const gopher = 70/tcp;
-#const finger = 79/tcp;
-#const http = 80/tcp;
-#const ident = 113/tcp;
-#const bgp = 179/tcp;
-#const rlogin = 513/tcp;
-
 # TCP values for :bro:see:`endpoint` *state* field.
 # todo::these should go into an enum to make them autodoc'able.
 const TCP_INACTIVE = 0;	##< Endpoint is still inactive.
diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro
index 1cf125c3ab..b2e52cedfb 100644
--- a/scripts/base/init-default.bro
+++ b/scripts/base/init-default.bro
@@ -29,6 +29,7 @@
 @load base/frameworks/metrics
 @load base/frameworks/intel
 @load base/frameworks/reporter
+@load base/frameworks/protocols
 
 @load base/protocols/conn
 @load base/protocols/dns
diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro
index 56107fd02d..4fa4b229f7 100644
--- a/scripts/base/protocols/dns/main.bro
+++ b/scripts/base/protocols/dns/main.bro
@@ -1,6 +1,7 @@
 ##! Base DNS analysis script which tracks and logs DNS queries along with
 ##! their responses.
 
+@load base/frameworks/protocols
 @load ./consts
 
 module DNS;
@@ -109,23 +110,11 @@ redef record connection += {
 	dns_state: State &optional;
 };
 
-# DPD configuration.
-redef capture_filters += {
-	["dns"] = "port 53",
-	["mdns"] = "udp and port 5353",
-	["llmns"] = "udp and port 5355",
-	["netbios-ns"] = "udp port 137",
-};
-
-const dns_ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
-redef dpd_config += { [ANALYZER_DNS] = [$ports = dns_ports] };
-
-const dns_udp_ports = { 53/udp, 137/udp, 5353/udp, 5355/udp };
-const dns_tcp_ports = { 53/tcp };
-redef dpd_config += { [ANALYZER_DNS_UDP_BINPAC] = [$ports = dns_udp_ports] };
-redef dpd_config += { [ANALYZER_DNS_TCP_BINPAC] = [$ports = dns_tcp_ports] };
-
-redef likely_server_ports += { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
+# Not attaching ANALYZER_DNS_UDP_BINPAC and ANALYZER_DNS_TCP_BINPAC right now.
+global analyzers = { ANALYZER_DNS };
+redef Protocols::analyzer_map["DNS"] = analyzers;
+global ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
+redef Protocols::common_ports["DNS"] = ports;
 
 event bro_init() &priority=5
 	{
diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro
index 9e16804a32..80a0e0a796 100644
--- a/scripts/base/protocols/ftp/main.bro
+++ b/scripts/base/protocols/ftp/main.bro
@@ -3,10 +3,12 @@
 ##! will take on the full path that the client is at along with the requested 
 ##! file name.
 
+@load base/frameworks/protocols
 @load ./utils-commands
 @load base/utils/paths
 @load base/utils/numbers
 
+
 module FTP;
 
 export {
@@ -92,12 +94,10 @@ redef record connection += {
 	ftp: Info &optional;
 };
 
-# Configure DPD
-const ports = { 21/tcp } &redef;
-redef capture_filters += { ["ftp"] = "port 21" };
-redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] };
-
-redef likely_server_ports += { 21/tcp };
+global analyzers = { ANALYZER_FTP };
+redef Protocols::analyzer_map["FTP"] = analyzers;
+global ports = { 21/tcp };
+redef Protocols::common_ports["FTP"] = ports;
 
 # Establish the variable for tracking expected connections.
 global ftp_data_expected: table[addr, port] of Info &create_expire=5mins;
diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro
index 6571548145..dbfbe4b6ff 100644
--- a/scripts/base/protocols/http/main.bro
+++ b/scripts/base/protocols/http/main.bro
@@ -2,6 +2,7 @@
 ##! to log request/response pairs and all relevant metadata together in 
 ##! a single record.
 
+@load base/frameworks/protocols
 @load base/utils/numbers
 @load base/utils/files
 
@@ -110,17 +111,15 @@ event bro_init() &priority=5
 	Log::create_stream(HTTP::LOG, [$columns=Info, $ev=log_http]);
 	}
 
-# DPD configuration.
-const ports = {
-	80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3138/tcp,
-	8000/tcp, 8080/tcp, 8888/tcp,
-};
-redef dpd_config += { 
-	[[ANALYZER_HTTP, ANALYZER_HTTP_BINPAC]] = [$ports = ports],
-};
-redef capture_filters +=  {
-	["http"] = "tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888)"
-};
+
+global analyzers = { ANALYZER_HTTP, ANALYZER_HTTP_BINPAC };
+redef Protocols::analyzer_map["HTTP"] = analyzers;
+global ports = { 80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3138/tcp, 8000/tcp, 8080/tcp, 8888/tcp };
+redef Protocols::common_ports["HTTP"] = ports;
+
+#redef dpd_config += {
+#	[[ANALYZER_HTTP, ANALYZER_HTTP_BINPAC]] = [$ports = Protocols::common_ports["HTTP"]],
+#};
 
 redef likely_server_ports += { 
 	80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3138/tcp,
diff --git a/scripts/base/protocols/irc/main.bro b/scripts/base/protocols/irc/main.bro
index 2bf2a9bbb9..a59e5043f6 100644
--- a/scripts/base/protocols/irc/main.bro
+++ b/scripts/base/protocols/irc/main.bro
@@ -2,6 +2,8 @@
 ##! IRC commands along with the associated response and some additional 
 ##! metadata about the connection if it's available.
 
+@load base/frameworks/protocols
+
 module IRC;
 
 export {
@@ -36,17 +38,10 @@ redef record connection += {
 	irc:  Info &optional;
 };
 
-# Some common IRC ports.
-redef capture_filters += { ["irc-6666"] = "port 6666" };
-redef capture_filters += { ["irc-6667"] = "port 6667" };
-redef capture_filters += { ["irc-6668"] = "port 6668" };
-redef capture_filters += { ["irc-6669"] = "port 6669" };
-
-# DPD configuration.
-const irc_ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp };
-redef dpd_config += { [ANALYZER_IRC] = [$ports = irc_ports] };
-
-redef likely_server_ports += { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp };
+global analyzers = { ANALYZER_IRC };
+redef Protocols::analyzer_map["IRC"] = analyzers;
+global ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp, 7000/tcp };
+redef Protocols::common_ports["IRC"] = ports;
 
 event bro_init() &priority=5
 	{
diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro
index 513b85e342..5676878b18 100644
--- a/scripts/base/protocols/smtp/main.bro
+++ b/scripts/base/protocols/smtp/main.bro
@@ -1,4 +1,5 @@
 @load base/frameworks/notice
+@load base/frameworks/protocols
 @load base/utils/addrs
 @load base/utils/directions-and-hosts
 
@@ -66,11 +67,9 @@ redef record connection += {
 	smtp_state: State &optional;
 };
 
-# Configure DPD
-redef capture_filters += { ["smtp"] = "tcp port 25 or tcp port 587" };
-redef dpd_config += { [ANALYZER_SMTP] = [$ports = ports] };
-
-redef likely_server_ports += { 25/tcp, 587/tcp };
+global analyzers = { ANALYZER_SMTP };
+redef Protocols::analyzer_map["SMTP"] = analyzers;
+redef Protocols::common_ports["SMTP"] = ports;
 
 event bro_init() &priority=5
 	{
diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro
index 0d3439bb1f..7ca71ba9df 100644
--- a/scripts/base/protocols/ssh/main.bro
+++ b/scripts/base/protocols/ssh/main.bro
@@ -6,6 +6,7 @@
 ##! is not attempted if the connection size analyzer isn't enabled.
 
 @load base/frameworks/notice
+@load base/frameworks/protocols
 @load base/utils/site
 @load base/utils/thresholds
 @load base/utils/conn-ids
@@ -73,11 +74,10 @@ export {
 	global log_ssh: event(rec: Info);
 }
 
-# Configure DPD and the packet filter
-redef capture_filters += { ["ssh"] = "tcp port 22" };
-redef dpd_config += { [ANALYZER_SSH] = [$ports = set(22/tcp)] };
-
-redef likely_server_ports += { 22/tcp };
+global analyzers = { ANALYZER_SSH };
+redef Protocols::analyzer_map["SSH"] = analyzers;
+global ports = { 22/tcp };
+redef Protocols::common_ports["SSH"] = ports;
 
 redef record connection += {
 	ssh: Info &optional;
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index 0b280a6bcf..6a75e01735 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -1,6 +1,7 @@
 ##! Base SSL analysis script.  This script logs information about the SSL/TLS
 ##! handshaking and encryption establishment process.
 
+@load base/frameworks/protocols
 @load ./consts
 
 module SSL;
@@ -70,35 +71,13 @@ event bro_init() &priority=5
 	{
 	Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
 	}
+	
+global analyzers = { ANALYZER_SSL };
+redef Protocols::analyzer_map["SSL"] = analyzers;
+global ports = { 443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp,
+                 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp };
+redef Protocols::common_ports["SSL"] = ports;
 
-redef capture_filters += {
-	["ssl"] = "tcp port 443",
-	["nntps"] = "tcp port 563",
-	["imap4-ssl"] = "tcp port 585",
-	["sshell"] = "tcp port 614",
-	["ldaps"] = "tcp port 636",
-	["ftps-data"] = "tcp port 989",
-	["ftps"] = "tcp port 990",
-	["telnets"] = "tcp port 992",
-	["imaps"] = "tcp port 993",
-	["ircs"] = "tcp port 994",
-	["pop3s"] = "tcp port 995",
-	["xmpps"] = "tcp port 5223",
-};
-
-const ports = {
-	443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp,
-	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
-};
-
-redef dpd_config += {
-	[[ANALYZER_SSL]] = [$ports = ports]
-};
-
-redef likely_server_ports += {
-	443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp,
-	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
-};
 
 function set_session(c: connection)
 	{
diff --git a/scripts/base/protocols/syslog/main.bro b/scripts/base/protocols/syslog/main.bro
index 79f89d5e71..786e3668a4 100644
--- a/scripts/base/protocols/syslog/main.bro
+++ b/scripts/base/protocols/syslog/main.bro
@@ -1,6 +1,7 @@
 ##! Core script support for logging syslog messages.  This script represents 
 ##! one syslog message as one logged record.
 
+@load base/frameworks/protocols
 @load ./consts
 
 module Syslog;
@@ -24,11 +25,10 @@ export {
 	};
 }
 
-redef capture_filters += { ["syslog"] = "port 514" };
-const ports = { 514/udp } &redef;
-redef dpd_config += { [ANALYZER_SYSLOG_BINPAC] = [$ports = ports] };
-
-redef likely_server_ports += { 514/udp };
+global analyzers = { ANALYZER_SYSLOG_BINPAC };
+redef Protocols::analyzer_map["SYSLOG"] = analyzers;
+global ports = { 514/udp };
+redef Protocols::common_ports["SYSLOG"] = ports;
 
 redef record connection += {
 	syslog: Info &optional;
diff --git a/scripts/policy/tuning/defaults/packet-fragments.bro b/scripts/policy/tuning/defaults/packet-fragments.bro
index 30d7e23729..24b18d5917 100644
--- a/scripts/policy/tuning/defaults/packet-fragments.bro
+++ b/scripts/policy/tuning/defaults/packet-fragments.bro
@@ -3,7 +3,9 @@
 
 ## This normally isn't used because of the default open packet filter 
 ## but we set it anyway in case the user is using a packet filter.
-redef capture_filters += { ["frag"] = "(ip[6:2] & 0x3fff != 0) and tcp" };
+## Note: This was removed because the default model now is to have a wide
+##       open packet filter.
+#redef capture_filters += { ["frag"] = "(ip[6:2] & 0x3fff != 0) and tcp" };
 
 ## Shorten the fragment timeout from never expiring to expiring fragments after
 ## five minutes.

From e0086005f8ff52f99a11e26169b0a35434e610ee Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 25 Apr 2012 17:12:12 -0400
Subject: [PATCH 2/8] Checkpoint on the packet filter framework.

- Packet loss interval changed to 5 minutes by default.  Users
  were getting too many notices from this.

- BPF load balancing (ipv4 and ipv6).  This will tie in with
  upcoming BroControl support for configuring this.

- BPF based connection sampling.

- Small improvements to how and when filters are installed.
---
 .../frameworks/packet-filter/__load__.bro     |   1 +
 .../frameworks/packet-filter/load-balance.bro | 120 ++++++++++++++++
 .../base/frameworks/packet-filter/main.bro    |  54 +++++--
 .../frameworks/packet-filter/netstats.bro     |   2 +-
 .../base/frameworks/packet-filter/shunt.bro   | 136 +++++++++++++++---
 .../base/frameworks/packet-filter/utils.bro   |  17 ++-
 6 files changed, 296 insertions(+), 34 deletions(-)
 create mode 100644 scripts/base/frameworks/packet-filter/load-balance.bro

diff --git a/scripts/base/frameworks/packet-filter/__load__.bro b/scripts/base/frameworks/packet-filter/__load__.bro
index 45c2488c00..14da4e4893 100644
--- a/scripts/base/frameworks/packet-filter/__load__.bro
+++ b/scripts/base/frameworks/packet-filter/__load__.bro
@@ -1,4 +1,5 @@
 @load ./utils
 @load ./main
 @load ./shunt
+@load ./load-balance
 @load ./netstats
diff --git a/scripts/base/frameworks/packet-filter/load-balance.bro b/scripts/base/frameworks/packet-filter/load-balance.bro
new file mode 100644
index 0000000000..105a37b617
--- /dev/null
+++ b/scripts/base/frameworks/packet-filter/load-balance.bro
@@ -0,0 +1,120 @@
+##! This script implements an automated BPF based load balancing solution for Bro clusters.
+##! It is completely automated when multiple worker processes are configured for a single
+##! interface on a host.  One caveat is that in order for this script to work, your traffic 
+##! can't have any headers above the Ethernet header (vlan, mpls).
+
+@load base/frameworks/cluster
+@load base/frameworks/packet-filter
+
+module PacketFilter;
+
+export {
+	redef record Cluster::Node += {
+		## A BPF filter for load balancing traffic sniffed on a single interface
+		## across a number of processes.  In normal uses, this will be assigned 
+		## dynamically by the manager and installed by the workers.
+		lb_filter: string &optional;
+	};
+	
+	## Control if BPF based load balancing is enabled on cluster deployments.
+	const enable_BPF_load_balancing = F &redef;
+	
+	# Configure the cluster framework to enable the load balancing filter configuration.
+	#global send_filter: event(for_node: string, filter: string);
+	#global confirm_filter_installation: event(success: bool);
+}
+
+#redef Cluster::manager2worker_events += /LoadBalancing::send_filter/;
+#redef Cluster::worker2manager_events += /LoadBalancing::confirm_filter_installation/;
+
+@if ( Cluster::is_enabled() )
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+event bro_init() &priority=5
+	{
+	if ( ! enable_BPF_load_balancing )
+		return;
+	
+	local worker_ip_interface: table[addr, string] of count = table();
+	for ( n in Cluster::nodes )
+		{
+		local this_node = Cluster::nodes[n];
+	
+		# Only workers!
+		if ( this_node$node_type != Cluster::WORKER || 
+		     ! this_node?$interface )
+			next;
+		
+		if ( [this_node$ip, this_node$interface] !in worker_ip_interface )
+			worker_ip_interface[this_node$ip, this_node$interface] = 0;
+		++worker_ip_interface[this_node$ip, this_node$interface];
+		}
+
+	# Now that we've counted up how many processes are running on an interface
+	# let's create the filters for each worker.
+	local lb_proc_track: table[addr, string] of count = table();
+	for ( no in Cluster::nodes )
+		{
+		local that_node = Cluster::nodes[no];
+		if ( that_node$node_type == Cluster::WORKER &&
+		     that_node?$interface && [that_node$ip, that_node$interface] in worker_ip_interface )
+			{
+			if ( [that_node$ip, that_node$interface] !in lb_proc_track )
+				lb_proc_track[that_node$ip, that_node$interface] = 0;
+			
+			local this_lb_proc = lb_proc_track[that_node$ip, that_node$interface];
+			local total_lb_procs = worker_ip_interface[that_node$ip, that_node$interface];
+			
+			++lb_proc_track[that_node$ip, that_node$interface];
+			if ( total_lb_procs > 1 )
+				{
+				that_node$lb_filter = PacketFilter::sample_filter(total_lb_procs, this_lb_proc);
+				Communication::nodes[no]$capture_filter = that_node$lb_filter;
+				}
+			}
+		}
+	}
+
+#event remote_connection_established(p: event_peer) &priority=-5
+#	{
+#	if ( is_remote_event() )
+#		return;
+#	
+#	local for_node = p$descr;
+#	# Send the filter to the peer.
+#	if ( for_node in Cluster::nodes && 
+#	     Cluster::nodes[for_node]?$lb_filter )
+#		{
+#		local filter = Cluster::nodes[for_node]$lb_filter;
+#		event LoadBalancing::send_filter(for_node, filter);
+#		}
+#	}
+
+#event LoadBalancing::confirm_filter_installation(success: bool)
+#	{
+#	# This doesn't really matter yet since we aren't getting back a meaningful success response.
+#	}
+
+@endif
+
+
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+#event LoadBalancing::send_filter(for_node: string, filter: string)
+event remote_capture_filter(p: event_peer, filter: string)
+	{
+	#if ( for_node !in Cluster::nodes )
+	#	return;
+	#
+	#if ( Cluster::node == for_node )
+	#	{
+		restrict_filters["lb_filter"] = filter;
+		PacketFilter::install();
+		#event LoadBalancing::confirm_filter_installation(T);
+	#	}
+	}
+
+@endif
+
+@endif
diff --git a/scripts/base/frameworks/packet-filter/main.bro b/scripts/base/frameworks/packet-filter/main.bro
index 66a557f53d..6e839c9210 100644
--- a/scripts/base/frameworks/packet-filter/main.bro
+++ b/scripts/base/frameworks/packet-filter/main.bro
@@ -55,6 +55,11 @@ export {
 	## dynamically built filter.
 	const unrestricted_filter = "" &redef;
 	
+	## Filter string which is unconditionally and'ed to the beginning of every
+	## dynamically built filter.  This is mostly used when a custom filter is being
+	## used but MPLS or VLAN tags are on the traffic.
+	const restricted_filter = "" &redef;
+	
 	## The maximum amount of time that you'd like to allow for filters to compile.
 	## If this time is exceeded, compensation measures may be taken by the framework
 	## to reduce the filter size.  This threshold being crossed also results in
@@ -69,7 +74,7 @@ export {
 	##
 	## filter: A BPF expression of traffic that should be excluded.
 	##
-	## Returns: A boolean value to indicate if the fitler was successfully
+	## Returns: A boolean value to indicate if the filter was successfully
 	##          installed or not.
 	global exclude: function(filter_id: string, filter: string): bool;
 	
@@ -90,7 +95,7 @@ export {
 	
 	## Call this function to build and install a new dynamically built
 	## packet filter.
-	global install: function();
+	global install: function(): bool;
 	
 	## A data structure to represent filter generating factories.
 	type FilterFactory: record {
@@ -121,6 +126,9 @@ redef capture_filters += { ["default"] = default_capture_filter };
 # install the filter.
 global currently_building = F;
 
+# Internal tracking for if the the filter being built has possibly been changed.
+global filter_changed = F;
+
 global filter_factories: set[FilterFactory] = {};
 
 redef enum PcapFilterID += {
@@ -139,7 +147,17 @@ function test_filter(filter: string): bool
 	return T;
 	}
 
-event bro_init() &priority=6
+# This tracks any changes for filtering mechanisms that play along nice 
+# and set filter_changed to T.
+event filter_change_tracking()
+	{
+	if ( filter_changed )
+		install();
+	
+	schedule 5min { filter_change_tracking() };
+	}
+
+event bro_init() &priority=5
 	{
 	Log::create_stream(PacketFilter::LOG, [$columns=Info]);
 	
@@ -155,8 +173,13 @@ event bro_init() &priority=6
 		if ( ! test_filter(restrict_filters[id]) )
 			Reporter::fatal(fmt("Invalid restrict filter named '%s' - '%s'", id, restrict_filters[id]));
 		}
+	}
 	
+event bro_init() &priority=-5
+	{
 	install();
+	
+	event filter_change_tracking();
 	}
 	
 function register_filter_factory(ff: FilterFactory)
@@ -233,27 +256,35 @@ function build(): string
 	
 	if ( unrestricted_filter != "" )
 		filter = combine_filters(unrestricted_filter, "or", filter);
+	if ( restricted_filter != "" )
+		filter = combine_filters(restricted_filter, "and", filter);
 	
 	currently_building = F;
 	return filter;
 	}
 
-function install()
+function install(): bool
 	{
 	if ( currently_building )
-		return;
+		return F;
 	
-	current_filter = build();
+	local tmp_filter = build();
 	
 	#local ts = current_time();
-	if ( ! precompile_pcap_filter(DefaultPcapFilter, current_filter) )
+	if ( ! precompile_pcap_filter(DefaultPcapFilter, tmp_filter) )
 		{
 		NOTICE([$note=Compile_Failure,
 		        $msg=fmt("Compiling packet filter failed"),
-		        $sub=current_filter]);
-		Reporter::fatal(fmt("Bad pcap filter '%s'", current_filter));
+		        $sub=tmp_filter]);
+		if ( network_time() == 0.0 )
+			Reporter::fatal(fmt("Bad pcap filter '%s'", tmp_filter));
+		else
+			Reporter::warning(fmt("Bad pcap filter '%s'", tmp_filter));
 		}
 	
+	# Set it to the current filter if it passed precompiling
+	current_filter = tmp_filter;
+	
 	#local diff = current_time()-ts;
 	#if ( diff > max_filter_compile_time )
 	#	NOTICE([$note=Too_Long_To_Compile_Filter,
@@ -278,8 +309,11 @@ function install()
 		        $msg=fmt("Installing packet filter failed"),
 		        $sub=current_filter]);
 		}
-		
 	
 	if ( reading_live_traffic() || reading_traces() )
 		Log::write(PacketFilter::LOG, info);
+
+	# Update the filter change tracking
+	filter_changed = F;
+	return T;
 	}
diff --git a/scripts/base/frameworks/packet-filter/netstats.bro b/scripts/base/frameworks/packet-filter/netstats.bro
index 9fbaa5cd1d..b5ffe24f54 100644
--- a/scripts/base/frameworks/packet-filter/netstats.bro
+++ b/scripts/base/frameworks/packet-filter/netstats.bro
@@ -13,7 +13,7 @@ export {
 	};
 	
 	## This is the interval between individual statistics collection.
-	const stats_collection_interval = 10secs;
+	const stats_collection_interval = 5min;
 }
 
 event net_stats_update(last_stat: NetStats)
diff --git a/scripts/base/frameworks/packet-filter/shunt.bro b/scripts/base/frameworks/packet-filter/shunt.bro
index b001da0640..5527592642 100644
--- a/scripts/base/frameworks/packet-filter/shunt.bro
+++ b/scripts/base/frameworks/packet-filter/shunt.bro
@@ -3,45 +3,73 @@
 module PacketFilter;
 
 export {
+	## The maximum number of BPF based shunts that Bro is allowed to perform.
 	const max_bpf_shunts = 100 &redef;
-
+	
+	## Call this function to use BPF to shunt a connection (to prevent the
+	## data packets from reaching Bro).  For TCP connections, control packets 
+	## are still allowed through so that Bro can continue logging the connection
+	## and it can stop shunting once the connection ends.
 	global shunt_conn: function(id: conn_id): bool;
 	
+	## This function will use a BPF expresssion to shunt traffic between
+	## the two hosts given in the `conn_id` so that the traffic is never
+	## exposed to Bro's traffic processing.
+	global shunt_host_pair: function(id: conn_id): bool;
+	
+	## Remove shunting for a host pair given as a `conn_id`.  The filter
+	## is not immediately removed.  It waits for the occassional filter
+	## update done by the `PacketFilter` framework.
+	global unshunt_host_pair: function(id: conn_id): bool;
+	
+	## Performs the same function as the `unshunt_host_pair` function, but
+	## it forces an immediate filter update.
+	global force_unshunt_host_pair: function(id: conn_id): bool;
+	
+	## Retrieve the currently shunted connections.
+	global current_shunted_conns: function(): set[conn_id];
+	
+	## Retrieve the currently shunted host pairs.
+	global current_shunted_host_pairs: function(): set[conn_id];
+	
 	redef enum Notice::Type += {
 		## Indicative that :bro:id:`max_bpf_shunts` connections are already
 		## being shunted with BPF filters and no more are allowed.
 		No_More_Conn_Shunts_Available,
+		
+		## Limitations in BPF make shunting some connections with BPF impossible.
+		## This notice encompasses those various cases.
+		Cannot_BPF_Shunt_Conn,
 	};
 }
 
 global shunted_conns: set[conn_id];
-global shunted_conns_non_flag_tracking: set[conn_id];
+global shunted_host_pairs: set[conn_id];
 
 function conn_shunt_filters()
 	{
-	# TODO: this could wrongly match if a connection happens with the ports reversed.
-	local filter = "";
-	local ipv4_tcp_filter = "";
+	# NOTE: this could wrongly match if a connection happens with the ports reversed.
+	local tcp_filter = "tcp and tcp[tcpflags] & (tcp-syn|tcp-fin|tcp-rst) == 0";
+	local udp_filter = "";
 	for ( id in shunted_conns )
 		{
 		local prot = get_port_transport_proto(id$resp_p);
 		
-		# TODO: add ipv6
-		#if ( prot == udp ) #|| is_ipv6_addr(id$orig_h) )
-		#	{
-		#	next;
-		#	shunt_for()
-		#	}
-		
-		if ( prot == tcp )
-			ipv4_tcp_filter = combine_filters(ipv4_tcp_filter, "and", fmt("host %s and port %d and host %s and port %d and %s", id$orig_h, id$orig_p, id$resp_h, id$resp_p, prot));
+		local filt = fmt("host %s and port %d and host %s and port %d", id$orig_h, id$orig_p, id$resp_h, id$resp_p);
+		if ( prot == udp )
+			udp_filter = combine_filters(udp_filter, "and", filt);
+		else if ( prot == tcp )
+			tcp_filter = combine_filters(tcp_filter, "and", filt);
 		}
+	local conn_shunt_filter = combine_filters(tcp_filter, "and", udp_filter);
 	
-	ipv4_tcp_filter = combine_filters(ipv4_tcp_filter, "and", "tcp[tcpflags] & (tcp-syn|tcp-fin|tcp-rst) == 0");
+	for ( id in shunted_host_pairs )
+		{
+		local hp_filter = fmt("host %s and host %s", id$orig_h, id$resp_h);
+		
+		}
 
-	if ( ipv4_tcp_filter == "" )
-		return;
-	PacketFilter::exclude("conn_shunt_filters", ipv4_tcp_filter);
+	PacketFilter::exclude("conn_shunt_filters", conn_shunt_filter);
 	}
 
 event bro_init() &priority=5
@@ -51,15 +79,79 @@ event bro_init() &priority=5
 		]);
 	}
 
-function shunt_conn(id: conn_id): bool
+function current_shunted_conns(): set[conn_id]
 	{
-	if ( |shunted_conns| + |shunted_conns_non_flag_tracking| > max_bpf_shunts )
+	return shunted_conns;
+	}
+	
+function current_shunted_host_pairs(): set[conn_id]
+	{
+	return shunted_host_pairs;
+	}
+	
+function reached_max_shunts(): bool
+	{
+	if ( |shunted_conns| + |shunted_host_pairs| > max_bpf_shunts )
 		{
 		NOTICE([$note=No_More_Conn_Shunts_Available,
 		        $msg=fmt("%d BPF shunts are in place and no more will be added until space clears.", max_bpf_shunts)]);
+		return T;
+		}
+	else
+		return F;
+	}
+
+function shunt_host_pair(id: conn_id): bool
+	{
+	PacketFilter::filter_changed = T;
+	
+	if ( reached_max_shunts() )
+		return F;
+	
+	add shunted_host_pairs[id];
+	install();
+	return T;
+	}
+
+function unshunt_host_pair(id: conn_id): bool
+	{
+	PacketFilter::filter_changed = T;
+	
+	if ( id in shunted_host_pairs )
+		{
+		delete shunted_host_pairs[id];
+		return T;
+		}
+	else
+		return F;
+	}
+	
+function force_unshunt_host_pair(id: conn_id): bool
+	{
+	if ( unshunt_host_pair(id) )
+		{
+		install();
+		return T;
+		}
+	else
+		return F;
+	}
+
+function shunt_conn(id: conn_id): bool
+	{
+	if ( is_v6_addr(id$orig_h) )
+		{
+		NOTICE([$note=Cannot_BPF_Shunt_Conn,
+		        $msg="IPv6 connections can't be shunted with BPF due to limitations in BPF",
+		        $sub="ipv6_conn", 
+		        $id=id, $identifier=string_cat(id)]);
 		return F;
 		}
 	
+	if ( reached_max_shunts() )
+		return F;
+	
+	PacketFilter::filter_changed = T;
 	add shunted_conns[id];
 	install();
 	return T;
@@ -67,8 +159,8 @@ function shunt_conn(id: conn_id): bool
 	
 event connection_state_remove(c: connection) &priority=-5
 	{
-	# Don't rebuild the filter right away because the packet filter framework will check every few minutes
-	# and update the filter if things have changed.
+	# Don't rebuild the filter right away because the packet filter framework 
+	# will check every few minutes and update the filter if things have changed.
 	if ( c$id in shunted_conns )
 		delete shunted_conns[c$id];
 	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/packet-filter/utils.bro b/scripts/base/frameworks/packet-filter/utils.bro
index 6ee2993050..242d30e45a 100644
--- a/scripts/base/frameworks/packet-filter/utils.bro
+++ b/scripts/base/frameworks/packet-filter/utils.bro
@@ -9,6 +9,13 @@ export {
 	## Returns: A valid BPF filter string for matching the port.
 	global port_to_bpf: function(p: port): string;
 	
+	## Create a BPF filter to sample IPv4 and IPv6 traffic.  
+	##
+	## num_parts: The number of parts the traffic should be split into.
+	##
+	## this_part: The part of the traffic this filter will accept.  0-based.
+	global sampling_filter: function(num_parts: count, this_part: count): string;
+	
 	## Combines two valid BPF filter strings with a string based operator
 	## to form a new filter.
 	##
@@ -40,4 +47,12 @@ function combine_filters(lfilter: string,  op: string, rfilter: string): string
 		return lfilter;
 	else
 		return fmt("(%s) %s (%s)", lfilter, op, rfilter);
-	}
\ No newline at end of file
+	}
+
+function sampling_filter(num_parts: count, this_part: count): string
+	{
+	local v4_filter = fmt("ip and ((ip[14:2]+ip[18:2]) - (%d*((ip[14:2]+ip[18:2])/%d)) == %d)", num_parts, num_parts, this_part);
+	# TODO: this is probably a fairly suboptimal filter, but it should work for now.
+	local v6_filter = fmt("ip6 and ((ip6[22:2]+ip6[38:2]) - (%d*((ip6[22:2]+ip6[38:2])/%d)) == %d)", num_parts, num_parts, this_part);
+	return combine_filters(v4_filter, "or", v6_filter);
+	}

From 2ec7fbae62fb50a40a4f535fdae05f3198d0eabc Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 25 Apr 2012 23:21:53 -0400
Subject: [PATCH 3/8] Packet filter framework checkpoint.

---
 .../base/frameworks/packet-filter/main.bro    | 20 +++++++++---------
 .../base/frameworks/packet-filter/shunt.bro   | 21 +++++++++----------
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/scripts/base/frameworks/packet-filter/main.bro b/scripts/base/frameworks/packet-filter/main.bro
index 6e839c9210..9ffd8cc4c3 100644
--- a/scripts/base/frameworks/packet-filter/main.bro
+++ b/scripts/base/frameworks/packet-filter/main.bro
@@ -97,14 +97,14 @@ export {
 	## packet filter.
 	global install: function(): bool;
 	
-	## A data structure to represent filter generating factories.
-	type FilterFactory: record {
+	## A data structure to represent filter generating plugins.
+	type FilterPlugin: record {
 		## A function that is directly called when generating the complete filter.
 		func : function();
 	};
 	
-	## API function to register a new factory for dynamic restriction filters.
-	global register_filter_factory: function(ff: FilterFactory);
+	## API function to register a new pluginfor dynamic restriction filters.
+	global register_filter_plugin: function(fp: FilterPlugin);
 	
 	## Enables the old filtering approach of "only watch common ports for 
 	## analyzed protocols".
@@ -129,7 +129,7 @@ global currently_building = F;
 # Internal tracking for if the the filter being built has possibly been changed.
 global filter_changed = F;
 
-global filter_factories: set[FilterFactory] = {};
+global filter_plugins: set[FilterPlugin] = {};
 
 redef enum PcapFilterID += {
 	DefaultPcapFilter,
@@ -182,9 +182,9 @@ event bro_init() &priority=-5
 	event filter_change_tracking();
 	}
 	
-function register_filter_factory(ff: FilterFactory)
+function register_filter_plugin(fp: FilterPlugin)
 	{
-	add filter_factories[ff];
+	add filter_plugins[fp];
 	}
 
 event remove_dynamic_filter(filter_id: string)
@@ -245,10 +245,10 @@ function build(): string
 	for ( filt in dynamic_restrict_filters )
 		rfilter = combine_filters(rfilter, "and", string_cat("not (", dynamic_restrict_filters[filt], ")"));
 
-	# Generate all of the plugin factory based filters.
-	for ( factory in filter_factories )
+	# Generate all of the plugin based filters.
+	for ( plugin in filter_plugins )
 		{
-		factory$func();
+		plugin$func();
 		}
 	
 	# Finally, join them into one filter.
diff --git a/scripts/base/frameworks/packet-filter/shunt.bro b/scripts/base/frameworks/packet-filter/shunt.bro
index 5527592642..be33f8085a 100644
--- a/scripts/base/frameworks/packet-filter/shunt.bro
+++ b/scripts/base/frameworks/packet-filter/shunt.bro
@@ -46,7 +46,7 @@ export {
 global shunted_conns: set[conn_id];
 global shunted_host_pairs: set[conn_id];
 
-function conn_shunt_filters()
+function shunt_filters()
 	{
 	# NOTE: this could wrongly match if a connection happens with the ports reversed.
 	local tcp_filter = "tcp and tcp[tcpflags] & (tcp-syn|tcp-fin|tcp-rst) == 0";
@@ -63,19 +63,18 @@ function conn_shunt_filters()
 		}
 	local conn_shunt_filter = combine_filters(tcp_filter, "and", udp_filter);
 	
+	local hp_shunt_filter = "";
 	for ( id in shunted_host_pairs )
-		{
-		local hp_filter = fmt("host %s and host %s", id$orig_h, id$resp_h);
-		
-		}
-
-	PacketFilter::exclude("conn_shunt_filters", conn_shunt_filter);
-	}
+		hp_shunt_filter = combine_filters(hp_shunt_filter, "and", fmt("host %s and host %s", id$orig_h, id$resp_h));
+	
+	local filter = combine_filters(conn_shunt_filter, "and", hp_shunt_filter);
+	PacketFilter::exclude("shunt_filters", filter);
+}
 
 event bro_init() &priority=5
 	{
-	register_filter_factory([
-		$func()={ return conn_shunt_filters(); }
+	register_filter_plugin([
+		$func()={ return shunt_filters(); }
 		]);
 	}
 
@@ -144,7 +143,7 @@ function shunt_conn(id: conn_id): bool
 		NOTICE([$note=Cannot_BPF_Shunt_Conn,
 		        $msg="IPv6 connections can't be shunted with BPF due to limitations in BPF",
 		        $sub="ipv6_conn", 
-		        $id=id, $identifier=string_cat(id)]);
+		        $id=id, $identifier=cat(id)]);
 		return F;
 		}
 	

From 0c97c3c1de57cd64279cfc94b92a1f2d92c76f1b Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 2 May 2012 21:16:30 -0400
Subject: [PATCH 4/8] Several final fixes for PacketFilter framework.

- Fixed how the dpd_* variables are written.

- Fixed a bug with the shunting code.

- Updated a few tests.
---
 .../frameworks/packet-filter/__load__.bro     |  1 -
 .../base/frameworks/packet-filter/main.bro    | 36 +++++++++---------
 .../base/frameworks/packet-filter/shunt.bro   |  9 ++++-
 scripts/base/frameworks/protocols/main.bro    | 24 ++++++++----
 scripts/base/protocols/dns/main.bro           |  4 +-
 scripts/base/protocols/ftp/main.bro           |  4 +-
 scripts/base/protocols/http/main.bro          |  8 +---
 scripts/base/protocols/irc/main.bro           |  4 +-
 scripts/base/protocols/smtp/main.bro          |  8 ++--
 scripts/base/protocols/ssh/main.bro           |  4 +-
 scripts/base/protocols/ssl/main.bro           |  4 +-
 scripts/base/protocols/syslog/main.bro        |  4 +-
 .../misc/load-balancing.bro}                  | 38 ++++++++++++-------
 src/main.cc                                   |  7 ++--
 .../Baseline/core.print-bpf-filters/conn.log  |  2 +-
 .../Baseline/core.print-bpf-filters/output    | 14 ++-----
 testing/btest/core/print-bpf-filters.bro      |  8 ++--
 17 files changed, 94 insertions(+), 85 deletions(-)
 rename scripts/{base/frameworks/packet-filter/load-balance.bro => policy/misc/load-balancing.bro} (79%)

diff --git a/scripts/base/frameworks/packet-filter/__load__.bro b/scripts/base/frameworks/packet-filter/__load__.bro
index 14da4e4893..45c2488c00 100644
--- a/scripts/base/frameworks/packet-filter/__load__.bro
+++ b/scripts/base/frameworks/packet-filter/__load__.bro
@@ -1,5 +1,4 @@
 @load ./utils
 @load ./main
 @load ./shunt
-@load ./load-balance
 @load ./netstats
diff --git a/scripts/base/frameworks/packet-filter/main.bro b/scripts/base/frameworks/packet-filter/main.bro
index 9ffd8cc4c3..c5a0677add 100644
--- a/scripts/base/frameworks/packet-filter/main.bro
+++ b/scripts/base/frameworks/packet-filter/main.bro
@@ -6,6 +6,7 @@
 
 @load base/frameworks/notice
 @load base/frameworks/protocols
+@load ./utils
 
 module PacketFilter;
 
@@ -15,10 +16,10 @@ export {
 
 	## Add notice types related to packet filter errors.
 	redef enum Notice::Type += {
-		## This notice is generated if a packet filter is unable to be compiled.
+		## This notice is generated if a packet filter cannot be compiled.
 		Compile_Failure,
-	
-		## This notice is generated if a packet filter is fails to install.
+		
+		## Generated if a packet filter is fails to install.
 		Install_Failure,
 		
 		## Generated when a notice takes too long to compile.
@@ -60,7 +61,7 @@ export {
 	## used but MPLS or VLAN tags are on the traffic.
 	const restricted_filter = "" &redef;
 	
-	## The maximum amount of time that you'd like to allow for filters to compile.
+	## The maximum amount of time that you'd like to allow for BPF filters to compile.
 	## If this time is exceeded, compensation measures may be taken by the framework
 	## to reduce the filter size.  This threshold being crossed also results in
 	## the :bro:enum:`PacketFilter::Too_Long_To_Compile_Filter` notice.
@@ -224,12 +225,16 @@ function build(): string
 	
 	currently_building = T;
 	
-	# Install the default capture filter.
-	local cfilter = "";
+	# Generate all of the plugin based filters.
+	for ( plugin in filter_plugins )
+		{
+		plugin$func();
+		}
 	
+	local cfilter = "";
 	if ( |capture_filters| == 0 && ! enable_auto_protocol_capture_filters )
 		cfilter = default_capture_filter;
-		
+	
 	for ( id in capture_filters )
 		cfilter = combine_filters(cfilter, "or", capture_filters[id]);
 	
@@ -244,12 +249,6 @@ function build(): string
 	# Apply the dynamic restriction filters.
 	for ( filt in dynamic_restrict_filters )
 		rfilter = combine_filters(rfilter, "and", string_cat("not (", dynamic_restrict_filters[filt], ")"));
-
-	# Generate all of the plugin based filters.
-	for ( plugin in filter_plugins )
-		{
-		plugin$func();
-		}
 	
 	# Finally, join them into one filter.
 	local filter = combine_filters(cfilter, "and", rfilter);
@@ -270,7 +269,7 @@ function install(): bool
 	
 	local tmp_filter = build();
 	
-	#local ts = current_time();
+	local ts = current_time();
 	if ( ! precompile_pcap_filter(DefaultPcapFilter, tmp_filter) )
 		{
 		NOTICE([$note=Compile_Failure,
@@ -281,15 +280,14 @@ function install(): bool
 		else
 			Reporter::warning(fmt("Bad pcap filter '%s'", tmp_filter));
 		}
+	local diff = current_time()-ts;
+	if ( diff > max_filter_compile_time )
+		NOTICE([$note=Too_Long_To_Compile_Filter,
+		        $msg=fmt("A BPF filter is taking longer than %0.6f seconds to compile", diff)]);
 	
 	# Set it to the current filter if it passed precompiling
 	current_filter = tmp_filter;
 	
-	#local diff = current_time()-ts;
-	#if ( diff > max_filter_compile_time )
-	#	NOTICE([$note=Too_Long_To_Compile_Filter,
-	#	        $msg=fmt("A BPF filter is taking longer than %0.6f seconds to compile", diff)]);
-	
 	# Do an audit log for the packet filter.
 	local info: Info;
 	info$ts = network_time();
diff --git a/scripts/base/frameworks/packet-filter/shunt.bro b/scripts/base/frameworks/packet-filter/shunt.bro
index be33f8085a..fcbdac85aa 100644
--- a/scripts/base/frameworks/packet-filter/shunt.bro
+++ b/scripts/base/frameworks/packet-filter/shunt.bro
@@ -1,4 +1,6 @@
 @load base/frameworks/notice
+@load ./main
+@load ./utils
 
 module PacketFilter;
 
@@ -49,7 +51,7 @@ global shunted_host_pairs: set[conn_id];
 function shunt_filters()
 	{
 	# NOTE: this could wrongly match if a connection happens with the ports reversed.
-	local tcp_filter = "tcp and tcp[tcpflags] & (tcp-syn|tcp-fin|tcp-rst) == 0";
+	local tcp_filter = "";
 	local udp_filter = "";
 	for ( id in shunted_conns )
 		{
@@ -61,6 +63,8 @@ function shunt_filters()
 		else if ( prot == tcp )
 			tcp_filter = combine_filters(tcp_filter, "and", filt);
 		}
+	if ( tcp_filter != "" )
+		tcp_filter = combine_filters("tcp and tcp[tcpflags] & (tcp-syn|tcp-fin|tcp-rst) == 0", "and", tcp_filter);
 	local conn_shunt_filter = combine_filters(tcp_filter, "and", udp_filter);
 	
 	local hp_shunt_filter = "";
@@ -68,7 +72,8 @@ function shunt_filters()
 		hp_shunt_filter = combine_filters(hp_shunt_filter, "and", fmt("host %s and host %s", id$orig_h, id$resp_h));
 	
 	local filter = combine_filters(conn_shunt_filter, "and", hp_shunt_filter);
-	PacketFilter::exclude("shunt_filters", filter);
+	if ( filter != "" )
+		PacketFilter::exclude("shunt_filters", filter);
 }
 
 event bro_init() &priority=5
diff --git a/scripts/base/frameworks/protocols/main.bro b/scripts/base/frameworks/protocols/main.bro
index 8924c4c259..43ea3b49f8 100644
--- a/scripts/base/frameworks/protocols/main.bro
+++ b/scripts/base/frameworks/protocols/main.bro
@@ -1,9 +1,10 @@
 
-@load base/frameworks/packet-filter
+@load base/frameworks/packet-filter/utils
 
 module Protocols;
 
 export {
+	
 	const common_ports: table[string] of set[port] = {} &redef;
 
 	## Automatically creates a BPF filter for the specified protocol based
@@ -15,14 +16,28 @@ export {
 	## Returns: BPF filter string.
 	global protocol_to_bpf: function(protocol: string): string;
 	
+	## Create a BPF filter which matches all of the ports defined 
+	## by the various protocol analysis scripts as "common ports"
+	## for the protocol.
 	global to_bpf: function(): string;
 	
 	## Maps between human readable protocol identifiers (like "HTTP")
 	## and the internal Bro representation for an analyzer (like ANALYZER_HTTP).
 	## This is typically fully populated by the base protocol analyzer scripts.
-	const analyzer_map: table[string] of set[count] = {} &redef;
+	const analyzer_map: table[string] of set[AnalyzerTag] = {} &redef;
 }
 
+event bro_init() &priority=10
+	{
+	for ( proto in common_ports )
+		{
+		for ( p in common_ports[proto] )
+			dpd_analyzer_ports[p] = analyzer_map[proto];
+		for ( a in analyzer_map[proto] )
+			dpd_config[a] = [$ports=common_ports[proto]];
+		}
+	}
+
 function protocol_to_bpf(protocol: string): string
 	{
 	# Return an empty string if an undefined protocol was given.
@@ -42,8 +57,3 @@ function to_bpf(): string
 		output = PacketFilter::combine_filters(output, "or", protocol_to_bpf(p));
 	return output;
 	}
-	
-	
-	
-	
-	
\ No newline at end of file
diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro
index 95259fb2e8..d6b44323d1 100644
--- a/scripts/base/protocols/dns/main.bro
+++ b/scripts/base/protocols/dns/main.bro
@@ -110,9 +110,9 @@ redef record connection += {
 
 # Not attaching ANALYZER_DNS_UDP_BINPAC and ANALYZER_DNS_TCP_BINPAC right now.
 global analyzers = { ANALYZER_DNS };
-redef Protocols::analyzer_map["DNS"] = analyzers;
+redef Protocols::analyzer_map += { ["DNS"] = analyzers };
 global ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
-redef Protocols::common_ports["DNS"] = ports;
+redef Protocols::common_ports += { ["DNS"] = ports };
 
 event bro_init() &priority=5
 	{
diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro
index 52366d28e1..3b2f0cca61 100644
--- a/scripts/base/protocols/ftp/main.bro
+++ b/scripts/base/protocols/ftp/main.bro
@@ -95,9 +95,9 @@ redef record connection += {
 };
 
 global analyzers = { ANALYZER_FTP };
-redef Protocols::analyzer_map["FTP"] = analyzers;
+redef Protocols::analyzer_map += { ["FTP"] = analyzers };
 global ports = { 21/tcp };
-redef Protocols::common_ports["FTP"] = ports;
+redef Protocols::common_ports += { ["FTP"] = ports };
 
 # Establish the variable for tracking expected connections.
 global ftp_data_expected: table[addr, port] of Info &create_expire=5mins;
diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro
index dbfbe4b6ff..86b59d1f10 100644
--- a/scripts/base/protocols/http/main.bro
+++ b/scripts/base/protocols/http/main.bro
@@ -113,13 +113,9 @@ event bro_init() &priority=5
 
 
 global analyzers = { ANALYZER_HTTP, ANALYZER_HTTP_BINPAC };
-redef Protocols::analyzer_map["HTTP"] = analyzers;
+redef Protocols::analyzer_map += { ["HTTP"] = analyzers };
 global ports = { 80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3138/tcp, 8000/tcp, 8080/tcp, 8888/tcp };
-redef Protocols::common_ports["HTTP"] = ports;
-
-#redef dpd_config += {
-#	[[ANALYZER_HTTP, ANALYZER_HTTP_BINPAC]] = [$ports = Protocols::common_ports["HTTP"]],
-#};
+redef Protocols::common_ports += { ["HTTP"] = ports };
 
 redef likely_server_ports += { 
 	80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3138/tcp,
diff --git a/scripts/base/protocols/irc/main.bro b/scripts/base/protocols/irc/main.bro
index a59e5043f6..acb0250fc8 100644
--- a/scripts/base/protocols/irc/main.bro
+++ b/scripts/base/protocols/irc/main.bro
@@ -39,9 +39,9 @@ redef record connection += {
 };
 
 global analyzers = { ANALYZER_IRC };
-redef Protocols::analyzer_map["IRC"] = analyzers;
+redef Protocols::analyzer_map += { ["IRC"] = analyzers };
 global ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp, 7000/tcp };
-redef Protocols::common_ports["IRC"] = ports;
+redef Protocols::common_ports += { ["IRC"] = ports };
 
 event bro_init() &priority=5
 	{
diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro
index 5676878b18..544e40bf12 100644
--- a/scripts/base/protocols/smtp/main.bro
+++ b/scripts/base/protocols/smtp/main.bro
@@ -57,9 +57,6 @@ export {
 	const mail_path_capture = ALL_HOSTS &redef;
 		
 	global log_smtp: event(rec: Info);
-	
-	## Configure the default ports for SMTP analysis.
-	const ports = { 25/tcp, 587/tcp } &redef;
 }
 
 redef record connection += { 
@@ -68,8 +65,9 @@ redef record connection += {
 };
 
 global analyzers = { ANALYZER_SMTP };
-redef Protocols::analyzer_map["SMTP"] = analyzers;
-redef Protocols::common_ports["SMTP"] = ports;
+redef Protocols::analyzer_map += { ["SMTP"] = analyzers };
+const ports = { 25/tcp, 587/tcp } &redef;
+redef Protocols::common_ports += { ["SMTP"] = ports };
 
 event bro_init() &priority=5
 	{
diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro
index 7ca71ba9df..290b1a9c89 100644
--- a/scripts/base/protocols/ssh/main.bro
+++ b/scripts/base/protocols/ssh/main.bro
@@ -75,9 +75,9 @@ export {
 }
 
 global analyzers = { ANALYZER_SSH };
-redef Protocols::analyzer_map["SSH"] = analyzers;
+redef Protocols::analyzer_map += { ["SSH"] = analyzers };
 global ports = { 22/tcp };
-redef Protocols::common_ports["SSH"] = ports;
+redef Protocols::common_ports += { ["SSH"] = ports };
 
 redef record connection += {
 	ssh: Info &optional;
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index 6a75e01735..5c16808532 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -73,10 +73,10 @@ event bro_init() &priority=5
 	}
 	
 global analyzers = { ANALYZER_SSL };
-redef Protocols::analyzer_map["SSL"] = analyzers;
+redef Protocols::analyzer_map += { ["SSL"] = analyzers };
 global ports = { 443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp,
                  989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp };
-redef Protocols::common_ports["SSL"] = ports;
+redef Protocols::common_ports += { ["SSL"] = ports };
 
 
 function set_session(c: connection)
diff --git a/scripts/base/protocols/syslog/main.bro b/scripts/base/protocols/syslog/main.bro
index 786e3668a4..502021f851 100644
--- a/scripts/base/protocols/syslog/main.bro
+++ b/scripts/base/protocols/syslog/main.bro
@@ -26,9 +26,9 @@ export {
 }
 
 global analyzers = { ANALYZER_SYSLOG_BINPAC };
-redef Protocols::analyzer_map["SYSLOG"] = analyzers;
+redef Protocols::analyzer_map += { ["SYSLOG"] = analyzers };
 global ports = { 514/udp };
-redef Protocols::common_ports["SYSLOG"] = ports;
+redef Protocols::common_ports += { ["SYSLOG"] = ports };
 
 redef record connection += {
 	syslog: Info &optional;
diff --git a/scripts/base/frameworks/packet-filter/load-balance.bro b/scripts/policy/misc/load-balancing.bro
similarity index 79%
rename from scripts/base/frameworks/packet-filter/load-balance.bro
rename to scripts/policy/misc/load-balancing.bro
index 105a37b617..fedf075217 100644
--- a/scripts/base/frameworks/packet-filter/load-balance.bro
+++ b/scripts/policy/misc/load-balancing.bro
@@ -1,27 +1,39 @@
-##! This script implements an automated BPF based load balancing solution for Bro clusters.
-##! It is completely automated when multiple worker processes are configured for a single
-##! interface on a host.  One caveat is that in order for this script to work, your traffic 
-##! can't have any headers above the Ethernet header (vlan, mpls).
+##! This script implements the "Bro side" of several load balancing 
+##! approaches for Bro clusters.
 
 @load base/frameworks/cluster
 @load base/frameworks/packet-filter
 
-module PacketFilter;
+module LoadBalancing;
 
 export {
+	
+	type Method: enum {
+		## Apply BPF filters to each worker in a way that causes them to
+		## automatically flow balance traffic between them.
+		AUTO_BPF,
+		## Load balance traffic across the workers by making each one apply
+		## a restrict filter to only listen to a single MAC address.  This
+		## is a somewhat common deployment option for sites doing network
+		## based load balancing with MAC address rewriting and passing the 
+		## traffic to a single interface.  Multiple MAC addresses will show
+		## up on the same interface and need filtered to a single address.
+		#MAC_ADDR_BPF,
+	};
+	
+	## Defines the method of load balancing to use.
+	const method = AUTO_BPF &redef;
+	
+	# Configure the cluster framework to enable the load balancing filter configuration.
+	#global send_filter: event(for_node: string, filter: string);
+	#global confirm_filter_installation: event(success: bool);
+	
 	redef record Cluster::Node += {
 		## A BPF filter for load balancing traffic sniffed on a single interface
 		## across a number of processes.  In normal uses, this will be assigned 
 		## dynamically by the manager and installed by the workers.
 		lb_filter: string &optional;
 	};
-	
-	## Control if BPF based load balancing is enabled on cluster deployments.
-	const enable_BPF_load_balancing = F &redef;
-	
-	# Configure the cluster framework to enable the load balancing filter configuration.
-	#global send_filter: event(for_node: string, filter: string);
-	#global confirm_filter_installation: event(success: bool);
 }
 
 #redef Cluster::manager2worker_events += /LoadBalancing::send_filter/;
@@ -33,7 +45,7 @@ export {
 
 event bro_init() &priority=5
 	{
-	if ( ! enable_BPF_load_balancing )
+	if ( method != AUTO_BPF )
 		return;
 	
 	local worker_ip_interface: table[addr, string] of count = table();
diff --git a/src/main.cc b/src/main.cc
index ff33a3859d..d70832efcb 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -1003,13 +1003,14 @@ int main(int argc, char** argv)
 		vl->append(new Val(i->include_level, TYPE_COUNT));
 		mgr.QueueEvent(bro_script_loaded, vl);
 		}
-
-	dpm->PostScriptInit();
-
+	
 	reporter->ReportViaEvents(true);
 
+	// Drain the event queue here to support the protocols framework configuring DPM
 	mgr.Drain();
 
+	dpm->PostScriptInit();
+
 	have_pending_timers = ! reading_traces && timer_mgr->Size() > 0;
 
 	io_sources.Register(thread_mgr, true);
diff --git a/testing/btest/Baseline/core.print-bpf-filters/conn.log b/testing/btest/Baseline/core.print-bpf-filters/conn.log
index 5ce968d5e6..c26fd152a4 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/conn.log
+++ b/testing/btest/Baseline/core.print-bpf-filters/conn.log
@@ -5,4 +5,4 @@
 #path	conn
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count
-1128727435.450898	UWkUyAuUGXf	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945
+1278600802.069419	UWkUyAuUGXf	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.004152	9	3429	SF	-	0	ShADadfF	7	381	7	3801
diff --git a/testing/btest/Baseline/core.print-bpf-filters/output b/testing/btest/Baseline/core.print-bpf-filters/output
index a2bf430fb4..5a345c9f4f 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/output
+++ b/testing/btest/Baseline/core.print-bpf-filters/output
@@ -5,7 +5,7 @@
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1328294052.330721	-	ip or not ip	T	T
+1335502481.107322	-	ip or not ip	T	T
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
@@ -13,7 +13,7 @@
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1328294052.542418	-	((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
+1335502481.417564	-	port 42	T	T
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
@@ -21,12 +21,4 @@
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1328294052.748480	-	port 42	T	T
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	packet_filter
-#fields	ts	node	filter	init	success
-#types	time	string	string	bool	bool
-1328294052.952845	-	port 56730	T	T
+1335502481.737329	-	(vlan) and (ip or not ip)	T	T
diff --git a/testing/btest/core/print-bpf-filters.bro b/testing/btest/core/print-bpf-filters.bro
index 6d9cef0220..383982eddf 100644
--- a/testing/btest/core/print-bpf-filters.bro
+++ b/testing/btest/core/print-bpf-filters.bro
@@ -1,10 +1,8 @@
-# @TEST-EXEC: bro -r $TRACES/empty.trace -e '' >output
+# @TEST-EXEC: bro -r $TRACES/empty.trace >output
 # @TEST-EXEC: cat packet_filter.log >>output
-# @TEST-EXEC: bro -r $TRACES/empty.trace PacketFilter::all_packets=F >>output
+# @TEST-EXEC: bro -r $TRACES/empty.trace -f "port 42" >>output
 # @TEST-EXEC: cat packet_filter.log >>output
-# @TEST-EXEC: bro -r $TRACES/empty.trace -f "port 42" -e '' >>output
-# @TEST-EXEC: cat packet_filter.log >>output
-# @TEST-EXEC: bro -r $TRACES/empty.trace -C -f "port 56730" -r $TRACES/mixed-vlan-mpls.trace >>output
+# @TEST-EXEC: bro -r $TRACES/mixed-vlan-mpls.trace PacketFilter::restricted_filter="vlan" >>output
 # @TEST-EXEC: cat packet_filter.log >>output
 # @TEST-EXEC: btest-diff output
 # @TEST-EXEC: btest-diff conn.log

From ed2a5d6ac2fee89e92413339e5fce47a5ff303fe Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 2 May 2012 22:54:39 -0400
Subject: [PATCH 5/8] Last test update for PacketFilter framework.

---
 .../canonified_loaded_scripts.log             |  4 +++
 .../send.log                                  | 30 ++++++++++---------
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index 92deb62edb..2b039e0cb2 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -48,7 +48,11 @@ scripts/base/init-default.bro
   scripts/base/frameworks/signatures/__load__.bro
     scripts/base/frameworks/signatures/./main.bro
   scripts/base/frameworks/packet-filter/__load__.bro
+    scripts/base/frameworks/packet-filter/./utils.bro
     scripts/base/frameworks/packet-filter/./main.bro
+      scripts/base/frameworks/protocols/__load__.bro
+        scripts/base/frameworks/protocols/./main.bro
+    scripts/base/frameworks/packet-filter/./shunt.bro
     scripts/base/frameworks/packet-filter/./netstats.bro
   scripts/base/frameworks/software/__load__.bro
     scripts/base/frameworks/software/./main.bro
diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
index d3c14c8603..2b20030056 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
@@ -5,17 +5,19 @@
 #path	communication
 #fields	ts	peer	src_name	connected_peer_desc	connected_peer_addr	connected_peer_port	level	message
 #types	time	string	string	string	addr	port	string	string
-1326492291.485390	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] added peer
-1326492291.491731	bro	child	-	-	-	info	[#1/127.0.0.1:47757] connected
-1326492291.492024	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer connected
-1326492291.492024	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: version
-1326492291.492740	bro	script	-	-	-	info	connection established
-1326492291.492740	bro	script	-	-	-	info	requesting events matching /^?(NOTHING)$?/
-1326492291.492740	bro	script	-	-	-	info	accepting state
-1326492291.493800	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: handshake
-1326492291.493800	bro	parent	-	-	-	info	warning: no events to request
-1326492291.494161	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer_description is bro
-1326492291.494404	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer supports keep-in-cache; using that
-1326492291.494404	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: running
-1326492291.494404	bro	parent	-	-	-	info	terminating...
-1326492291.494404	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] closing connection
+1336011989.763783	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] added peer
+1336011989.763783	bro	child	-	-	-	info	[#1/127.0.0.1:47757] connected
+1336011989.763783	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer connected
+1336011989.763783	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: version
+1336011989.763783	bro	script	-	-	-	info	connection established
+1336011989.763783	bro	script	-	-	-	info	requesting events matching /^?(NOTHING)$?/
+1336011989.763783	bro	script	-	-	-	info	accepting state
+1336011989.863273	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: handshake
+1336011989.863273	bro	parent	-	-	-	info	warning: no events to request
+1336011989.873923	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer_description is bro
+1336011989.881442	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer supports keep-in-cache; using that
+1336011989.881442	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: running
+1336011989.881442	bro	parent	-	-	-	info	terminating...
+1336011989.881442	bro	script	-	-	-	info	connection closed
+1336011989.881442	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer disconnected
+1336011989.881442	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] closing connection

From 4149724f5978c82750ca5b9e47dd2f7785a406f6 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 5 Jul 2013 01:12:22 -0400
Subject: [PATCH 6/8] Updates for the PacketFilter framework to simplify it.

---
 scripts/base/frameworks/analyzer/main.bro     | 40 ++++++++++++-
 .../frameworks/packet-filter/__load__.bro     |  1 -
 .../base/frameworks/packet-filter/main.bro    | 25 ++++----
 .../base/frameworks/protocols/__load__.bro    |  1 -
 scripts/base/frameworks/protocols/main.bro    | 59 -------------------
 scripts/base/protocols/dns/main.bro           |  8 ---
 scripts/base/protocols/ftp/main.bro           |  9 +--
 scripts/base/protocols/http/main.bro          |  7 ---
 scripts/base/protocols/irc/main.bro           |  7 ---
 scripts/base/protocols/modbus/main.bro        |  3 -
 scripts/base/protocols/smtp/main.bro          |  3 -
 scripts/base/protocols/socks/main.bro         |  4 --
 scripts/base/protocols/ssh/main.bro           | 10 +---
 scripts/base/protocols/ssl/main.bro           | 20 +------
 scripts/base/protocols/syslog/main.bro        |  8 +--
 .../frameworks/packet-filter/shunt.bro        |  3 +-
 16 files changed, 64 insertions(+), 144 deletions(-)
 delete mode 100644 scripts/base/frameworks/protocols/__load__.bro
 delete mode 100644 scripts/base/frameworks/protocols/main.bro
 rename scripts/{base => policy}/frameworks/packet-filter/shunt.bro (99%)

diff --git a/scripts/base/frameworks/analyzer/main.bro b/scripts/base/frameworks/analyzer/main.bro
index c7bfd1ce34..e6c98dec5b 100644
--- a/scripts/base/frameworks/analyzer/main.bro
+++ b/scripts/base/frameworks/analyzer/main.bro
@@ -10,6 +10,8 @@
 ##! the analyzers themselves, and documented in their analyzer-specific
 ##! description along with the events that they generate.
 
+@load base/frameworks/packet-filter/utils
+
 module Analyzer;
 
 export {
@@ -96,7 +98,21 @@ export {
 	##
 	## Returns: True if succesful.
 	global schedule_analyzer: function(orig: addr, resp: addr, resp_p: port,
-					   analyzer: Analyzer::Tag, tout: interval) : bool;
+	                                   analyzer: Analyzer::Tag, tout: interval) : bool;
+
+	## Automatically creates a BPF filter for the specified protocol based
+	## on the data supplied for the protocol through the 
+	## :bro:see:`Analyzer::register_for_ports` function.
+	##
+	## tag: The analyzer tag.
+	##
+	## Returns: BPF filter string.
+	global analyzer_to_bpf: function(tag: Analyzer::Tag): string;
+
+	## Create a BPF filter which matches all of the ports defined 
+	## by the various protocol analysis scripts as "registered ports"
+	## for the protocol.
+	global get_bpf: function(): string;
 
 	## A set of analyzers to disable by default at startup. The default set
 	## contains legacy analyzers that are no longer supported.
@@ -177,3 +193,25 @@ function schedule_analyzer(orig: addr, resp: addr, resp_p: port,
 	return __schedule_analyzer(orig, resp, resp_p, analyzer, tout);
 	}
 
+function analyzer_to_bpf(tag: Analyzer::Tag): string
+	{
+	# Return an empty string if an undefined analyzer was given.
+	if ( tag !in ports )
+		return "";
+	
+	local output = "";
+	for ( p in ports[tag] )
+		output = PacketFilter::combine_filters(output, "or", PacketFilter::port_to_bpf(p));
+	return output;
+	}
+
+function get_bpf(): string
+	{
+	local output = "";
+	for ( tag in ports )
+		{
+		output = PacketFilter::combine_filters(output, "or", analyzer_to_bpf(tag));
+		}
+	return output;
+	}
+
diff --git a/scripts/base/frameworks/packet-filter/__load__.bro b/scripts/base/frameworks/packet-filter/__load__.bro
index 45c2488c00..011885e8b7 100644
--- a/scripts/base/frameworks/packet-filter/__load__.bro
+++ b/scripts/base/frameworks/packet-filter/__load__.bro
@@ -1,4 +1,3 @@
 @load ./utils
 @load ./main
-@load ./shunt
 @load ./netstats
diff --git a/scripts/base/frameworks/packet-filter/main.bro b/scripts/base/frameworks/packet-filter/main.bro
index c5a0677add..b4885a19f2 100644
--- a/scripts/base/frameworks/packet-filter/main.bro
+++ b/scripts/base/frameworks/packet-filter/main.bro
@@ -5,7 +5,7 @@
 ##! :bro:id:`capture_filters` and :bro:id:`restrict_filters` variables.
 
 @load base/frameworks/notice
-@load base/frameworks/protocols
+@load base/frameworks/analyzer
 @load ./utils
 
 module PacketFilter;
@@ -64,13 +64,13 @@ export {
 	## The maximum amount of time that you'd like to allow for BPF filters to compile.
 	## If this time is exceeded, compensation measures may be taken by the framework
 	## to reduce the filter size.  This threshold being crossed also results in
-	## the :bro:enum:`PacketFilter::Too_Long_To_Compile_Filter` notice.
+	## the :bro:see:`PacketFilter::Too_Long_To_Compile_Filter` notice.
 	const max_filter_compile_time = 100msec &redef;
 	
 	## Install a BPF filter to exclude some traffic.  The filter should positively
 	## match what is to be excluded, it will be wrapped in a "not".
 	##
-	## filter_id: A somewhat arbitrary string that can be used to identify 
+	## filter_id: An arbitrary string that can be used to identify 
 	##            the filter.
 	##
 	## filter: A BPF expression of traffic that should be excluded.
@@ -83,7 +83,7 @@ export {
 	## the BPF filter.  The filter should match the traffic you don't want
 	## to see (it will be wrapped in a "not" condition).
 	##
-	## filter_id: A somewhat arbitrary string that can be used to identify 
+	## filter_id: An arbitrary string that can be used to identify 
 	##            the filter.
 	##
 	## filter: A BPF expression of traffic that should be excluded.
@@ -119,11 +119,8 @@ export {
 
 global dynamic_restrict_filters: table[string] of string = {};
 
-# Set the default capture filter.
-redef capture_filters += { ["default"] = default_capture_filter };
-
-# Track if a filter is currenlty building so functions that would ultimately 
-# install a filter immediately can still be used buy they won't try to build or
+# Track if a filter is currently building so functions that would ultimately 
+# install a filter immediately can still be used but they won't try to build or
 # install the filter.
 global currently_building = F;
 
@@ -239,7 +236,7 @@ function build(): string
 		cfilter = combine_filters(cfilter, "or", capture_filters[id]);
 	
 	if ( enable_auto_protocol_capture_filters )
-		cfilter = combine_filters(cfilter, "or", Protocols::to_bpf());
+		cfilter = combine_filters(cfilter, "or", Analyzer::get_bpf());
 	
 	# Apply the restriction filters.
 	local rfilter = "";
@@ -269,6 +266,10 @@ function install(): bool
 	
 	local tmp_filter = build();
 	
+	# No need to proceed if the filter hasn't changed.
+	if ( tmp_filter == current_filter )
+		return F;
+
 	local ts = current_time();
 	if ( ! precompile_pcap_filter(DefaultPcapFilter, tmp_filter) )
 		{
@@ -283,8 +284,8 @@ function install(): bool
 	local diff = current_time()-ts;
 	if ( diff > max_filter_compile_time )
 		NOTICE([$note=Too_Long_To_Compile_Filter,
-		        $msg=fmt("A BPF filter is taking longer than %0.6f seconds to compile", diff)]);
-	
+		        $msg=fmt("A BPF filter is taking longer than %0.1f seconds to compile", diff)]);
+
 	# Set it to the current filter if it passed precompiling
 	current_filter = tmp_filter;
 	
diff --git a/scripts/base/frameworks/protocols/__load__.bro b/scripts/base/frameworks/protocols/__load__.bro
deleted file mode 100644
index d551be57d3..0000000000
--- a/scripts/base/frameworks/protocols/__load__.bro
+++ /dev/null
@@ -1 +0,0 @@
-@load ./main
\ No newline at end of file
diff --git a/scripts/base/frameworks/protocols/main.bro b/scripts/base/frameworks/protocols/main.bro
deleted file mode 100644
index 43ea3b49f8..0000000000
--- a/scripts/base/frameworks/protocols/main.bro
+++ /dev/null
@@ -1,59 +0,0 @@
-
-@load base/frameworks/packet-filter/utils
-
-module Protocols;
-
-export {
-	
-	const common_ports: table[string] of set[port] = {} &redef;
-
-	## Automatically creates a BPF filter for the specified protocol based
-	## on the data supplied for the protocol in the :bro:id:`common_ports`
-	## variable.
-	##
-	## protocol: A string representation for a protocol, e.g. "HTTP"
-	##
-	## Returns: BPF filter string.
-	global protocol_to_bpf: function(protocol: string): string;
-	
-	## Create a BPF filter which matches all of the ports defined 
-	## by the various protocol analysis scripts as "common ports"
-	## for the protocol.
-	global to_bpf: function(): string;
-	
-	## Maps between human readable protocol identifiers (like "HTTP")
-	## and the internal Bro representation for an analyzer (like ANALYZER_HTTP).
-	## This is typically fully populated by the base protocol analyzer scripts.
-	const analyzer_map: table[string] of set[AnalyzerTag] = {} &redef;
-}
-
-event bro_init() &priority=10
-	{
-	for ( proto in common_ports )
-		{
-		for ( p in common_ports[proto] )
-			dpd_analyzer_ports[p] = analyzer_map[proto];
-		for ( a in analyzer_map[proto] )
-			dpd_config[a] = [$ports=common_ports[proto]];
-		}
-	}
-
-function protocol_to_bpf(protocol: string): string
-	{
-	# Return an empty string if an undefined protocol was given.
-	if ( protocol !in common_ports )
-		return "";
-	
-	local output = "";
-	for ( one_port in common_ports[protocol] )
-		output = PacketFilter::combine_filters(output, "or", PacketFilter::port_to_bpf(one_port));
-	return output;
-	}
-	
-function to_bpf(): string
-	{
-	local output = "";
-	for ( p in common_ports )
-		output = PacketFilter::combine_filters(output, "or", protocol_to_bpf(p));
-	return output;
-	}
diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro
index 15da9aa7b7..ea3ec016de 100644
--- a/scripts/base/protocols/dns/main.bro
+++ b/scripts/base/protocols/dns/main.bro
@@ -122,14 +122,6 @@ redef record connection += {
 	dns_state: State &optional;
 };
 
-# DPD configuration.
-redef capture_filters += {
-	["dns"] = "port 53",
-	["mdns"] = "udp and port 5353",
-	["llmns"] = "udp and port 5355",
-	["netbios-ns"] = "udp port 137",
-};
-
 const ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
 redef likely_server_ports += { ports };
 
diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro
index e524c32c4b..448eccd454 100644
--- a/scripts/base/protocols/ftp/main.bro
+++ b/scripts/base/protocols/ftp/main.bro
@@ -111,21 +111,18 @@ redef record connection += {
 	ftp_data_reuse: bool &default=F;
 };
 
-# Configure DPD
-redef capture_filters += { ["ftp"] = "port 21 and port 2811" };
-
 const ports = { 21/tcp, 2811/tcp };
 redef likely_server_ports += { ports };
 
-# Establish the variable for tracking expected connections.
-global ftp_data_expected: table[addr, port] of Info &read_expire=5mins;
-
 event bro_init() &priority=5
 	{
 	Log::create_stream(FTP::LOG, [$columns=Info, $ev=log_ftp]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_FTP, ports);
 	}
 
+# Establish the variable for tracking expected connections.
+global ftp_data_expected: table[addr, port] of Info &read_expire=5mins;
+
 ## A set of commands where the argument can be expected to refer
 ## to a file or directory.
 const file_cmds = {
diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro
index 1c9c1cad2d..6d06376183 100644
--- a/scripts/base/protocols/http/main.bro
+++ b/scripts/base/protocols/http/main.bro
@@ -123,19 +123,12 @@ redef record connection += {
 	http_state:  State &optional;
 };
 
-# DPD configuration.
-redef capture_filters +=  {
-	["http"] = "tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888)"
-};
-
 const ports = {
 	80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3128/tcp,
 	8000/tcp, 8080/tcp, 8888/tcp,
 };
-
 redef likely_server_ports += { ports };
 
-
 # Initialize the HTTP logging stream and ports.
 event bro_init() &priority=5
 	{
diff --git a/scripts/base/protocols/irc/main.bro b/scripts/base/protocols/irc/main.bro
index 490c39f54f..a57fc95448 100644
--- a/scripts/base/protocols/irc/main.bro
+++ b/scripts/base/protocols/irc/main.bro
@@ -38,13 +38,6 @@ redef record connection += {
 	irc:  Info &optional;
 };
 
-# Some common IRC ports.
-redef capture_filters += { ["irc-6666"] = "port 6666" };
-redef capture_filters += { ["irc-6667"] = "port 6667" };
-redef capture_filters += { ["irc-6668"] = "port 6668" };
-redef capture_filters += { ["irc-6669"] = "port 6669" };
-
-# DPD configuration.
 const ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp };
 redef likely_server_ports += { ports };
 
diff --git a/scripts/base/protocols/modbus/main.bro b/scripts/base/protocols/modbus/main.bro
index a418873306..d484e7582b 100644
--- a/scripts/base/protocols/modbus/main.bro
+++ b/scripts/base/protocols/modbus/main.bro
@@ -29,9 +29,6 @@ redef record connection += {
 	modbus: Info &optional;
 };
 
-# Configure DPD and the packet filter.
-redef capture_filters += { ["modbus"] = "tcp port 502" };
-
 const ports = { 502/tcp };
 redef likely_server_ports += { ports };
 
diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro
index c7b3a452d2..d53128b06c 100644
--- a/scripts/base/protocols/smtp/main.bro
+++ b/scripts/base/protocols/smtp/main.bro
@@ -81,9 +81,6 @@ redef record connection += {
 	smtp_state: State &optional;
 };
 
-# Configure DPD
-redef capture_filters += { ["smtp"] = "tcp port 25 or tcp port 587" };
-
 const ports = { 25/tcp, 587/tcp };
 redef likely_server_ports += { ports };
 
diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro
index a188646515..f697b355c1 100644
--- a/scripts/base/protocols/socks/main.bro
+++ b/scripts/base/protocols/socks/main.bro
@@ -47,10 +47,6 @@ redef record connection += {
 	socks: SOCKS::Info &optional;
 };
 
-# Configure DPD
-redef capture_filters += { ["socks"] = "tcp port 1080" };
-redef likely_server_ports += { 1080/tcp };
-
 function set_session(c: connection, version: count)
 	{
 	if ( ! c?$socks )
diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro
index ddd3e8b834..53b61f00d8 100644
--- a/scripts/base/protocols/ssh/main.bro
+++ b/scripts/base/protocols/ssh/main.bro
@@ -70,17 +70,13 @@ export {
 	global log_ssh: event(rec: Info);
 }
 
-# Configure DPD and the packet filter
-
-const ports = { 22/tcp };
- 
-redef capture_filters += { ["ssh"] = "tcp port 22" };
-redef likely_server_ports += { ports };
-
 redef record connection += {
 	ssh: Info &optional;
 };
 
+const ports = { 22/tcp };
+redef likely_server_ports += { ports };
+
 event bro_init() &priority=5
 {
 	Log::create_stream(SSH::LOG, [$columns=Info, $ev=log_ssh]);
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index 61d8d2fdb4..65526182ac 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -94,26 +94,10 @@ redef record Info += {
 		delay_tokens: set[string] &optional;
 };
 
-redef capture_filters += {
-	["ssl"] = "tcp port 443",
-	["nntps"] = "tcp port 563",
-	["imap4-ssl"] = "tcp port 585",
-	["sshell"] = "tcp port 614",
-	["ldaps"] = "tcp port 636",
-	["ftps-data"] = "tcp port 989",
-	["ftps"] = "tcp port 990",
-	["telnets"] = "tcp port 992",
-	["imaps"] = "tcp port 993",
-	["ircs"] = "tcp port 994",
-	["pop3s"] = "tcp port 995",
-	["xmpps"] = "tcp port 5223",
-};
-
 const ports = {
 	443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp,
 	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
-} &redef;
-
+};
 redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
@@ -154,7 +138,7 @@ function log_record(info: Info)
 			{
 			log_record(info);
 			}
-		timeout max_log_delay
+		timeout SSL::max_log_delay
 			{
 			Reporter::info(fmt("SSL delay tokens not released in time (%s tokens remaining)",
 			                   |info$delay_tokens|));
diff --git a/scripts/base/protocols/syslog/main.bro b/scripts/base/protocols/syslog/main.bro
index 7c15fb4fae..afe562c890 100644
--- a/scripts/base/protocols/syslog/main.bro
+++ b/scripts/base/protocols/syslog/main.bro
@@ -26,15 +26,13 @@ export {
 	};
 }
 
-redef capture_filters += { ["syslog"] = "port 514" };
-
-const ports = { 514/udp };
-redef likely_server_ports += { ports };
-
 redef record connection += {
 	syslog: Info &optional;
 };
 
+const ports = { 514/udp };
+redef likely_server_ports += { ports };
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(Syslog::LOG, [$columns=Info]);
diff --git a/scripts/base/frameworks/packet-filter/shunt.bro b/scripts/policy/frameworks/packet-filter/shunt.bro
similarity index 99%
rename from scripts/base/frameworks/packet-filter/shunt.bro
rename to scripts/policy/frameworks/packet-filter/shunt.bro
index fcbdac85aa..fba66e60f3 100644
--- a/scripts/base/frameworks/packet-filter/shunt.bro
+++ b/scripts/policy/frameworks/packet-filter/shunt.bro
@@ -1,6 +1,5 @@
 @load base/frameworks/notice
-@load ./main
-@load ./utils
+@load base/frameworks/packet-filter
 
 module PacketFilter;
 

From af8712652178a1138e9e4df8e632ecfc32e52b89 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 5 Jul 2013 01:27:59 -0400
Subject: [PATCH 7/8] Updating test baselines.

---
 doc/scripts/DocSourcesList.cmake              |  3 ++
 .../Baseline/core.print-bpf-filters/conn.log  |  6 ++--
 .../Baseline/core.print-bpf-filters/output    | 28 ++++++-------------
 .../canonified_loaded_scripts.log             |  5 ++--
 .../canonified_loaded_scripts.log             |  5 ++--
 5 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake
index fdd919f86b..529b03ca83 100644
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@@ -112,6 +112,7 @@ rest_target(${psd} base/frameworks/notice/non-cluster.bro)
 rest_target(${psd} base/frameworks/notice/weird.bro)
 rest_target(${psd} base/frameworks/packet-filter/main.bro)
 rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
+rest_target(${psd} base/frameworks/packet-filter/utils.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
@@ -190,6 +191,7 @@ rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro)
 rest_target(${psd} policy/frameworks/intel/smtp.bro)
 rest_target(${psd} policy/frameworks/intel/ssl.bro)
 rest_target(${psd} policy/frameworks/intel/where-locations.bro)
+rest_target(${psd} policy/frameworks/packet-filter/shunt.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
@@ -198,6 +200,7 @@ rest_target(${psd} policy/integration/collective-intel/main.bro)
 rest_target(${psd} policy/misc/app-metrics.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
 rest_target(${psd} policy/misc/detect-traceroute/main.bro)
+rest_target(${psd} policy/misc/load-balancing.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
 rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/scan.bro)
diff --git a/testing/btest/Baseline/core.print-bpf-filters/conn.log b/testing/btest/Baseline/core.print-bpf-filters/conn.log
index 0fd86b8dc4..ac366f679e 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/conn.log
+++ b/testing/btest/Baseline/core.print-bpf-filters/conn.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#open	2005-10-07-23-23-57
+#open	2013-07-05-05-19-59
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
-1128727435.450898	UWkUyAuUGXf	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945	(empty)
-#close	2005-10-07-23-23-57
+1278600802.069419	UWkUyAuUGXf	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.004152	9	3429	SF	-	0	ShADadfF	7	381	7	3801	(empty)
+#close	2013-07-05-05-19-59
diff --git a/testing/btest/Baseline/core.print-bpf-filters/output b/testing/btest/Baseline/core.print-bpf-filters/output
index cadc8b22db..292d7ab457 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/output
+++ b/testing/btest/Baseline/core.print-bpf-filters/output
@@ -3,38 +3,28 @@
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-11-06-00-53-09
+#open	2013-07-05-05-14-42
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1352163189.729807	-	ip or not ip	T	T
-#close	2012-11-06-00-53-09
+1373001282.736785	-	ip or not ip	T	T
+#close	2013-07-05-05-14-42
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-11-06-00-53-10
+#open	2013-07-05-05-14-42
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1352163190.114261	-	((((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 502)) or (tcp port 995)) or (tcp port 22)) or (port 21 and port 2811)) or (tcp port 25 or tcp port 587)) or (tcp port 614)) or (tcp port 990)) or (port 6667)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
-#close	2012-11-06-00-53-10
+1373001282.899854	-	port 42	T	T
+#close	2013-07-05-05-14-42
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-11-06-00-53-10
+#open	2013-07-05-05-14-43
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1352163190.484506	-	port 42	T	T
-#close	2012-11-06-00-53-10
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	packet_filter
-#open	2012-11-06-00-53-10
-#fields	ts	node	filter	init	success
-#types	time	string	string	bool	bool
-1352163190.855090	-	port 56730	T	T
-#close	2012-11-06-00-53-10
+1373001283.061158	-	(vlan) and (ip or not ip)	T	T
+#close	2013-07-05-05-14-43
diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
index 9d3fb87861..b7585a1477 100644
--- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-06-10-19-50-56
+#open	2013-07-05-05-20-50
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -82,10 +82,11 @@ scripts/base/init-bare.bro
     scripts/base/frameworks/input/readers/sqlite.bro
   scripts/base/frameworks/analyzer/__load__.bro
     scripts/base/frameworks/analyzer/main.bro
+      scripts/base/frameworks/packet-filter/utils.bro
       build/scripts/base/bif/analyzer.bif.bro
   scripts/base/frameworks/file-analysis/__load__.bro
     scripts/base/frameworks/file-analysis/main.bro
       build/scripts/base/bif/file_analysis.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2013-06-10-19-50-56
+#close	2013-07-05-05-20-50
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index b861f44266..28430aacd8 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-06-10-19-50-57
+#open	2013-07-05-05-21-48
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -82,6 +82,7 @@ scripts/base/init-bare.bro
     scripts/base/frameworks/input/readers/sqlite.bro
   scripts/base/frameworks/analyzer/__load__.bro
     scripts/base/frameworks/analyzer/main.bro
+      scripts/base/frameworks/packet-filter/utils.bro
       build/scripts/base/bif/analyzer.bif.bro
   scripts/base/frameworks/file-analysis/__load__.bro
     scripts/base/frameworks/file-analysis/main.bro
@@ -192,4 +193,4 @@ scripts/base/init-default.bro
     scripts/base/protocols/syslog/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-06-10-19-50-57
+#close	2013-07-05-05-21-48

From 1e5906af08a91c4b9eaad51837df276b71318191 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 5 Jul 2013 01:52:37 -0400
Subject: [PATCH 8/8] Missed a test fix.

---
 scripts/test-all-policy.bro | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index daad03d9b6..1fd34d6f2f 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -24,6 +24,7 @@
 @load frameworks/intel/smtp.bro
 @load frameworks/intel/ssl.bro
 @load frameworks/intel/where-locations.bro
+@load frameworks/packet-filter/shunt.bro
 @load frameworks/software/version-changes.bro
 @load frameworks/software/vulnerable.bro
 @load integration/barnyard2/__load__.bro
@@ -35,6 +36,7 @@
 @load misc/capture-loss.bro
 @load misc/detect-traceroute/__load__.bro
 @load misc/detect-traceroute/main.bro
+@load misc/load-balancing.bro
 @load misc/loaded-scripts.bro
 @load misc/profiling.bro
 @load misc/scan.bro