Initial rework of packet filter framework.

- Large rework on packet filter framework to make many things easier.
   - Removed the PacketFilter::all_packets variable because it was confusing.
   - New variable (PacketFilter::enable_auto_protocol_capture_filters) to re-enable the old filtering model of only sniffing ports for analyzed protocols.
   - In progress plugin model for adding filtering mechanisms.
   - New default single item for capture_filters = { ["default"] = PacketFilter::default_capture_filter };
   - Mechanism and helper functions to "shunt" traffic with filters.

- Created the Protocols framework to assist with reworking how base protocol scripts are registered with DPD and other things.
   - Protocols framework creates BPF filters for registered analyzers. (if using PacketFilter framework in that mode).

scripts/base/frameworks/communication/main.bro

@@ -196,12 +196,9 @@ function setup_peer(p: event_peer, node: Node)
 		request_remote_events(p, node$events);
 		}
 
-	if ( node?$capture_filter )
+	if ( node?$capture_filter && node$capture_filter != "" )
 		{
 		local filter = node$capture_filter;
-		if ( filter == "" )
-			filter = PacketFilter::default_filter;
-
 		do_script_log(p, fmt("sending capture_filter: %s", filter));
 		send_capture_filter(p, filter);
 		}

scripts/base/frameworks/packet-filter/__load__.bro

@@ -1,2 +1,4 @@
+@load ./utils
 @load ./main
+@load ./shunt
 @load ./netstats

scripts/base/frameworks/packet-filter/main.bro

@@ -1,10 +1,11 @@
 ##! This script supports how Bro sets it's BPF capture filter.  By default
-##! Bro sets an unrestricted filter that allows all traffic.  If a filter
+##! Bro sets a capture filter that allows all traffic.  If a filter
 ##! is set on the command line, that filter takes precedence over the default
 ##! open filter and all filters defined in Bro scripts with the
 ##! :bro:id:`capture_filters` and :bro:id:`restrict_filters` variables.
 
 @load base/frameworks/notice
+@load base/frameworks/protocols
 
 module PacketFilter;
 
@@ -19,6 +20,9 @@ export {
 	
 		## This notice is generated if a packet filter is fails to install.
 		Install_Failure,
+		
+		## Generated when a notice takes too long to compile.
+		Too_Long_To_Compile_Filter
 	};
 
 	## The record type defining columns to be logged in the packet filter
@@ -41,94 +45,220 @@ export {
 		## Indicate if the filter was applied successfully.
 		success: bool  &log &default=T;
 	};
-
-	## By default, Bro will examine all packets. If this is set to false,
-	## it will dynamically build a BPF filter that only select protocols
-	## for which the user has loaded a corresponding analysis script.
-	## The latter used to be default for Bro versions < 2.0. That has now
-	## changed however to enable port-independent protocol analysis.
-	const all_packets = T &redef;
+	
+	## The BPF filter that is used by default to define what traffic should
+	## be captured.  Filters defined in :bro:id:`restrict_filters` will still
+	## be applied to reduce the captured traffic.
+	const default_capture_filter = "ip or not ip" &redef;
 	
 	## Filter string which is unconditionally or'ed to the beginning of every 
 	## dynamically built filter.
 	const unrestricted_filter = "" &redef;
 	
+	## The maximum amount of time that you'd like to allow for filters to compile.
+	## If this time is exceeded, compensation measures may be taken by the framework
+	## to reduce the filter size.  This threshold being crossed also results in
+	## the :bro:enum:`PacketFilter::Too_Long_To_Compile_Filter` notice.
+	const max_filter_compile_time = 100msec &redef;
+	
+	## Install a BPF filter to exclude some traffic.  The filter should positively
+	## match what is to be excluded, it will be wrapped in a "not".
+	##
+	## filter_id: A somewhat arbitrary string that can be used to identify 
+	##            the filter.
+	##
+	## filter: A BPF expression of traffic that should be excluded.
+	##
+	## Returns: A boolean value to indicate if the fitler was successfully
+	##          installed or not.
+	global exclude: function(filter_id: string, filter: string): bool;
+	
+	## Install a temporary filter to traffic which should not be passed through
+	## the BPF filter.  The filter should match the traffic you don't want
+	## to see (it will be wrapped in a "not" condition).
+	##
+	## filter_id: A somewhat arbitrary string that can be used to identify 
+	##            the filter.
+	##
+	## filter: A BPF expression of traffic that should be excluded.
+	##
+	## length: The duration for which this filter should be put in place.
+	##
+	## Returns: A boolean value to indicate if the filter was successfully 
+	##          installed or not.
+	global exclude_for: function(filter_id: string, filter: string, span: interval): bool;
+	
 	## Call this function to build and install a new dynamically built
 	## packet filter.
 	global install: function();
 	
+	## A data structure to represent filter generating factories.
+	type FilterFactory: record {
+		## A function that is directly called when generating the complete filter.
+		func : function();
+	};
+	
+	## API function to register a new factory for dynamic restriction filters.
+	global register_filter_factory: function(ff: FilterFactory);
+	
+	## Enables the old filtering approach of "only watch common ports for 
+	## analyzed protocols".
+	## Unless you know what you are doing, leave this set to F.
+	const enable_auto_protocol_capture_filters = F &redef;
+	
 	## This is where the default packet filter is stored and it should not 
 	## normally be modified by users.
-	global default_filter = "<not set yet>";
+	global current_filter = "<not set yet>";
 }
 
+global dynamic_restrict_filters: table[string] of string = {};
+
+# Set the default capture filter.
+redef capture_filters += { ["default"] = default_capture_filter };
+
+# Track if a filter is currenlty building so functions that would ultimately 
+# install a filter immediately can still be used buy they won't try to build or
+# install the filter.
+global currently_building = F;
+
+global filter_factories: set[FilterFactory] = {};
+
 redef enum PcapFilterID += {
 	DefaultPcapFilter,
+	FilterTester,
 };
 
-function combine_filters(lfilter: string, rfilter: string, op: string): string
+function test_filter(filter: string): bool
 	{
-	if ( lfilter == "" && rfilter == "" )
-		return "";
-	else if ( lfilter == "" )
-		return rfilter;
-	else if ( rfilter == "" )
-		return lfilter;
-	else
-		return fmt("(%s) %s (%s)", lfilter, op, rfilter);
+	if ( ! precompile_pcap_filter(FilterTester, filter) )
+		{
+		# The given filter was invalid
+		# TODO: generate a notice.
+		return F;
+		}
+	return T;
 	}
 
-function build_default_filter(): string
+event bro_init() &priority=6
+	{
+	Log::create_stream(PacketFilter::LOG, [$columns=Info]);
+	
+	# Preverify the capture and restrict filters to give more granular failure messages.
+	for ( id in capture_filters )
+		{
+		if ( ! test_filter(capture_filters[id]) )
+			Reporter::fatal(fmt("Invalid capture_filter named '%s' - '%s'", id, capture_filters[id]));
+		}
+		
+	for ( id in restrict_filters )
+		{
+		if ( ! test_filter(restrict_filters[id]) )
+			Reporter::fatal(fmt("Invalid restrict filter named '%s' - '%s'", id, restrict_filters[id]));
+		}
+	
+	install();
+	}
+	
+function register_filter_factory(ff: FilterFactory)
+	{
+	add filter_factories[ff];
+	}
+
+event remove_dynamic_filter(filter_id: string)
+	{
+	if ( filter_id in dynamic_restrict_filters )
+		{
+		delete dynamic_restrict_filters[filter_id];
+		install();
+		}
+	}
+
+function exclude(filter_id: string, filter: string): bool
+	{
+	if ( ! test_filter(filter) )
+		return F;
+		
+	dynamic_restrict_filters[filter_id] = filter;
+	install();
+	return T;
+	}
+
+function exclude_for(filter_id: string, filter: string, span: interval): bool
+	{
+	if ( exclude(filter_id, filter) )
+		{
+		schedule span { remove_dynamic_filter(filter_id) };
+		return T;
+		}
+	return F;
+	}
+
+function build(): string
 	{
 	if ( cmd_line_bpf_filter != "" )
 		# Return what the user specified on the command line;
 		return cmd_line_bpf_filter;
-
-	if ( all_packets )
-		{
-		# Return an "always true" filter.
-		if ( bro_has_ipv6() )
-			return "ip or not ip";
-		else
-			return "not ip6";
-		}
-
-	# Build filter dynamically.
 	
-	# First the capture_filter.
+	currently_building = T;
+	
+	# Install the default capture filter.
 	local cfilter = "";
-	for ( id in capture_filters )
-		cfilter = combine_filters(cfilter, capture_filters[id], "or");
+	
+	if ( |capture_filters| == 0 && ! enable_auto_protocol_capture_filters )
+		cfilter = default_capture_filter;
 		
-	# Then the restrict_filter.
+	for ( id in capture_filters )
+		cfilter = combine_filters(cfilter, "or", capture_filters[id]);
+	
+	if ( enable_auto_protocol_capture_filters )
+		cfilter = combine_filters(cfilter, "or", Protocols::to_bpf());
+	
+	# Apply the restriction filters.
 	local rfilter = "";
 	for ( id in restrict_filters )
-		rfilter = combine_filters(rfilter, restrict_filters[id], "and");
-		
+		rfilter = combine_filters(rfilter, "and", restrict_filters[id]);
+	
+	# Apply the dynamic restriction filters.
+	for ( filt in dynamic_restrict_filters )
+		rfilter = combine_filters(rfilter, "and", string_cat("not (", dynamic_restrict_filters[filt], ")"));
+
+	# Generate all of the plugin factory based filters.
+	for ( factory in filter_factories )
+		{
+		factory$func();
+		}
+	
 	# Finally, join them into one filter.
-	local filter = combine_filters(rfilter, cfilter, "and");
+	local filter = combine_filters(cfilter, "and", rfilter);
+	
 	if ( unrestricted_filter != "" )
-		filter = combine_filters(unrestricted_filter, filter, "or");
-	
-	# Exclude IPv6 if we don't support it.
-	if ( ! bro_has_ipv6() )
-		filter = combine_filters(filter, "not ip6", "and");
+		filter = combine_filters(unrestricted_filter, "or", filter);
 	
+	currently_building = F;
 	return filter;
 	}
 
 function install()
 	{
-	default_filter = build_default_filter();
-
-	if ( ! precompile_pcap_filter(DefaultPcapFilter, default_filter) )
+	if ( currently_building )
+		return;
+	
+	current_filter = build();
+	
+	#local ts = current_time();
+	if ( ! precompile_pcap_filter(DefaultPcapFilter, current_filter) )
 		{
 		NOTICE([$note=Compile_Failure, 
 		        $msg=fmt("Compiling packet filter failed"),
-		        $sub=default_filter]);
-		Reporter::fatal(fmt("Bad pcap filter '%s'", default_filter));
+		        $sub=current_filter]);
+		Reporter::fatal(fmt("Bad pcap filter '%s'", current_filter));
 		}
 	
+	#local diff = current_time()-ts;
+	#if ( diff > max_filter_compile_time )
+	#	NOTICE([$note=Too_Long_To_Compile_Filter,
+	#	        $msg=fmt("A BPF filter is taking longer than %0.6f seconds to compile", diff)]);
+	
 	# Do an audit log for the packet filter.
 	local info: Info;
 	info$ts = network_time();
@@ -138,7 +268,7 @@ function install()
 		info$ts = current_time();
 		info$init = T;
 		}
-	info$filter = default_filter;
+	info$filter = current_filter;
 	
 	if ( ! install_pcap_filter(DefaultPcapFilter) )
 		{
@@ -146,15 +276,10 @@ function install()
 		info$success = F;
 		NOTICE([$note=Install_Failure, 
 		        $msg=fmt("Installing packet filter failed"),
-		        $sub=default_filter]);
+		        $sub=current_filter]);
 		}
+		
 	
 	if ( reading_live_traffic() || reading_traces() )
 		Log::write(PacketFilter::LOG, info);
 	}
-
-event bro_init() &priority=10
-	{
-	Log::create_stream(PacketFilter::LOG, [$columns=Info]);
-	PacketFilter::install();
-	}

scripts/base/frameworks/packet-filter/shunt.bro

@@ -0,0 +1,74 @@
+@load base/frameworks/notice
+
+module PacketFilter;
+
+export {
+	const max_bpf_shunts = 100 &redef;
+
+	global shunt_conn: function(id: conn_id): bool;
+	
+	redef enum Notice::Type += {
+		## Indicative that :bro:id:`max_bpf_shunts` connections are already
+		## being shunted with BPF filters and no more are allowed.
+		No_More_Conn_Shunts_Available,
+	};
+}
+
+global shunted_conns: set[conn_id];
+global shunted_conns_non_flag_tracking: set[conn_id];
+
+function conn_shunt_filters()
+	{
+	# TODO: this could wrongly match if a connection happens with the ports reversed.
+	local filter = "";
+	local ipv4_tcp_filter = "";
+	for ( id in shunted_conns )
+		{
+		local prot = get_port_transport_proto(id$resp_p);
+		
+		# TODO: add ipv6
+		#if ( prot == udp ) #|| is_ipv6_addr(id$orig_h) )
+		#	{
+		#	next;
+		#	shunt_for()
+		#	}
+		
+		if ( prot == tcp )
+			ipv4_tcp_filter = combine_filters(ipv4_tcp_filter, "and", fmt("host %s and port %d and host %s and port %d and %s", id$orig_h, id$orig_p, id$resp_h, id$resp_p, prot));
+		}
+	
+	ipv4_tcp_filter = combine_filters(ipv4_tcp_filter, "and", "tcp[tcpflags] & (tcp-syn|tcp-fin|tcp-rst) == 0");
+
+	if ( ipv4_tcp_filter == "" )
+		return;
+	PacketFilter::exclude("conn_shunt_filters", ipv4_tcp_filter);
+	}
+
+event bro_init() &priority=5
+	{
+	register_filter_factory([
+		$func()={ return conn_shunt_filters(); }
+		]);
+	}
+
+function shunt_conn(id: conn_id): bool
+	{
+	if ( |shunted_conns| + |shunted_conns_non_flag_tracking| > max_bpf_shunts )
+		{
+		NOTICE([$note=No_More_Conn_Shunts_Available,
+		        $msg=fmt("%d BPF shunts are in place and no more will be added until space clears.", max_bpf_shunts)]);
+		return F;
+		}
+	
+	add shunted_conns[id];
+	install();
+	return T;
+	}
+	
+event connection_state_remove(c: connection) &priority=-5
+	{
+	# Don't rebuild the filter right away because the packet filter framework will check every few minutes
+	# and update the filter if things have changed.
+	if ( c$id in shunted_conns )
+		delete shunted_conns[c$id];
+	}

scripts/base/frameworks/packet-filter/utils.bro

@@ -0,0 +1,43 @@
+module PacketFilter;
+
+export {
+	## Takes a :bro:type:`port` and returns a BPF expression which will
+	## match the port.
+	##
+	## p: The port.
+	##
+	## Returns: A valid BPF filter string for matching the port.
+	global port_to_bpf: function(p: port): string;
+	
+	## Combines two valid BPF filter strings with a string based operator
+	## to form a new filter.
+	##
+	## lfilter: Filter which will go on the left side.
+	##
+	## op: Operation being applied (typically "or" or "and").
+	##
+	## rfilter: Filter which will go on the right side.
+	## 
+	## Returns: A new string representing the two filters combined with
+	##          the operator.  Either filter being an empty string will
+	##          still result in a valid filter.
+	global combine_filters: function(lfilter: string, op: string, rfilter: string): string;
+}
+
+function port_to_bpf(p: port): string
+	{
+	local tp = get_port_transport_proto(p);
+	return cat(tp, " and ", fmt("port %d", p));
+	}
+
+function combine_filters(lfilter: string,  op: string, rfilter: string): string
+	{
+	if ( lfilter == "" && rfilter == "" )
+		return "";
+	else if ( lfilter == "" )
+		return rfilter;
+	else if ( rfilter == "" )
+		return lfilter;
+	else
+		return fmt("(%s) %s (%s)", lfilter, op, rfilter);
+	}

scripts/base/frameworks/protocols/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/protocols/main.bro

@@ -0,0 +1,49 @@
+
+@load base/frameworks/packet-filter
+
+module Protocols;
+
+export {
+	const common_ports: table[string] of set[port] = {} &redef;
+
+	## Automatically creates a BPF filter for the specified protocol based
+	## on the data supplied for the protocol in the :bro:id:`common_ports`
+	## variable.
+	##
+	## protocol: A string representation for a protocol, e.g. "HTTP"
+	##
+	## Returns: BPF filter string.
+	global protocol_to_bpf: function(protocol: string): string;
+	
+	global to_bpf: function(): string;
+	
+	## Maps between human readable protocol identifiers (like "HTTP")
+	## and the internal Bro representation for an analyzer (like ANALYZER_HTTP).
+	## This is typically fully populated by the base protocol analyzer scripts.
+	const analyzer_map: table[string] of set[count] = {} &redef;
+}
+
+function protocol_to_bpf(protocol: string): string
+	{
+	# Return an empty string if an undefined protocol was given.
+	if ( protocol !in common_ports )
+		return "";
+	
+	local output = "";
+	for ( one_port in common_ports[protocol] )
+		output = PacketFilter::combine_filters(output, "or", PacketFilter::port_to_bpf(one_port));
+	return output;
+	}
+	
+function to_bpf(): string
+	{
+	local output = "";
+	for ( p in common_ports )
+		output = PacketFilter::combine_filters(output, "or", protocol_to_bpf(p));
+	return output;
+	}
+	
+	
+	
+	
+