Merge remote-tracking branch 'origin/topic/seth/packet-filter-updates'

Closes #1030.

* origin/topic/seth/packet-filter-updates:
  Missed a test fix.
  Updating test baselines.
  Updates for the PacketFilter framework to simplify it.
  Last test update for PacketFilter framework.
  Several final fixes for PacketFilter framework.
  Packet filter framework checkpoint.
  Checkpoint on the packet filter framework.
  Initial rework of packet filter framework.
This commit is contained in:
Robin Sommer 2013-07-07 20:45:07 -07:00
commit b62927e9de
31 changed files with 685 additions and 165 deletions

View file

@ -10,6 +10,8 @@
##! the analyzers themselves, and documented in their analyzer-specific
##! description along with the events that they generate.
@load base/frameworks/packet-filter/utils
module Analyzer;
export {
@ -96,7 +98,21 @@ export {
##
## Returns: True if succesful.
global schedule_analyzer: function(orig: addr, resp: addr, resp_p: port,
analyzer: Analyzer::Tag, tout: interval) : bool;
analyzer: Analyzer::Tag, tout: interval) : bool;
## Automatically creates a BPF filter for the specified protocol based
## on the data supplied for the protocol through the
## :bro:see:`Analyzer::register_for_ports` function.
##
## tag: The analyzer tag.
##
## Returns: BPF filter string.
global analyzer_to_bpf: function(tag: Analyzer::Tag): string;
## Create a BPF filter which matches all of the ports defined
## by the various protocol analysis scripts as "registered ports"
## for the protocol.
global get_bpf: function(): string;
## A set of analyzers to disable by default at startup. The default set
## contains legacy analyzers that are no longer supported.
@ -177,3 +193,25 @@ function schedule_analyzer(orig: addr, resp: addr, resp_p: port,
return __schedule_analyzer(orig, resp, resp_p, analyzer, tout);
}
function analyzer_to_bpf(tag: Analyzer::Tag): string
{
# Return an empty string if an undefined analyzer was given.
if ( tag !in ports )
return "";
local output = "";
for ( p in ports[tag] )
output = PacketFilter::combine_filters(output, "or", PacketFilter::port_to_bpf(p));
return output;
}
function get_bpf(): string
{
local output = "";
for ( tag in ports )
{
output = PacketFilter::combine_filters(output, "or", analyzer_to_bpf(tag));
}
return output;
}

View file

@ -216,12 +216,9 @@ function setup_peer(p: event_peer, node: Node)
request_remote_events(p, node$events);
}
if ( node?$capture_filter )
if ( node?$capture_filter && node$capture_filter != "" )
{
local filter = node$capture_filter;
if ( filter == "" )
filter = PacketFilter::default_filter;
do_script_log(p, fmt("sending capture_filter: %s", filter));
send_capture_filter(p, filter);
}

View file

@ -1,2 +1,3 @@
@load ./utils
@load ./main
@load ./netstats

View file

@ -1,10 +1,12 @@
##! This script supports how Bro sets it's BPF capture filter. By default
##! Bro sets an unrestricted filter that allows all traffic. If a filter
##! Bro sets a capture filter that allows all traffic. If a filter
##! is set on the command line, that filter takes precedence over the default
##! open filter and all filters defined in Bro scripts with the
##! :bro:id:`capture_filters` and :bro:id:`restrict_filters` variables.
@load base/frameworks/notice
@load base/frameworks/analyzer
@load ./utils
module PacketFilter;
@ -14,11 +16,14 @@ export {
## Add notice types related to packet filter errors.
redef enum Notice::Type += {
## This notice is generated if a packet filter is unable to be compiled.
## This notice is generated if a packet filter cannot be compiled.
Compile_Failure,
## This notice is generated if a packet filter is fails to install.
## Generated if a packet filter is fails to install.
Install_Failure,
## Generated when a notice takes too long to compile.
Too_Long_To_Compile_Filter
};
## The record type defining columns to be logged in the packet filter
@ -42,83 +47,248 @@ export {
success: bool &log &default=T;
};
## By default, Bro will examine all packets. If this is set to false,
## it will dynamically build a BPF filter that only select protocols
## for which the user has loaded a corresponding analysis script.
## The latter used to be default for Bro versions < 2.0. That has now
## changed however to enable port-independent protocol analysis.
const all_packets = T &redef;
## The BPF filter that is used by default to define what traffic should
## be captured. Filters defined in :bro:id:`restrict_filters` will still
## be applied to reduce the captured traffic.
const default_capture_filter = "ip or not ip" &redef;
## Filter string which is unconditionally or'ed to the beginning of every
## dynamically built filter.
const unrestricted_filter = "" &redef;
## Filter string which is unconditionally and'ed to the beginning of every
## dynamically built filter. This is mostly used when a custom filter is being
## used but MPLS or VLAN tags are on the traffic.
const restricted_filter = "" &redef;
## The maximum amount of time that you'd like to allow for BPF filters to compile.
## If this time is exceeded, compensation measures may be taken by the framework
## to reduce the filter size. This threshold being crossed also results in
## the :bro:see:`PacketFilter::Too_Long_To_Compile_Filter` notice.
const max_filter_compile_time = 100msec &redef;
## Install a BPF filter to exclude some traffic. The filter should positively
## match what is to be excluded, it will be wrapped in a "not".
##
## filter_id: An arbitrary string that can be used to identify
## the filter.
##
## filter: A BPF expression of traffic that should be excluded.
##
## Returns: A boolean value to indicate if the filter was successfully
## installed or not.
global exclude: function(filter_id: string, filter: string): bool;
## Install a temporary filter to traffic which should not be passed through
## the BPF filter. The filter should match the traffic you don't want
## to see (it will be wrapped in a "not" condition).
##
## filter_id: An arbitrary string that can be used to identify
## the filter.
##
## filter: A BPF expression of traffic that should be excluded.
##
## length: The duration for which this filter should be put in place.
##
## Returns: A boolean value to indicate if the filter was successfully
## installed or not.
global exclude_for: function(filter_id: string, filter: string, span: interval): bool;
## Call this function to build and install a new dynamically built
## packet filter.
global install: function();
global install: function(): bool;
## A data structure to represent filter generating plugins.
type FilterPlugin: record {
## A function that is directly called when generating the complete filter.
func : function();
};
## API function to register a new plugin for dynamic restriction filters.
global register_filter_plugin: function(fp: FilterPlugin);
## Enables the old filtering approach of "only watch common ports for
## analyzed protocols".
##
## Unless you know what you are doing, leave this set to F.
const enable_auto_protocol_capture_filters = F &redef;
## This is where the default packet filter is stored and it should not
## normally be modified by users.
global default_filter = "<not set yet>";
global current_filter = "<not set yet>";
}
global dynamic_restrict_filters: table[string] of string = {};
# Track if a filter is currently building so functions that would ultimately
# install a filter immediately can still be used but they won't try to build or
# install the filter.
global currently_building = F;
# Internal tracking for if the the filter being built has possibly been changed.
global filter_changed = F;
global filter_plugins: set[FilterPlugin] = {};
redef enum PcapFilterID += {
DefaultPcapFilter,
FilterTester,
};
function combine_filters(lfilter: string, rfilter: string, op: string): string
function test_filter(filter: string): bool
{
if ( lfilter == "" && rfilter == "" )
return "";
else if ( lfilter == "" )
return rfilter;
else if ( rfilter == "" )
return lfilter;
else
return fmt("(%s) %s (%s)", lfilter, op, rfilter);
if ( ! precompile_pcap_filter(FilterTester, filter) )
{
# The given filter was invalid
# TODO: generate a notice.
return F;
}
return T;
}
function build_default_filter(): string
# This tracks any changes for filtering mechanisms that play along nice
# and set filter_changed to T.
event filter_change_tracking()
{
if ( filter_changed )
install();
schedule 5min { filter_change_tracking() };
}
event bro_init() &priority=5
{
Log::create_stream(PacketFilter::LOG, [$columns=Info]);
# Preverify the capture and restrict filters to give more granular failure messages.
for ( id in capture_filters )
{
if ( ! test_filter(capture_filters[id]) )
Reporter::fatal(fmt("Invalid capture_filter named '%s' - '%s'", id, capture_filters[id]));
}
for ( id in restrict_filters )
{
if ( ! test_filter(restrict_filters[id]) )
Reporter::fatal(fmt("Invalid restrict filter named '%s' - '%s'", id, restrict_filters[id]));
}
}
event bro_init() &priority=-5
{
install();
event filter_change_tracking();
}
function register_filter_plugin(fp: FilterPlugin)
{
add filter_plugins[fp];
}
event remove_dynamic_filter(filter_id: string)
{
if ( filter_id in dynamic_restrict_filters )
{
delete dynamic_restrict_filters[filter_id];
install();
}
}
function exclude(filter_id: string, filter: string): bool
{
if ( ! test_filter(filter) )
return F;
dynamic_restrict_filters[filter_id] = filter;
install();
return T;
}
function exclude_for(filter_id: string, filter: string, span: interval): bool
{
if ( exclude(filter_id, filter) )
{
schedule span { remove_dynamic_filter(filter_id) };
return T;
}
return F;
}
function build(): string
{
if ( cmd_line_bpf_filter != "" )
# Return what the user specified on the command line;
return cmd_line_bpf_filter;
if ( all_packets )
# Return an "always true" filter.
return "ip or not ip";
currently_building = T;
# Build filter dynamically.
# Generate all of the plugin based filters.
for ( plugin in filter_plugins )
{
plugin$func();
}
# First the capture_filter.
local cfilter = "";
for ( id in capture_filters )
cfilter = combine_filters(cfilter, capture_filters[id], "or");
if ( |capture_filters| == 0 && ! enable_auto_protocol_capture_filters )
cfilter = default_capture_filter;
# Then the restrict_filter.
for ( id in capture_filters )
cfilter = combine_filters(cfilter, "or", capture_filters[id]);
if ( enable_auto_protocol_capture_filters )
cfilter = combine_filters(cfilter, "or", Analyzer::get_bpf());
# Apply the restriction filters.
local rfilter = "";
for ( id in restrict_filters )
rfilter = combine_filters(rfilter, restrict_filters[id], "and");
rfilter = combine_filters(rfilter, "and", restrict_filters[id]);
# Apply the dynamic restriction filters.
for ( filt in dynamic_restrict_filters )
rfilter = combine_filters(rfilter, "and", string_cat("not (", dynamic_restrict_filters[filt], ")"));
# Finally, join them into one filter.
local filter = combine_filters(rfilter, cfilter, "and");
if ( unrestricted_filter != "" )
filter = combine_filters(unrestricted_filter, filter, "or");
local filter = combine_filters(cfilter, "and", rfilter);
if ( unrestricted_filter != "" )
filter = combine_filters(unrestricted_filter, "or", filter);
if ( restricted_filter != "" )
filter = combine_filters(restricted_filter, "and", filter);
currently_building = F;
return filter;
}
function install()
function install(): bool
{
default_filter = build_default_filter();
if ( currently_building )
return F;
if ( ! precompile_pcap_filter(DefaultPcapFilter, default_filter) )
local tmp_filter = build();
# No need to proceed if the filter hasn't changed.
if ( tmp_filter == current_filter )
return F;
local ts = current_time();
if ( ! precompile_pcap_filter(DefaultPcapFilter, tmp_filter) )
{
NOTICE([$note=Compile_Failure,
$msg=fmt("Compiling packet filter failed"),
$sub=default_filter]);
Reporter::fatal(fmt("Bad pcap filter '%s'", default_filter));
$sub=tmp_filter]);
if ( network_time() == 0.0 )
Reporter::fatal(fmt("Bad pcap filter '%s'", tmp_filter));
else
Reporter::warning(fmt("Bad pcap filter '%s'", tmp_filter));
}
local diff = current_time()-ts;
if ( diff > max_filter_compile_time )
NOTICE([$note=Too_Long_To_Compile_Filter,
$msg=fmt("A BPF filter is taking longer than %0.1f seconds to compile", diff)]);
# Set it to the current filter if it passed precompiling
current_filter = tmp_filter;
# Do an audit log for the packet filter.
local info: Info;
@ -129,7 +299,7 @@ function install()
info$ts = current_time();
info$init = T;
}
info$filter = default_filter;
info$filter = current_filter;
if ( ! install_pcap_filter(DefaultPcapFilter) )
{
@ -137,15 +307,13 @@ function install()
info$success = F;
NOTICE([$note=Install_Failure,
$msg=fmt("Installing packet filter failed"),
$sub=default_filter]);
$sub=current_filter]);
}
if ( reading_live_traffic() || reading_traces() )
Log::write(PacketFilter::LOG, info);
}
event bro_init() &priority=10
{
Log::create_stream(PacketFilter::LOG, [$columns=Info]);
PacketFilter::install();
# Update the filter change tracking
filter_changed = F;
return T;
}

View file

@ -13,7 +13,7 @@ export {
};
## This is the interval between individual statistics collection.
const stats_collection_interval = 10secs;
const stats_collection_interval = 5min;
}
event net_stats_update(last_stat: NetStats)

View file

@ -0,0 +1,58 @@
module PacketFilter;
export {
## Takes a :bro:type:`port` and returns a BPF expression which will
## match the port.
##
## p: The port.
##
## Returns: A valid BPF filter string for matching the port.
global port_to_bpf: function(p: port): string;
## Create a BPF filter to sample IPv4 and IPv6 traffic.
##
## num_parts: The number of parts the traffic should be split into.
##
## this_part: The part of the traffic this filter will accept. 0-based.
global sampling_filter: function(num_parts: count, this_part: count): string;
## Combines two valid BPF filter strings with a string based operator
## to form a new filter.
##
## lfilter: Filter which will go on the left side.
##
## op: Operation being applied (typically "or" or "and").
##
## rfilter: Filter which will go on the right side.
##
## Returns: A new string representing the two filters combined with
## the operator. Either filter being an empty string will
## still result in a valid filter.
global combine_filters: function(lfilter: string, op: string, rfilter: string): string;
}
function port_to_bpf(p: port): string
{
local tp = get_port_transport_proto(p);
return cat(tp, " and ", fmt("port %d", p));
}
function combine_filters(lfilter: string, op: string, rfilter: string): string
{
if ( lfilter == "" && rfilter == "" )
return "";
else if ( lfilter == "" )
return rfilter;
else if ( rfilter == "" )
return lfilter;
else
return fmt("(%s) %s (%s)", lfilter, op, rfilter);
}
function sampling_filter(num_parts: count, this_part: count): string
{
local v4_filter = fmt("ip and ((ip[14:2]+ip[18:2]) - (%d*((ip[14:2]+ip[18:2])/%d)) == %d)", num_parts, num_parts, this_part);
# TODO: this is probably a fairly suboptimal filter, but it should work for now.
local v6_filter = fmt("ip6 and ((ip6[22:2]+ip6[38:2]) - (%d*((ip6[22:2]+ip6[38:2])/%d)) == %d)", num_parts, num_parts, this_part);
return combine_filters(v4_filter, "or", v6_filter);
}