Support for (mixed) MPLS and VLAN traffic, and a new default BPF

filter. (Seth Hall and Robin Sommer)

- Merging in the patch from #264, which provides support for mixed
  VLAN and MPLS traffic.

- Changing Bro's default filter from being built dynamically to being
  a static "ip or not ip". To get the old behaviour back (i.e., the
  dynamically built filter), redef "all_packets" to false.

- print-filter.bro now always prints the filter that Bro is actually
  using, even if overriden from the command line.

policy/bro.init

@@ -1171,6 +1171,10 @@ function string_escape(s: string, chars: string): string
 	return s;
 	}
 
+# The filter the user has set via the -f command line options, or
+# empty if none.
+const cmd_line_bpf_filter = "" &redef;
+
 @load pcap.bro
 
 # Rotate logs every x seconds.

policy/pcap.bro

@@ -4,8 +4,16 @@
 global capture_filters: table[string] of string &redef;
 global restrict_filters: table[string] of string &redef;
 
-# Filter string which is unconditionally or'ed to every pcap filter.
+# By default, Bro will examine all packets. If this is set to false,
-global unrestricted_filter = "" &redef;
+# it will dynamically build a BPF filter that only select protocols
+# for which the user has loaded a corresponding analysis script.
+# The latter used to be default for Bro versions < 1.6. That has now
+# changed however to enable port-independent protocol analysis.
+const all_packets = T &redef;
+
+# Filter string which is unconditionally or'ed to every dynamically
+# built pcap filter.
+const unrestricted_filter = "" &redef;
 
 redef enum PcapFilterID += {
 	DefaultPcapFilter,
@@ -27,6 +35,7 @@ function join_filters(capture_filter: string, restrict_filter: string): string
 
 	if ( capture_filter != "" && restrict_filter != "" )
 		filter = fmt( "(%s) and (%s)", restrict_filter, capture_filter );
+
 	else if ( capture_filter != "" )
 		filter = capture_filter;
 
@@ -34,7 +43,7 @@ function join_filters(capture_filter: string, restrict_filter: string): string
 		filter = restrict_filter;
 
 	else
-		filter = "tcp or udp or icmp";
+		filter = "ip or not ip";
 
 	if ( unrestricted_filter != "" )
 		filter = fmt( "(%s) or (%s)", unrestricted_filter, filter );
@@ -44,28 +53,30 @@ function join_filters(capture_filter: string, restrict_filter: string): string
 
 function build_default_pcap_filter(): string
 	{
-	# Build capture_filter.
+	if ( cmd_line_bpf_filter != "" )
-	local cfilter = "";
+		# Return what the user specified on the command line;
+		return cmd_line_bpf_filter;
 
+	if ( all_packets )
+		# Return an "always true" filter.
+		return "ip or not ip";
+
+	## Build filter dynamically.
+
+	# First the capture_filter.
+	local cfilter = "";
 	for ( id in capture_filters )
 		cfilter = add_to_pcap_filter(cfilter, capture_filters[id], "or");
 
-	# Build restrict_filter.
+	# Then the restrict_filter.
 	local rfilter = "";
-	local saw_VLAN = F;
 	for ( id in restrict_filters )
-		{
+		rfilter = add_to_pcap_filter(rfilter, restrict_filters[id], "and");
-		if ( restrict_filters[id] == "vlan" )
-			# These are special - they need to come first.
-			saw_VLAN = T;
-		else
-			rfilter = add_to_pcap_filter(rfilter, restrict_filters[id], "and");
-		}
 
-	if ( saw_VLAN )
+	# Finally, join them.
-		rfilter = add_to_pcap_filter("vlan", rfilter, "and");
+	local filter = join_filters(cfilter, rfilter);
 
-	return join_filters(cfilter, rfilter);
+	return filter;
 	}
 
 function install_default_pcap_filter()

policy/vlan.bro

@@ -1,5 +0,0 @@
-# $Id: vlan.bro 416 2004-09-17 03:52:28Z vern $
-
-redef restrict_filters += { ["vlan"] = "vlan" };
-
-redef encap_hdr_size = 4;

src/Net.cc

@@ -51,7 +51,6 @@ int reading_live = 0;
 int reading_traces = 0;
 int have_pending_timers = 0;
 double pseudo_realtime = 0.0;
-char* user_pcap_filter = 0;
 bool using_communication = false;
 
 double network_time = 0.0;	// time according to last packet timestamp

src/Net.h

@@ -58,9 +58,6 @@ extern int have_pending_timers;
 // is the speedup (1 = real-time, 0.5 = half real-time, etc.).
 extern double pseudo_realtime;
 
-// Pcap filter supplied by the user on the command line (if any).
-extern char* user_pcap_filter;
-
 // When we started processing the current packet and corresponding event
 // queue.
 extern double processing_start_time;

src/NetVar.cc

@@ -260,6 +260,8 @@ int record_all_packets;
 RecordType* script_id;
 TableType* id_table;
 
+StringVal* cmd_line_bpf_filter;
+
 #include "const.bif.netvar_def"
 #include "types.bif.netvar_def"
 #include "event.bif.netvar_def"
@@ -313,6 +315,9 @@ void init_general_global_var()
 	trace_output_file = internal_val("trace_output_file")->AsStringVal();
 
 	record_all_packets = opt_internal_int("record_all_packets");
+
+	cmd_line_bpf_filter =
+		internal_val("cmd_line_bpf_filter")->AsStringVal();
 	}
 
 void init_net_var()

src/NetVar.h

@@ -264,6 +264,8 @@ extern int record_all_packets;
 extern RecordType* script_id;
 extern TableType* id_table;
 
+extern StringVal* cmd_line_bpf_filter;
+
 // Initializes globals that don't pertain to network/event analysis.
 extern void init_general_global_var();
 

src/PktSrc.cc

@@ -181,16 +181,98 @@ void PktSrc::Process()
 
 	current_timestamp = next_timestamp;
 
+	int pkt_hdr_size = hdr_size;
+
+	// Unfortunately some packets on the link might have MPLS labels
+	// while others don't. That means we need to ask the link-layer if
+	// labels are in place.
+	bool have_mpls = false;
+
+	int protocol = 0;
+
+	switch ( datalink ) {
+	case DLT_NULL:
+		{
+		protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0];
+
+		if ( protocol != AF_INET && protocol != AF_INET6 )
+			{
+			sessions->Weird("non_ip_packet_in_null_transport", &hdr, data);
+			data = 0;
+			return;
+			}
+
+		break;
+		}
+
+	case DLT_EN10MB:
+		{
+		// Get protocol being carried from the ethernet frame.
+		protocol = (data[12] << 8) + data[13];
+
+		// MPLS carried over the ethernet frame.
+		if ( protocol == 0x8847 )
+			have_mpls = true;
+
+		// VLAN carried over ethernet frame.
+		else if ( protocol == 0x8100 )
+			{
+			data += get_link_header_size(datalink);
+			data += 4; // Skip the vlan header
+			pkt_hdr_size = 0;
+			}
+
+		break;
+		}
+
+	case DLT_PPP_SERIAL:
+		{
+		// Get PPP protocol.
+		protocol = (data[2] << 8) + data[3];
+
+		if ( protocol == 0x0281 )
+			// MPLS Unicast
+			have_mpls = true;
+
+		else if ( protocol != 0x0021 && protocol != 0x0057 )
+			{
+			// Neither IPv4 nor IPv6.
+			sessions->Weird("non_ip_packet_in_ppp_encapsulation", &hdr, data);
+			data = 0;
+			return;
+			}
+		break;
+		}
+	}
+
+	if ( have_mpls )
+		{
+		// Remove the data link layer
+		data += get_link_header_size(datalink);
+
+		// Denote a header size of zero before the IP header
+		pkt_hdr_size = 0;
+
+		// Skip the MPLS label stack.
+		bool end_of_stack = false;
+
+		while ( ! end_of_stack )
+			{
+			end_of_stack = *(data + 2) & 0x01;
+			data += 4;
+			}
+		}
+
 	if ( pseudo_realtime )
 		{
 		current_pseudo = CheckPseudoTime();
-		net_packet_arrival(current_pseudo, &hdr, data, hdr_size, this);
+		net_packet_arrival(current_pseudo, &hdr, data, pkt_hdr_size, this);
 		if ( ! first_wallclock )
 			first_wallclock = current_time(true);
 		}
 
 	else
-		net_packet_arrival(current_timestamp, &hdr, data, hdr_size, this);
+		net_packet_arrival(current_timestamp, &hdr, data, pkt_hdr_size, this);
 
 	data = 0;
 	}
@@ -399,6 +481,11 @@ PktInterfaceSrc::PktInterfaceSrc(const char* arg_interface, const char* filter,
 	if ( PrecompileFilter(0, filter) && SetFilter(0) )
 		{
 		SetHdrSize();
+
+		if ( closed )
+			// Couldn't get header size.
+			return;
+
 		fprintf(stderr, "listening on %s\n", interface);
 		}
 	else
@@ -647,6 +734,9 @@ int get_link_header_size(int dl)
 		return 16;
 #endif
 
+	case DLT_PPP_SERIAL:	// PPP_SERIAL
+		return 4;
+
 	case DLT_RAW:
 		return 0;
 	}

src/Sessions.cc

@@ -220,7 +220,7 @@ void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 			}
 
 		else
-			// Blanket encapsulation (e.g., for VLAN).
+			// Blanket encapsulation
 			hdr_size += encap_hdr_size;
 		}
 

src/bro.bif

@@ -1979,9 +1979,14 @@ function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool
 # Install precompiled pcap filter.
 function install_pcap_filter%(id: PcapFilterID%): bool
 	%{
-	// Don't allow the script-level to change the filter when
+	ID* user_filter = global_scope()->Lookup("cmd_line_bpf_filter");
-	// the user has specified one on the command line.
+
-	if ( user_pcap_filter )
+	if ( ! user_filter )
+		internal_error("global cmd_line_bpf_filter not defined");
+
+	if ( user_filter->ID_Val()->AsStringVal()->Len() )
+		// Don't allow the script-level to change the filter when
+		// the user has specified one on the command line.
 		return new Val(0, TYPE_BOOL);
 
 	bool success = true;

src/main.cc

@@ -349,6 +349,7 @@ int main(int argc, char** argv)
 	char* events_file = 0;
 	char* seed_load_file = getenv("BRO_SEED_FILE");
 	char* seed_save_file = 0;
+	char* user_pcap_filter = 0;
 	int seed = 0;
 	int dump_cfg = false;
 	int to_xml = 0;
@@ -743,6 +744,16 @@ int main(int argc, char** argv)
 
 	init_general_global_var();
 
+	if ( user_pcap_filter )
+		{
+		ID* id = global_scope()->Lookup("cmd_line_bpf_filter");
+
+		if ( ! id )
+			internal_error("global cmd_line_bpf_filter not defined");
+
+		id->SetVal(new StringVal(user_pcap_filter));
+		}
+
 	// Parse rule files defined on the script level.
 	char* script_rule_files =
 		copy_string(internal_val("signature_files")->AsString()->CheckString());
@@ -800,8 +811,7 @@ int main(int argc, char** argv)
 
 	if ( dns_type != DNS_PRIME )
 		net_init(interfaces, read_files, netflows, flow_files,
-			writefile,
+			writefile, "tcp or udp or icmp",
-			user_pcap_filter ? user_pcap_filter : "tcp or udp",
 			secondary_path->Filter(), do_watchdog);
 
 	if ( ! reading_traces )

testing/btest/Baseline/core.print-bpf-filters/conn.log

@@ -0,0 +1 @@
+1128727435.450898 1.733303 141.42.64.125 125.190.109.199 http 56730 80 tcp 98 9417 SF X

testing/btest/Baseline/core.print-bpf-filters/output

@@ -0,0 +1,4 @@
+ip or not ip
+ip or not ip
+tcp[13] & 7 != 0
+port 42

testing/btest/Baseline/core.vlan-mpls/conn.log

@@ -0,0 +1,3 @@
+952109346.874907 2.102560 10.1.2.1 10.34.0.1 telnet 11001 23 tcp 25 ? SH X cc=1
+1128727435.450898 1.733303 141.42.64.125 125.190.109.199 http 56730 80 tcp 98 9417 SF X
+1278600802.069419 0.004152 10.20.80.1 10.0.0.15 http 50343 80 tcp 9 3429 SF X

testing/btest/Baseline/logging.types/ssh.log

@@ -1,2 +1,2 @@
 # b	i	e	c	p	sn	n	a	d	t	iv	s	sc	ss	se	vc	ve
-T	-42	SSH::SSH	21	123	10.0.0.0/24	10.0.0.0	1.2.3.4	3.14	1303438960.30366	100.0	hurz	1,4,2,3	CC,AA,BB	EMPTY	10,20,30	EMPTY
+T	-42	SSH::SSH	21	123	10.0.0.0/24	10.0.0.0	1.2.3.4	3.14	1304093423.61814	100.0	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY

testing/btest/core/print-bpf-filters.bro

@@ -0,0 +1,7 @@
+# @TEST-EXEC: bro print-filter >output 2>&1
+# @TEST-EXEC: bro tcp print-filter >>output
+# @TEST-EXEC: bro tcp print-filter all_packets=F >>output
+# @TEST-EXEC: bro -f "port 42" print-filter >>output
+# @TEST-EXEC: bro -C -f "port 50343" -r $TRACES/mixed-vlan-mpls.trace tcp
+# @TEST-EXEC: btest-diff output
+# @TEST-EXEC: btest-diff conn.log

testing/btest/core/vlan-mpls.bro

@@ -0,0 +1,2 @@
+# @TEST-EXEC: bro -C -r $TRACES/mixed-vlan-mpls.trace tcp
+# @TEST-EXEC: btest-diff conn.log