Merge remote-tracking branch 'origin/master' into topic/robin/http-connect

Conflicts:
	scripts/base/protocols/ssl/consts.bro

CHANGES

@@ -1,4 +1,75 @@
 
+2.2-197 | 2014-02-28 15:36:58 -0800
+
+  * Remove test code. (Robin Sommer)
+
+2.2-194 | 2014-02-28 14:50:53 -0800
+
+  * Remove packet sorter. Addresses BIT-700. (Bernhard Amann)
+    
+2.2-192 | 2014-02-28 09:46:43 -0800
+
+  * Update Mozilla root bundle. (Bernhard Amann)
+    
+2.2-190 | 2014-02-27 07:34:44 -0800
+
+  * Adjust timings of a few leak tests. (Bernhard Amann)
+
+2.2-187 | 2014-02-25 07:24:42 -0800
+
+  * More Google TLS extensions that are being actively used. (Bernhard
+    Amann)
+
+  * Remove unused, and potentially unsafe, function
+    ListVal::IncludedInString. (Bernhard Amann)
+
+2.2-184 | 2014-02-24 07:28:18 -0800
+
+  * New TLS constants from
+    https://tools.ietf.org/html/draft-bmoeller-tls-downgrade-scsv-01.
+    (Bernhard Amann)
+
+2.2-180 | 2014-02-20 17:29:14 -0800
+
+  * New SSL alert descriptions from
+    https://tools.ietf.org/html/draft-ietf-tls-applayerprotoneg-04.
+    (Bernhard Amann)
+
+  * Update SQLite. (Bernhard Amann)
+
+2.2-177 | 2014-02-20 17:27:46 -0800
+
+  * Update to libmagic version 5.17. Addresses BIT-1136. (Jon Siwek)
+    
+2.2-174 | 2014-02-14 12:07:04 -0800
+
+  * Support for MPLS over VLAN. (Chris Kanich)
+
+2.2-173 | 2014-02-14 10:50:15 -0800
+
+  * Fix misidentification of SOCKS traffic that in particiular seemed
+    to happen a lot with DCE/RPC traffic. (Vlad Grigorescu)
+
+2.2-170 | 2014-02-13 16:42:07 -0800
+
+  * Refactor DNS script's state management to improve performance.
+    (Jon Siwek)
+
+  * Revert "Expanding the HTTP methods used in the signature to detect
+    HTTP traffic." (Robin Sommer)
+
+2.2-167 | 2014-02-12 20:17:39 -0800
+
+  * Increase timeouts of some unit tests. (Jon Siwek)
+
+  * Fix memory leak in modbus analyzer. Would happen if there's a
+    'modbus_read_fifo_queue_response' event handler. (Jon Siwek)
+
+  * Add channel_id TLS extension number. This number is not IANA
+    defined, but we see it being actively used. (Bernhard Amann)
+
+  * Test baseline updates for DNS change. (Robin Sommer)
+    
 2.2-158 | 2014-02-09 23:45:39 -0500
 
   * Change dns.log to include only standard DNS queries. (Jon Siwek)

CMakeLists.txt

@@ -56,7 +56,7 @@ set(LIBMAGIC_LIB_DIR ${LIBMAGIC_PREFIX}/lib)
 set(LIBMAGIC_LIBRARY ${LIBMAGIC_LIB_DIR}/libmagic.a)
 ExternalProject_Add(libmagic
     PREFIX ${LIBMAGIC_PREFIX}
-    URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.16.tar.gz
+    URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.17.tar.gz
     CONFIGURE_COMMAND ./configure --enable-static --disable-shared
                                   --prefix=${LIBMAGIC_PREFIX}
                                   --includedir=${LIBMAGIC_INCLUDE_DIR}

NEWS

@@ -47,6 +47,8 @@ Changed Functionality
 
   TODO: Update if we add a detector for filtered traces.
 
+- We have removed the packet sorter component.
+
 Bro 2.2
 =======
 

VERSION

@@ -1 +1 @@
-2.2-158
+2.2-197

scripts/base/init-bare.bro

@@ -1028,13 +1028,6 @@ const rpc_timeout = 24 sec &redef;
 ## means "forever", which resists evasion, but can lead to state accrual.
 const frag_timeout = 0.0 sec &redef;
 
-## Time window for reordering packets. This is used for dealing with timestamp
-## discrepancy between multiple packet sources.
-##
-## .. note:: Setting this can have a major performance impact as now packets
-##    need to be potentially copied and buffered.
-const packet_sort_window = 0 usecs &redef;
-
 ## If positive, indicates the encapsulation header size that should
 ## be skipped. This applies to all packets.
 const encap_hdr_size = 0 &redef;

scripts/base/protocols/dns/main.bro

@@ -109,16 +109,6 @@ export {
 	## DNS message query/transaction ID.
 	type PendingMessages: table[count] of Queue::Queue;
 
-	## Called when a pending DNS query has not been matched with a reply (or
-	## vice versa) in a sufficent amount of time.
-	##
-	## pending: table of pending messages, indexed by transaction ID.
-	##
-	## id: the index of he element being expired.
-	##
-	## Returns: amount of time to delay expiration of the element.
-	global expire_pending_msg: function(pending: PendingMessages, id: count): interval;
-
 	## The amount of time that DNS queries or replies for a given
 	## query/transaction ID are allowed to be queued while waiting for
 	## a matching reply or query.
@@ -131,16 +121,21 @@ export {
 	## response is ongoing).
 	const max_pending_msgs = 50 &redef;
 
+	## Give up trying to match pending DNS queries or replies across all
+	## query/transaction IDs once there is at least one unmatched query or
+	## reply across this number of different query IDs.
+	const max_pending_query_ids = 50 &redef;
+
 	## A record type which tracks the status of DNS queries for a given
 	## :bro:type:`connection`.
 	type State: record {
 		## Indexed by query id, returns Info record corresponding to
 		## queries that haven't been matched with a response yet.
-		pending_queries: PendingMessages &read_expire=pending_msg_expiry_interval &expire_func=expire_pending_msg;
+		pending_queries: PendingMessages;
 
 		## Indexed by query id, returns Info record corresponding to
 		## replies that haven't been matched with a query yet.
-		pending_replies: PendingMessages &read_expire=pending_msg_expiry_interval &expire_func=expire_pending_msg;
+		pending_replies: PendingMessages;
 	};
 }
 
@@ -176,7 +171,11 @@ function log_unmatched_msgs_queue(q: Queue::Queue)
 	Queue::get_vector(q, infos);
 
 	for ( i in infos )
+		{
+		event flow_weird("dns_unmatched_msg",
+		                 infos[i]$id$orig_h, infos[i]$id$resp_h);
 		Log::write(DNS::LOG, infos[i]);
+		}
 	}
 
 function log_unmatched_msgs(msgs: PendingMessages)
@@ -191,16 +190,28 @@ function log_unmatched_msgs(msgs: PendingMessages)
 function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
 	{
 	if ( id !in msgs )
-		msgs[id] = Queue::init();
-	else if ( Queue::len(msgs[id]) > max_pending_msgs )
 		{
-		local info: Info = Queue::peek(msgs[id]);
+		if ( |msgs| > max_pending_query_ids )
-		event flow_weird("dns_unmatched_msg_quantity", info$id$orig_h,
+			{
-		                 info$id$resp_h);
+			event flow_weird("dns_unmatched_query_id_quantity",
-		log_unmatched_msgs_queue(msgs[id]);
+			                 msg$id$orig_h, msg$id$resp_h);
-		# Throw away all unmatched on assumption they'll never be matched.
+			# Throw away all unmatched on assumption they'll never be matched.
+			log_unmatched_msgs(msgs);
+			}
+
 		msgs[id] = Queue::init();
 		}
+	else
+		{
+		if ( Queue::len(msgs[id]) > max_pending_msgs )
+			{
+			event flow_weird("dns_unmatched_msg_quantity",
+			                 msg$id$orig_h, msg$id$resp_h);
+			log_unmatched_msgs_queue(msgs[id]);
+			# Throw away all unmatched on assumption they'll never be matched.
+			msgs[id] = Queue::init();
+			}
+		}
 
 	Queue::put(msgs[id], msg);
 	}
@@ -447,18 +458,3 @@ event connection_state_remove(c: connection) &priority=-5
 	log_unmatched_msgs(c$dns_state$pending_queries);
 	log_unmatched_msgs(c$dns_state$pending_replies);
 	}
-
-function expire_pending_msg(pending: PendingMessages, id: count): interval
-	{
-	local infos: vector of Info;
-	Queue::get_vector(pending[id], infos);
-
-	for ( i in infos )
-		{
-		Log::write(DNS::LOG, infos[i]);
-		event flow_weird("dns_unmatched_msg", infos[i]$id$orig_h,
-		                 infos[i]$id$resp_h);
-		}
-
-	return 0sec;
-	}

scripts/base/protocols/http/dpd.sig

@@ -1,8 +1,6 @@
-# List of HTTP headers pulled from:
-#   http://annevankesteren.nl/2007/10/http-methods
 signature dpd_http_client {
   ip-proto == tcp
-  payload /^[[:space:]]*(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH|BCOPY|BDELETE|BMOVE|BPROPFIND|BPROPPATCH|NOTIFY|POLL|SUBSCRIBE|UNSUBSCRIBE|X-MS-ENUMATTS|RPC_OUT_DATA|RPC_IN_DATA)[[:space:]]*/
+  payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/
   tcp-state originator
 }
 
@@ -13,5 +11,3 @@ signature dpd_http_server {
   requires-reverse-signature dpd_http_client
   enable "http"
 }
-
-

scripts/base/protocols/ssl/consts.bro

@@ -47,6 +47,7 @@ export {
 		[70] = "protocol_version",
 		[71] = "insufficient_security",
 		[80] = "internal_error",
+		[86] = "inappropriate_fallback",
 		[90] = "user_canceled",
 		[100] = "no_renegotiation",
 		[110] = "unsupported_extension",
@@ -55,6 +56,7 @@ export {
 		[113] = "bad_certificate_status_response",
 		[114] = "bad_certificate_hash_value",
 		[115] = "unknown_psk_identity",
+		[120] = "no_application_protocol",
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
 	
 	## Mapping between numeric codes and human readable strings for SSL/TLS
@@ -87,6 +89,8 @@ export {
 		[13175] = "origin_bound_certificates",
 		[13180] = "encrypted_client_certificates",
 		[30031] = "channel_id",
+		[30032] = "channel_id_new",
+		[35655] = "padding",
 		[65281] = "renegotiation_info"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
 	
@@ -263,6 +267,8 @@ export {
 	const TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C3;
 	const TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C4;
 	const TLS_DH_ANON_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C5;
+	# draft-bmoeller-tls-downgrade-scsv-01
+	const TLS_FALLBACK_SCSV = 0x5600;
 	# RFC 4492
 	const TLS_ECDH_ECDSA_WITH_NULL_SHA = 0xC001;
 	const TLS_ECDH_ECDSA_WITH_RC4_128_SHA = 0xC002;
@@ -629,6 +635,7 @@ export {
 		[TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256] = "TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256",
 		[TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256] = "TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256",
 		[TLS_DH_ANON_WITH_CAMELLIA_256_CBC_SHA256] = "TLS_DH_ANON_WITH_CAMELLIA_256_CBC_SHA256",
+		[TLS_FALLBACK_SCSV] = "TLS_FALLBACK_SCSV",
 		[TLS_ECDH_ECDSA_WITH_NULL_SHA] = "TLS_ECDH_ECDSA_WITH_NULL_SHA",
 		[TLS_ECDH_ECDSA_WITH_RC4_128_SHA] = "TLS_ECDH_ECDSA_WITH_RC4_128_SHA",
 		[TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA] = "TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA",

src/CMakeLists.txt

@@ -293,7 +293,6 @@ set(bro_SRCS
     OpaqueVal.cc
     OSFinger.cc
     PacketFilter.cc
-    PacketSort.cc
     PersistenceSerializer.cc
     PktSrc.cc
     PolicyFile.cc

src/Net.cc

@@ -27,7 +27,6 @@
 #include "Reporter.h"
 #include "Net.h"
 #include "Anon.h"
-#include "PacketSort.h"
 #include "Serializer.h"
 #include "PacketDumper.h"
 
@@ -58,8 +57,6 @@ double bro_start_network_time;	// timestamp of first packet
 double last_watchdog_proc_time = 0.0;	// value of above during last watchdog
 bool terminating = false;	// whether we're done reading and finishing up
 
-PacketSortGlobalPQ* packet_sorter = 0;
-
 const struct pcap_pkthdr* current_hdr = 0;
 const u_char* current_pkt = 0;
 int current_dispatched = 0;
@@ -286,9 +283,6 @@ void net_init(name_list& interfaces, name_list& readfiles,
 
 	init_ip_addr_anonymizers();
 
-	if ( packet_sort_window > 0 )
-		packet_sorter = new PacketSortGlobalPQ();
-
 	sessions = new NetSessions();
 
 	if ( do_watchdog )
@@ -313,7 +307,7 @@ void expire_timers(PktSrc* src_ps)
 
 void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps, PacketSortElement* pkt_elem)
+			PktSrc* src_ps)
 	{
 	if ( ! bro_start_network_time )
 		bro_start_network_time = t;
@@ -351,7 +345,7 @@ void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			}
 		}
 
-	sessions->DispatchPacket(t, hdr, pkt, hdr_size, src_ps, pkt_elem);
+	sessions->DispatchPacket(t, hdr, pkt, hdr_size, src_ps);
 	mgr.Drain();
 
 	if ( sp )
@@ -367,62 +361,11 @@ void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 	current_pktsrc = 0;
 	}
 
-int process_packet_sorter(double latest_packet_time)
-	{
-	if ( ! packet_sorter )
-		return 0;
-
-	double min_t = latest_packet_time - packet_sort_window;
-
-	int num_pkts_dispatched = 0;
-	PacketSortElement* pkt_elem;
-
-	// Dispatch packets in the packet_sorter until timestamp min_t.
-	// It's possible that zero or multiple packets are dispatched.
-	while ( (pkt_elem = packet_sorter->RemoveMin(min_t)) != 0 )
-		{
-		net_packet_dispatch(pkt_elem->TimeStamp(),
-			pkt_elem->Hdr(), pkt_elem->Pkt(),
-			pkt_elem->HdrSize(), pkt_elem->Src(),
-			pkt_elem);
-		++num_pkts_dispatched;
-		delete pkt_elem;
-		}
-
-	return num_pkts_dispatched;
-	}
-
-void net_packet_arrival(double t, const struct pcap_pkthdr* hdr,
-			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps)
-	{
-	if ( packet_sorter )
-		{
-		// Note that when we enable packet sorter, there will
-		// be a small window between the time packet arrives
-		// to Bro and when it is processed ("dispatched").  We
-		// define network_time to be the latest timestamp for
-		// packets *dispatched* so far (usually that's the
-		// timestamp of the current packet).
-
-		// Add the packet to the packet_sorter.
-		packet_sorter->Add(
-			new PacketSortElement(src_ps, t, hdr, pkt, hdr_size));
-
-		// Do we have any packets to dispatch from packet_sorter?
-		process_packet_sorter(t);
-		}
-	else
-		// Otherwise we dispatch the packet immediately
-		net_packet_dispatch(t, hdr, pkt, hdr_size, src_ps, 0);
-	}
-
 void net_run()
 	{
 	set_processing_status("RUNNING", "net_run");
 
 	while ( io_sources.Size() ||
-		(packet_sorter && ! packet_sorter->Empty()) ||
 		(BifConst::exit_only_after_terminate && ! terminating) )
 		{
 		double ts;
@@ -445,14 +388,12 @@ void net_run()
 		current_iosrc = src;
 
 		if ( src )
-			src->Process();	// which will call net_packet_arrival()
+			src->Process();	// which will call net_packet_dispatch()
 
 		else if ( reading_live && ! pseudo_realtime)
 			{ // live but  no source is currently active
 			double ct = current_time();
-			if ( packet_sorter && ! packet_sorter->Empty() )
+			if ( ! net_is_processing_suspended() )
-				process_packet_sorter(ct);
-			else if ( ! net_is_processing_suspended() )
 				{
 				// Take advantage of the lull to get up to
 				// date on timers and events.
@@ -462,15 +403,6 @@ void net_run()
 				}
 			}
 
-		else if ( packet_sorter && ! packet_sorter->Empty() )
-			{
-			// We are no longer reading live; done with all the
-			// sources.
-			// Drain packets remaining in the packet sorter.
-			process_packet_sorter(
-				network_time + packet_sort_window + 1000000);
-			}
-
 		else if ( (have_pending_timers || using_communication) &&
 			  ! pseudo_realtime )
 			{
@@ -581,7 +513,6 @@ void net_delete()
 	set_processing_status("TERMINATING", "net_delete");
 
 	delete sessions;
-	delete packet_sorter;
 
 	for ( int i = 0; i < NUM_ADDR_ANONYMIZATION_METHODS; ++i )
 		delete ip_anonymizer[i];

src/Net.h

@@ -20,7 +20,7 @@ extern void net_run();
 extern void net_get_final_stats();
 extern void net_finish(int drain_events);
 extern void net_delete();	// Reclaim all memory, etc.
-extern void net_packet_arrival(double t, const struct pcap_pkthdr* hdr,
+extern void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
 			PktSrc* src_ps);
 extern int net_packet_match(BPF_Program* fp, const u_char* pkt,

src/NetVar.cc

@@ -156,8 +156,6 @@ int table_incremental_step;
 
 RecordType* packet_type;
 
-double packet_sort_window;
-
 double connection_status_update_interval;
 
 StringVal* state_dir;
@@ -481,8 +479,6 @@ void init_net_var()
 
 	packet_type = internal_type("packet")->AsRecordType();
 
-	packet_sort_window = opt_internal_double("packet_sort_window");
-
 	orig_addr_anonymization = opt_internal_int("orig_addr_anonymization");
 	resp_addr_anonymization = opt_internal_int("resp_addr_anonymization");
 	other_addr_anonymization = opt_internal_int("other_addr_anonymization");

src/NetVar.h

@@ -159,8 +159,6 @@ extern int table_incremental_step;
 
 extern RecordType* packet_type;
 
-extern double packet_sort_window;
-
 extern int orig_addr_anonymization, resp_addr_anonymization;
 extern int other_addr_anonymization;
 extern TableVal* preserve_orig_addr;

src/PacketSort.cc

@@ -1,364 +0,0 @@
-#include "IP.h"
-#include "PacketSort.h"
-
-const bool DEBUG_packetsort = false;
-
-PacketSortElement::PacketSortElement(PktSrc* arg_src,
-			double arg_timestamp, const struct pcap_pkthdr* arg_hdr,
-			const u_char* arg_pkt, int arg_hdr_size)
-	{
-	src = arg_src;
-	timestamp = arg_timestamp;
-	hdr = *arg_hdr;
-	hdr_size = arg_hdr_size;
-
-	pkt = new u_char[hdr.caplen];
-	memcpy(pkt, arg_pkt, hdr.caplen);
-
-	is_tcp = 0;
-	ip_hdr = 0;
-	tcp_flags = 0;
-	endp = 0;
-	payload_length = 0;
-	key = 0;
-
-	// Now check if it is a "parsable" TCP packet.
-	uint32 caplen = hdr.caplen;
-	uint32 tcp_offset;
-
-	if ( caplen >= sizeof(struct ip) + hdr_size )
-		{
-		const struct ip* ip = (const struct ip*) (pkt + hdr_size);
-		if ( ip->ip_v == 4 )
-			ip_hdr = new IP_Hdr(ip, false);
-		else if ( ip->ip_v == 6 && (caplen >= sizeof(struct ip6_hdr) + hdr_size) )
-			ip_hdr = new IP_Hdr((const struct ip6_hdr*) ip, false, caplen - hdr_size);
-		else
-			// Weird will be generated later in NetSessions::NextPacket.
-			return;
-
-		if ( ip_hdr->NextProto() == IPPROTO_TCP &&
-		      // Note: can't sort fragmented packets
-		     ( ! ip_hdr->IsFragment() ) )
-			{
-			tcp_offset = hdr_size + ip_hdr->HdrLen();
-			if ( caplen >= tcp_offset + sizeof(struct tcphdr) )
-				{
-				const struct tcphdr* tp = (const struct tcphdr*)
-							(pkt + tcp_offset);
-
-				id.src_addr = ip_hdr->SrcAddr();
-				id.dst_addr = ip_hdr->DstAddr();
-				id.src_port = tp->th_sport;
-				id.dst_port = tp->th_dport;
-				id.is_one_way = 0;
-
-				endp = addr_port_canon_lt(id.src_addr,
-							  id.src_port,
-							  id.dst_addr,
-							  id.dst_port) ? 0 : 1;
-
-				seq[endp] = ntohl(tp->th_seq);
-
-				if ( tp->th_flags & TH_ACK )
-					seq[1-endp] = ntohl(tp->th_ack);
-				else
-					seq[1-endp] = 0;
-
-				tcp_flags = tp->th_flags;
-
-				// DEBUG_MSG("%.6f: %u, %u\n", timestamp, seq[0], seq[1]);
-
-				payload_length = ip_hdr->PayloadLen() - tp->th_off * 4;
-
-				key = BuildConnIDHashKey(id);
-
-				is_tcp = 1;
-				}
-			}
-		}
-
-	if ( DEBUG_packetsort && ! is_tcp )
-		DEBUG_MSG("%.6f non-TCP packet\n", timestamp);
-	}
-
-PacketSortElement::~PacketSortElement()
-	{
-	delete [] pkt;
-	delete ip_hdr;
-	delete key;
-	}
-
-int PacketSortPQ::Timestamp_Cmp(PacketSortElement* a, PacketSortElement* b)
-	{
-	double d = a->timestamp - b->timestamp;
-
-	if ( d > 0 ) return 1;
-	else if ( d < 0 ) return -1;
-	else return 0;
-	}
-
-int PacketSortPQ::UpdatePQ(PacketSortElement* prev_e, PacketSortElement* new_e)
-	{
-	int index = prev_e->pq_index[pq_level];
-
-	new_e->pq_index[pq_level] = index;
-	pq[index] = new_e;
-
-	if ( Cmp(prev_e, new_e) > 0 )
-		return FixUp(new_e, index);
-	else
-		{
-		FixDown(new_e, index);
-		return index == 0;
-		}
-	}
-
-int PacketSortPQ::AddToPQ(PacketSortElement* new_e)
-	{
-	int index = pq.size();
-
-	new_e->pq_index[pq_level] = index;
-	pq.push_back(new_e);
-
-	return FixUp(new_e, index);
-	}
-
-int PacketSortPQ::RemoveFromPQ(PacketSortElement* prev_e)
-	{
-	if ( pq.size() > 1 )
-		{
-		PacketSortElement* new_e = pq[pq.size() - 1];
-		pq.pop_back();
-		return UpdatePQ(prev_e, new_e);		
-		}
-	else
-		{
-		pq.pop_back();
-		return 1;
-		}
-	}
-
-void PacketSortPQ::Assign(int k, PacketSortElement* e)
-	{
-	pq[k] = e;
-	e->pq_index[pq_level] = k;
-	}
-
-PacketSortConnPQ::~PacketSortConnPQ()
-	{
-	// Delete elements only in ConnPQ (not in GlobalPQ) to avoid
-	// double delete.
-	for ( int i = 0; i < (int) pq.size(); ++i )
-		{
-		delete pq[i];
-		pq[i] = 0;
-		}
-	}
-
-int PacketSortConnPQ::Cmp(PacketSortElement* a, PacketSortElement* b)
-	{
-	// Note: here we do not distinguish between packets without
-	// an ACK and packets with seq/ack of 0. The later will sorted
-	// only by their timestamps.
-
-	if ( a->seq[0] && b->seq[0] && a->seq[0] != b->seq[0] )
-		return (a->seq[0] > b->seq[0]) ? 1 : -1;
-
-	else if ( a->seq[1] && b->seq[1] && a->seq[1] != b->seq[1] )
-		return (a->seq[1] > b->seq[1]) ? 1 : -1;
-
-	else
-		return Timestamp_Cmp(a, b);
-	}
-
-int PacketSortPQ::FixUp(PacketSortElement* e, int k)
-	{
-	if ( k == 0 )
-		{
-		Assign(0, e);
-		return 1;
-		}
-
-	int parent = (k-1) / 2;
-	if ( Cmp(pq[parent], e) > 0 )
-		{
-		Assign(k, pq[parent]);
-		return FixUp(e, parent);
-		}
-	else
-		{
-		Assign(k, e);
-		return 0;
-		}
-	}
-
-void PacketSortPQ::FixDown(PacketSortElement* e, int k)
-	{
-	uint32 kid = k * 2 + 1;
-
-	if ( kid >= pq.size() )
-		{
-		Assign(k, e);
-		return;
-		}
-
-	if ( kid + 1 < pq.size() && Cmp(pq[kid], pq[kid+1]) > 0 )
-		++kid;
-
-	if ( Cmp(e, pq[kid]) > 0 )
-		{
-		Assign(k, pq[kid]);
-		FixDown(e, kid);
-		}
-	else
-		Assign(k, e);
-	}
-
-
-int PacketSortConnPQ::Add(PacketSortElement* e)
-	{
-#if 0
-	int endp = e->endp;
-	uint32 end_seq = e->seq[endp] + e->payload_length;
-
-	int p = 1 - endp;
-	if ( (e->tcp_flags & TH_RST) && ! (e->tcp_flags & TH_ACK) )
-		{
-		DEBUG_MSG("%.6f %c: %u -> %u\n",
-			  e->TimeStamp(), (p == endp) ? 'S' : 'A',
-			  e->seq[p], next_seq[p]);
-		e->seq[p] = next_seq[p];
-		}
-
-	if ( end_seq > next_seq[endp] )
-		next_seq[endp] = end_seq;
-#endif
-
-	return AddToPQ(e);
-	}
-
-void PacketSortConnPQ::UpdateDeliveredSeq(int endp, int seq, int len, int ack)
-	{
-	if ( delivered_seq[endp] == 0 || delivered_seq[endp] == seq )
-		delivered_seq[endp] = seq + len;
-	if ( ack > delivered_seq[1 - endp] )
-		delivered_seq[endp] = ack;
-	}
-
-bool PacketSortConnPQ::IsContentGapSafe(PacketSortElement* e)
-	{
-	int ack = e->seq[1 - e->endp];
-	return ack <= delivered_seq[1 - e->endp];
-	}
-
-int PacketSortConnPQ::Remove(PacketSortElement* e)
-	{
-	int ret = RemoveFromPQ(e);
-	UpdateDeliveredSeq(e->endp, e->seq[e->endp], e->payload_length,
-				e->seq[1 - e->endp]);
-	return ret;
-	}
-
-static void DeleteConnPQ(void* p)
-	{
-	delete (PacketSortConnPQ*) p;
-	}
-
-PacketSortGlobalPQ::PacketSortGlobalPQ()
-	{
-	pq_level = GLOBAL_PQ;
-	conn_pq_table.SetDeleteFunc(DeleteConnPQ);
-	}
-
-PacketSortGlobalPQ::~PacketSortGlobalPQ()
-	{
-	// Destruction of PacketSortConnPQ will delete all conn_pq's.
-	}
-
-int PacketSortGlobalPQ::Add(PacketSortElement* e)
-	{
-	if ( e->is_tcp )
-		{
-		// TCP packets are sorted by sequence numbers
-		PacketSortConnPQ* conn_pq = FindConnPQ(e);
-		PacketSortElement* prev_min = conn_pq->Min();
-
-		if ( conn_pq->Add(e) )
-			{
-			ASSERT(conn_pq->Min() != prev_min);
-
-			if ( prev_min )
-				return UpdatePQ(prev_min, e);
-			else
-				return AddToPQ(e);
-			}
-
-		else
-			{
-			ASSERT(conn_pq->Min() == prev_min);
-			return 0;
-			}
-		}
-	else
-		return AddToPQ(e);
-	}
-
-PacketSortElement* PacketSortGlobalPQ::RemoveMin(double timestamp)
-	{
-	PacketSortElement* e = Min();
-
-	if ( ! e )
-		return 0;
-
-	if ( e->is_tcp )
-		{
-		PacketSortConnPQ* conn_pq = FindConnPQ(e);
-
-#if 0
-		// Note: the content gap safety check does not work
-		// because we remove the state for a connection once
-		// it has no packet in the priority queue.
-
-		// Do not deliver e if it arrives later than timestamp,
-		// and is not content-gap-safe.
-		if ( e->timestamp > timestamp &&
-		     ! conn_pq->IsContentGapSafe(e) )
-			return 0;
-#else
-		if ( e->timestamp > timestamp )
-			return 0;
-#endif
-
-		conn_pq->Remove(e);
-		PacketSortElement* new_e = conn_pq->Min();
-
-		if ( new_e )
-			UpdatePQ(e, new_e);
-		else
-			{
-			RemoveFromPQ(e);
-			conn_pq_table.Remove(e->key);
-			delete conn_pq;
-			}
-		}
-	else
-		RemoveFromPQ(e);
-
-	return e;
-	}
-
-PacketSortConnPQ* PacketSortGlobalPQ::FindConnPQ(PacketSortElement* e)
-	{
-	if ( ! e->is_tcp )
-		reporter->InternalError("cannot find a connection for an invalid id");
-
-	PacketSortConnPQ* pq = (PacketSortConnPQ*) conn_pq_table.Lookup(e->key);
-	if ( ! pq )
-		{
-		pq = new PacketSortConnPQ();
-		conn_pq_table.Insert(e->key, pq);
-		}
-
-	return pq;
-	}

src/PacketSort.h

@@ -1,132 +0,0 @@
-#ifndef packetsort_h
-#define packetsort_h
-
-// Timestamps can be imprecise and even inconsistent among packets
-// from different sources. This class tries to guess a "correct"
-// order by looking at TCP sequence numbers.
-//
-// In particular, it tries to eliminate "false" content gaps.
-
-#include "Dict.h"
-#include "Conn.h"
-
-enum {
-	CONN_PQ,
-	GLOBAL_PQ,
-	NUM_OF_PQ_LEVEL,
-};
-
-class PktSrc;
-
-class PacketSortElement {
-public:
-	PacketSortElement(PktSrc* src, double timestamp,
-				const struct pcap_pkthdr* hdr,
-				const u_char* pkt, int hdr_size);
-	~PacketSortElement();
-
-	PktSrc* Src() const			{ return src; }
-	double TimeStamp() const		{ return timestamp; }
-	const struct pcap_pkthdr* Hdr() const	{ return &hdr; }
-	const u_char* Pkt() const		{ return pkt; }
-	int HdrSize() const			{ return hdr_size; }
-	const IP_Hdr* IPHdr() const		{ return ip_hdr; }
-
-protected:
-	PktSrc* src;
-	double timestamp;
-	struct pcap_pkthdr hdr;
-	u_char* pkt;
-	int hdr_size;
-
-	IP_Hdr* ip_hdr;
-	int is_tcp;
-	ConnID id;
-	uint32 seq[2];	// indexed by endpoint
-	int tcp_flags;
-	int endp;	// 0 or 1
-	int payload_length;
-
-	HashKey* key;
-
-	int pq_index[NUM_OF_PQ_LEVEL];
-
-	friend class PacketSortPQ;
-	friend class PacketSortConnPQ;
-	friend class PacketSortGlobalPQ;
-};
-
-class PacketSortPQ {
-public:
-	PacketSortPQ()
-		{ pq_level = -1; }
-	virtual ~PacketSortPQ() {}
-
-	PacketSortElement* Min() const	{ return (pq.size() > 0) ? pq[0] : 0; }
-
-protected:
-	virtual int Cmp(PacketSortElement* a, PacketSortElement* b) = 0;
-	int Timestamp_Cmp(PacketSortElement* a, PacketSortElement* b);
-
-	int UpdatePQ(PacketSortElement* prev_e, PacketSortElement* new_e);
-	int AddToPQ(PacketSortElement* e);
-	int RemoveFromPQ(PacketSortElement* e);
-
-	void Assign(int k, PacketSortElement* e);
-	int FixUp(PacketSortElement* e, int k);
-	void FixDown(PacketSortElement* e, int k);
-
-	vector<PacketSortElement*> pq;
-	int pq_level;
-};
-
-// Sort by sequence numbers within a connection
-class PacketSortConnPQ : public PacketSortPQ {
-public:
-	PacketSortConnPQ()
-		{
-		pq_level = CONN_PQ;
-		delivered_seq[0] = delivered_seq[1] = 0;
-		}
-	~PacketSortConnPQ();
-
-	int Add(PacketSortElement* e);
-
-	int Remove(PacketSortElement* e);
-
-	bool IsContentGapSafe(PacketSortElement* e);
-
-protected:
-	int Cmp(PacketSortElement* a, PacketSortElement* b);
-	void UpdateDeliveredSeq(int endp, int seq, int len, int ack);
-
-	int delivered_seq[2];
-};
-
-declare(PDict, PacketSortConnPQ);
-
-// Sort by timestamps.
-class PacketSortGlobalPQ : public PacketSortPQ {
-public:
-	PacketSortGlobalPQ();
-	~PacketSortGlobalPQ();
-
-	int Add(PacketSortElement* e);
-
-	int Empty() const { return conn_pq_table.Length() == 0; }
-
-	// Returns the next packet to dispatch if it arrives earlier than the
-	// given timestamp, otherwise returns 0.
-	// The packet, if to be returned, is also removed from the
-	// priority queue.
-	PacketSortElement* RemoveMin(double timestamp);
-
-protected:
-	int Cmp(PacketSortElement* a, PacketSortElement* b)
-		{ return Timestamp_Cmp(a, b); }
-	PacketSortConnPQ* FindConnPQ(PacketSortElement* e);
-
-	PDict(PacketSortConnPQ) conn_pq_table;
-};
-
-#endif

src/PktSrc.cc

@@ -229,12 +229,21 @@ void PktSrc::Process()
 			{
 			// MPLS carried over the ethernet frame.
 			case 0x8847:
+				// Remove the data link layer and denote a
+				// header size of zero before the IP header.
 				have_mpls = true;
+				data += get_link_header_size(datalink);
+				pkt_hdr_size = 0;
 				break;
 
 			// VLAN carried over the ethernet frame.
 			case 0x8100:
 				data += get_link_header_size(datalink);
+
+				// Check for MPLS in VLAN.
+				if ( ((data[2] << 8) + data[3]) == 0x8847 )
+					have_mpls = true;
+
 				data += 4; // Skip the vlan header
 				pkt_hdr_size = 0;
 
@@ -274,8 +283,13 @@ void PktSrc::Process()
 		protocol = (data[2] << 8) + data[3];
 
 		if ( protocol == 0x0281 )
-			// MPLS Unicast
+			{
+			// MPLS Unicast. Remove the data link layer and
+			// denote a header size of zero before the IP header.
 			have_mpls = true;
+			data += get_link_header_size(datalink);
+			pkt_hdr_size = 0;
+			}
 
 		else if ( protocol != 0x0021 && protocol != 0x0057 )
 			{
@@ -290,12 +304,6 @@ void PktSrc::Process()
 
 	if ( have_mpls )
 		{
-		// Remove the data link layer
-		data += get_link_header_size(datalink);
-
-		// Denote a header size of zero before the IP header
-		pkt_hdr_size = 0;
-
 		// Skip the MPLS label stack.
 		bool end_of_stack = false;
 
@@ -309,13 +317,13 @@ void PktSrc::Process()
 	if ( pseudo_realtime )
 		{
 		current_pseudo = CheckPseudoTime();
-		net_packet_arrival(current_pseudo, &hdr, data, pkt_hdr_size, this);
+		net_packet_dispatch(current_pseudo, &hdr, data, pkt_hdr_size, this);
 		if ( ! first_wallclock )
 			first_wallclock = current_time(true);
 		}
 
 	else
-		net_packet_arrival(current_timestamp, &hdr, data, pkt_hdr_size, this);
+		net_packet_dispatch(current_timestamp, &hdr, data, pkt_hdr_size, this);
 
 	data = 0;
 	}

src/RemoteSerializer.cc

@@ -1466,7 +1466,7 @@ void RemoteSerializer::Process()
 		current_pkt = p->pkt;
 		current_pktsrc = 0;
 		current_iosrc = this;
-		sessions->NextPacket(p->time, p->hdr, p->pkt, p->hdr_size, 0);
+		sessions->NextPacket(p->time, p->hdr, p->pkt, p->hdr_size);
 		mgr.Drain();
 
 		current_hdr = 0;	// done with these

src/Sessions.cc

@@ -30,7 +30,6 @@
 #include "Discard.h"
 #include "RuleMatcher.h"
 
-#include "PacketSort.h"
 #include "TunnelEncapsulation.h"
 
 #include "analyzer/Manager.h"
@@ -168,7 +167,7 @@ void NetSessions::Done()
 
 void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps, PacketSortElement* pkt_elem)
+			PktSrc* src_ps)
 	{
 	const struct ip* ip_hdr = 0;
 	const u_char* ip_data = 0;
@@ -186,14 +185,13 @@ void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 		hdr_size += encap_hdr_size;
 
 	if ( src_ps->FilterType() == TYPE_FILTER_NORMAL )
-		NextPacket(t, hdr, pkt, hdr_size, pkt_elem);
+		NextPacket(t, hdr, pkt, hdr_size);
 	else
 		NextPacketSecondary(t, hdr, pkt, hdr_size, src_ps);
 	}
 
 void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
-			     const u_char* const pkt, int hdr_size,
+			     const u_char* const pkt, int hdr_size)
-			     PacketSortElement* pkt_elem)
 	{
 	SegmentProfiler(segment_logger, "processing-packet");
 	if ( pkt_profiler )
@@ -206,70 +204,58 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
 	if ( record_all_packets )
 		DumpPacket(hdr, pkt);
 
-	if ( pkt_elem && pkt_elem->IPHdr() )
+	// ### The following isn't really correct.  What we *should*
-		// Fast path for "normal" IP packets if an IP_Hdr is
+	// do is understanding the different link layers in order to
-		// already extracted when doing PacketSort. Otherwise
+	// find the network-layer protocol ID.  That's a big
-		// the code below tries to extract the IP header, the
+	// portability pain, though, unless we just assume everything's
-		// difference here is that header extraction in
+	// Ethernet .... not great, given the potential need to deal
-		// PacketSort does not generate Weird events.
+	// with PPP or FDDI (for some older traces).  So instead
+	// we look to see if what we have is consistent with an
+	// IPv4 packet.  If not, it's either ARP or IPv6 or weird.
 
-		DoNextPacket(t, hdr, pkt_elem->IPHdr(), pkt, hdr_size, 0);
+	if ( hdr_size > static_cast<int>(hdr->caplen) )
-
-	else
 		{
-		// ### The following isn't really correct.  What we *should*
+		Weird("truncated_link_frame", hdr, pkt);
-		// do is understanding the different link layers in order to
+		return;
-		// find the network-layer protocol ID.  That's a big
+		}
-		// portability pain, though, unless we just assume everything's
-		// Ethernet .... not great, given the potential need to deal
-		// with PPP or FDDI (for some older traces).  So instead
-		// we look to see if what we have is consistent with an
-		// IPv4 packet.  If not, it's either ARP or IPv6 or weird.
 
-		if ( hdr_size > static_cast<int>(hdr->caplen) )
+	uint32 caplen = hdr->caplen - hdr_size;
-			{
+	if ( caplen < sizeof(struct ip) )
-			Weird("truncated_link_frame", hdr, pkt);
+		{
-			return;
+		Weird("truncated_IP", hdr, pkt);
-			}
+		return;
+		}
 
-		uint32 caplen = hdr->caplen - hdr_size;
+	const struct ip* ip = (const struct ip*) (pkt + hdr_size);
-		if ( caplen < sizeof(struct ip) )
+
+	if ( ip->ip_v == 4 )
+		{
+		IP_Hdr ip_hdr(ip, false);
+		DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size, 0);
+		}
+
+	else if ( ip->ip_v == 6 )
+		{
+		if ( caplen < sizeof(struct ip6_hdr) )
 			{
 			Weird("truncated_IP", hdr, pkt);
 			return;
 			}
 
-		const struct ip* ip = (const struct ip*) (pkt + hdr_size);
+		IP_Hdr ip_hdr((const struct ip6_hdr*) (pkt + hdr_size), false, caplen);
+		DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size, 0);
+		}
 
-		if ( ip->ip_v == 4 )
+	else if ( analyzer::arp::ARP_Analyzer::IsARP(pkt, hdr_size) )
-			{
+		{
-			IP_Hdr ip_hdr(ip, false);
+		if ( arp_analyzer )
-			DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size, 0);
+			arp_analyzer->NextPacket(t, hdr, pkt, hdr_size);
-			}
+		}
 
-		else if ( ip->ip_v == 6 )
+	else
-			{
+		{
-			if ( caplen < sizeof(struct ip6_hdr) )
+		Weird("unknown_packet_type", hdr, pkt);
-				{
+		return;
-				Weird("truncated_IP", hdr, pkt);
-				return;
-				}
-
-			IP_Hdr ip_hdr((const struct ip6_hdr*) (pkt + hdr_size), false, caplen);
-			DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size, 0);
-			}
-
-		else if ( analyzer::arp::ARP_Analyzer::IsARP(pkt, hdr_size) )
-			{
-			if ( arp_analyzer )
-				arp_analyzer->NextPacket(t, hdr, pkt, hdr_size);
-			}
-
-		else
-			{
-			Weird("unknown_packet_type", hdr, pkt);
-			return;
-			}
 		}
 
 	if ( dump_this_packet && ! record_all_packets )

src/Sessions.h

@@ -28,7 +28,6 @@ declare(PDict,FragReassembler);
 
 class Discarder;
 class PacketFilter;
-class PacketSortElement;
 
 namespace analyzer { namespace stepping_stone { class SteppingStoneManager; } }
 namespace analyzer { namespace arp { class ARP_Analyzer; } }
@@ -74,7 +73,7 @@ public:
 	// employing the packet sorter first.
 	void DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 			const u_char* const pkt, int hdr_size,
-			PktSrc* src_ps, PacketSortElement* pkt_elem);
+			PktSrc* src_ps);
 
 	void Done();	// call to drain events before destructing
 
@@ -220,8 +219,7 @@ protected:
 				uint8 tcp_flags, bool& flip_roles);
 
 	void NextPacket(double t, const struct pcap_pkthdr* hdr,
-			const u_char* const pkt, int hdr_size,
+			const u_char* const pkt, int hdr_size);
-			PacketSortElement* pkt_elem);
 
 	void NextPacketSecondary(double t, const struct pcap_pkthdr* hdr,
 			const u_char* const pkt, int hdr_size,

src/Val.cc

@@ -1171,23 +1171,6 @@ ListVal::~ListVal()
 	Unref(type);
 	}
 
-const char* ListVal::IncludedInString(const char* str) const
-	{
-	if ( tag != TYPE_STRING )
-		Internal("non-string list in ListVal::IncludedInString");
-
-	loop_over_list(vals, i)
-		{
-		const char* vs = (const char*) (vals[i]->AsString()->Bytes());
-
-		const char* embedded = strstr(str, vs);
-		if ( embedded )
-			return embedded;
-		}
-
-	return 0;
-	}
-
 RE_Matcher* ListVal::BuildRE() const
 	{
 	if ( tag != TYPE_STRING )

src/Val.h

@@ -669,13 +669,6 @@ public:
 	Val* Index(const int n)		{ return vals[n]; }
 	const Val* Index(const int n) const	{ return vals[n]; }
 
-	// Returns offset of where str includes one of the strings in this
-	// ListVal (which had better be a list of strings), nil if none.
-	//
-	// Assumes that all of the strings in the list are NUL-terminated
-	// and do not have any embedded NULs.
-	const char* IncludedInString(const char* str) const;
-
 	// Returns an RE_Matcher() that will match any string that
 	// includes embedded within it one of the patterns listed
 	// (as a string, e.g., "foo|bar") in this ListVal.

src/analyzer/protocol/arp/ARP.h

@@ -24,7 +24,10 @@
 #endif
 
 #include "NetVar.h"
-#include "PacketSort.h"
+
+extern "C" {
+#include <pcap.h>
+}
 
 namespace analyzer { namespace arp {
 

src/analyzer/protocol/modbus/modbus-analyzer.pac

@@ -10,6 +10,7 @@
 %header{
 	VectorVal* bytestring_to_coils(bytestring coils, uint quantity);
 	RecordVal* HeaderToBro(ModbusTCP_TransportHeader *header);
+	VectorVal* create_vector_of_count();
 	%}
 
 %code{
@@ -30,6 +31,14 @@
 		return modbus_header;
 		}
 
+	VectorVal* create_vector_of_count()
+		{
+		VectorType* vt = new VectorType(base_type(TYPE_COUNT));
+		VectorVal* vv = new VectorVal(vt);
+		Unref(vt);
+		return vv;
+		}
+
 	%}
 
 refine flow ModbusTCP_Flow += {
@@ -367,7 +376,7 @@ refine flow ModbusTCP_Flow += {
 		if ( ::modbus_read_file_record_request )
 			{
 			//TODO: this need to be a vector of some Reference Request record type
-			//VectorVal *t = new VectorVal(new VectorType(base_type(TYPE_COUNT)));
+			//VectorVal *t = create_vector_of_count();
 			//for ( unsigned int i = 0; i < (${message.references}->size()); ++i )
 			//	{
 			//	Val* r = new Val((${message.references[i].ref_type}), TYPE_COUNT);
@@ -393,7 +402,7 @@ refine flow ModbusTCP_Flow += {
 		%{
 		if ( ::modbus_read_file_record_response )
 			{
-			//VectorVal *t = new VectorVal(new VectorType(base_type(TYPE_COUNT)));
+			//VectorVal *t = create_vector_of_count();
 			//for ( unsigned int i = 0; i < ${message.references}->size(); ++i )
 			//	{
 			//	//TODO: work the reference type in here somewhere
@@ -414,7 +423,7 @@ refine flow ModbusTCP_Flow += {
 		%{
 		if ( ::modbus_write_file_record_request )
 			{
-			//VectorVal* t = new VectorVal(new VectorType(base_type(TYPE_COUNT)));
+			//VectorVal* t = create_vector_of_count();
 			//for ( unsigned int i = 0; i < (${message.references}->size()); ++i )
 			//	{
 			//	Val* r = new Val((${message.references[i].ref_type}), TYPE_COUNT);
@@ -447,7 +456,7 @@ refine flow ModbusTCP_Flow += {
 		%{
 		if ( ::modbus_write_file_record_response )
 			{
-			//VectorVal* t = new VectorVal(new VectorType(base_type(TYPE_COUNT)));
+			//VectorVal* t = create_vector_of_count();
 			//for ( unsigned int i = 0; i < (${messages.references}->size()); ++i )
 			//	{
 			//	Val* r = new Val((${message.references[i].ref_type}), TYPE_COUNT);
@@ -589,7 +598,7 @@ refine flow ModbusTCP_Flow += {
 
 		if ( ::modbus_read_fifo_queue_response )
 			{
-			VectorVal* t = new VectorVal(new VectorType(base_type(TYPE_COUNT)));
+			VectorVal* t = create_vector_of_count();
 			for ( unsigned int i = 0; i < (${message.register_data})->size(); ++i )
 				{
 				Val* r = new Val(${message.register_data[i]}, TYPE_COUNT);

src/analyzer/protocol/socks/socks-analyzer.pac

@@ -62,6 +62,14 @@ refine connection SOCKS_Conn += {
 		if ( ${request.reserved} != 0 )
 			{
 			bro_analyzer()->ProtocolViolation(fmt("invalid value in reserved field: %d", ${request.reserved}));
+			bro_analyzer()->SetSkip(true);
+			return false;
+			}
+
+		if ( (${request.command} == 0) || (${request.command} > 3) )
+			{
+			bro_analyzer()->ProtocolViolation(fmt("undefined value in command field: %d", ${request.command}));
+			bro_analyzer()->SetSkip(true);
 			return false;
 			}
 

testing/btest/Baseline/core.mpls-in-vlan/conn.log

@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2014-02-14-20-04-20
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1371685686.536606	CXWv6p3arKYeMETxOg	65.65.65.65	19244	65.65.65.65	80	tcp	-	-	-	-	OTH	-	0	D	1	257	0	0	(empty)
+1371686961.156859	CjhGID4nQcgTWjvg4c	65.65.65.65	32828	65.65.65.65	80	tcp	-	-	-	-	OTH	-	0	d	0	0	1	1500	(empty)
+1371686961.479321	CCvvfg3TEfuqmmG4bh	65.65.65.65	61193	65.65.65.65	80	tcp	-	-	-	-	OTH	-	0	D	1	710	0	0	(empty)
+#close	2014-02-14-20-04-20

testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/weird.log

@@ -3,9 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-36-33
+#open	2014-02-13-20-36-35
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1363716396.798286	CXWv6p3arKYeMETxOg	55.247.223.174	27285	222.195.43.124	53	DNS_RR_unknown_type	-	F	bro
 1363716396.798374	CXWv6p3arKYeMETxOg	55.247.223.174	27285	222.195.43.124	53	dns_unmatched_reply	-	F	bro
-#close	2013-08-26-19-36-33
+1363716396.798374	-	-	-	-	-	dns_unmatched_msg	-	F	bro
+#close	2014-02-13-20-36-35

testing/btest/bifs/identify_data.bro

@@ -10,7 +10,7 @@ event bro_init()
 	print identify_data(a, T);
 
 	# PNG image
-	local b = "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a";
+	local b = "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00";
 	print identify_data(b, F);
 	print identify_data(b, T);
 	}

testing/btest/core/leaks/ayiya.test

@@ -5,4 +5,4 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/tunnels/ayiya3.trace
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30

testing/btest/core/leaks/basic-cluster.bro

@@ -9,7 +9,7 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m %INPUT
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 25
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {

testing/btest/core/leaks/bloomfilter.bro

@@ -5,7 +5,7 @@
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/wikipedia.trace %INPUT
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30
 
 function test_basic_bloom_filter()
   {

testing/btest/core/leaks/dataseries.bro

@@ -8,4 +8,4 @@
 #
 # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 25

testing/btest/core/leaks/file-analysis-http-get.bro

@@ -5,7 +5,7 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 25
 
 redef test_file_analysis_source = "HTTP";
 

testing/btest/core/leaks/gridftp.test

@@ -5,7 +5,7 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/globus-url-copy.trace %INPUT
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30
 
 @load base/protocols/ftp/gridftp
 

testing/btest/core/leaks/gtp_opt_header.test

@@ -5,7 +5,7 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/tunnels/gtp/gtp6_gtp_0x32.pcap %INPUT >out
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30
 
 # Some GTPv1 headers have some optional fields totaling to a 4-byte extension
 # of the mandatory header.

testing/btest/core/leaks/hll_cluster.bro

@@ -10,7 +10,7 @@
 # @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro runnumber=1 %INPUT
 # @TEST-EXEC: btest-bg-run worker-2 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro runnumber=2 %INPUT
-# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-bg-wait 25
 #
 # @TEST-EXEC: btest-diff manager-1/.stdout
 # @TEST-EXEC: btest-diff worker-1/.stdout

testing/btest/core/leaks/input-reread.bro

@@ -6,15 +6,15 @@
 #
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: sleep 10
 # @TEST-EXEC: cp input2.log input.log
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: sleep 10
 # @TEST-EXEC: cp input3.log input.log
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: sleep 10
 # @TEST-EXEC: cp input4.log input.log
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: sleep 10
 # @TEST-EXEC: cp input5.log input.log
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30
 
 @TEST-START-FILE input1.log
 #separator \x09

testing/btest/core/leaks/teredo.bro

@@ -5,7 +5,7 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/tunnels/Teredo.pcap %INPUT >output
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30
 
 function print_teredo(name: string, outer: connection, inner: teredo_hdr)
 	{

testing/btest/core/leaks/test-all.bro

@@ -5,4 +5,4 @@
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/wikipedia.trace test-all-policy
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 25

testing/btest/core/mpls-in-vlan.bro

@@ -0,0 +1,2 @@
+# @TEST-EXEC: bro -C -r $TRACES/mpls-in-vlan.trace
+# @TEST-EXEC: btest-diff conn.log