Merge remote-tracking branch 'origin/master' into topic/bernhard/file-analysis-x509

CHANGES

@@ -1,4 +1,85 @@
 
+2.2-221 | 2014-03-12 17:23:18 -0700
+
+  * Teach configure script --enable-jemalloc, --with-jemalloc.
+    Addresses BIT-1128. (Jon Siwek)
+
+2.2-218 | 2014-03-12 17:19:45 -0700
+
+  * Improve DBG_LOG macro (perf. improvement for --enable-debug mode).
+    (Jon Siwek)
+
+  * Silences some documentation warnings from Sphinx. (Jon Siwek)
+    
+2.2-215 | 2014-03-10 11:10:15 -0700
+
+  * Fix non-deterministic logging of unmatched DNS msgs. Addresses
+    BIT-1153 (Jon Siwek)
+
+2.2-213 | 2014-03-09 08:57:37 -0700
+
+  * No longer accidentally attempting to parse NBSTAT RRs as SRV RRs
+    in DNS analyzer. (Seth Hall)
+
+  * Fix DNS SRV responses and a small issue with NBNS queries and
+    label length. (Seth Hall)
+
+     - DNS SRV responses never had the code written to actually
+       generate the dns_SRV_reply event.  Adding this required
+       extending the event a bit to add extra information.  SRV responses
+       now appear in the dns.log file correctly.
+
+     - Fixed an issue where some Microsoft NetBIOS Name Service lookups
+       would exceed the max label length for DNS and cause an incorrect
+       "DNS_label_too_long" weird.
+
+2.2-210 | 2014-03-06 22:52:36 -0500
+
+  * Improve SSL logging so that connections are logged even when the
+    ssl_established event is not generated as well as other small SSL
+    fixes. (Bernhard Amann)
+
+2.2-206 | 2014-03-03 16:52:28 -0800
+
+  * HTTP CONNECT proxy support. The HTTP analyzer now supports
+    handling HTTP CONNECT proxies. (Seth Hall)
+
+  * Expanding the HTTP methods used in the DPD signature to detect
+    HTTP traffic. (Seth Hall)
+
+  * Fixing removal of support analyzers. (Robin Sommer)
+    
+2.2-199 | 2014-03-03 16:34:20 -0800
+
+  * Allow iterating over bif functions with result type vector of any.
+    This changes the internal type that is used to signal that a
+    vector is unspecified from any to void. Addresses BIT-1144
+    (Bernhard Amann)
+
+2.2-197 | 2014-02-28 15:36:58 -0800
+
+  * Remove test code. (Robin Sommer)
+
+2.2-194 | 2014-02-28 14:50:53 -0800
+
+  * Remove packet sorter. Addresses BIT-700. (Bernhard Amann)
+    
+2.2-192 | 2014-02-28 09:46:43 -0800
+
+  * Update Mozilla root bundle. (Bernhard Amann)
+    
+2.2-190 | 2014-02-27 07:34:44 -0800
+
+  * Adjust timings of a few leak tests. (Bernhard Amann)
+
+2.2-187 | 2014-02-25 07:24:42 -0800
+
+  * More Google TLS extensions that are being actively used. (Bernhard
+    Amann)
+
+  * Remove unused, and potentially unsafe, function
+    ListVal::IncludedInString. (Bernhard Amann)
+
 2.2-184 | 2014-02-24 07:28:18 -0800
 
   * New TLS constants from

CMakeLists.txt

@@ -91,6 +91,10 @@ if (NOT BinPAC_ROOT_DIR AND
 endif ()
 FindRequiredPackage(BinPAC)
 
+if (ENABLE_JEMALLOC)
+    find_package(JeMalloc)
+endif ()
+
 if (MISSING_PREREQS)
     foreach (prereq ${MISSING_PREREQ_DESCS})
         message(SEND_ERROR ${prereq})
@@ -105,6 +109,7 @@ include_directories(BEFORE
                     ${BinPAC_INCLUDE_DIR}
                     ${LIBMAGIC_INCLUDE_DIR}
                     ${ZLIB_INCLUDE_DIR}
+                    ${JEMALLOC_INCLUDE_DIR}
 )
 
 # Optional Dependencies
@@ -184,6 +189,7 @@ set(brodeps
     ${BIND_LIBRARY}
     ${LIBMAGIC_LIBRARY}
     ${ZLIB_LIBRARY}
+    ${JEMALLOC_LIBRARIES}
     ${OPTLIBS}
 )
 
@@ -268,6 +274,7 @@ message(
     "\ngperftools found:  ${HAVE_PERFTOOLS}"
     "\n        tcmalloc:  ${USE_PERFTOOLS_TCMALLOC}"
     "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
+    "\njemalloc:          ${ENABLE_JEMALLOC}"
     "\ncURL:              ${USE_CURL}"
     "\n"
     "\nDataSeries:        ${USE_DATASERIES}"

NEWS

@@ -25,6 +25,9 @@ New Functionality
   parsing past the GRE header in between the delivery and payload IP
   packets.
 
+- The DNS analyzer now actually generates the dns_SRV_reply() event.
+  It had been documented before, yet was never raised.
+
 Changed Functionality
 ---------------------
 
@@ -47,6 +50,8 @@ Changed Functionality
 
   TODO: Update if we add a detector for filtered traces.
 
+- We have removed the packet sorter component.
+
 Bro 2.2
 =======
 

VERSION

@@ -1 +1 @@
-2.2-184
+2.2-221

aux/binpac

@@ -1 +1 @@
-Subproject commit 54b321009b750268526419bdbd841f421c839313
+Subproject commit fe271628492b7b837b3fbcf4626061c8b3568589

aux/bro-aux

@@ -1 +1 @@
-Subproject commit ebf9c0d88ae8230845b91f15755156f93ff21aa8
+Subproject commit d7ac87294f415b5ddf3fc81bcae29815d2f835b1

aux/broccoli

@@ -1 +1 @@
-Subproject commit 52ba12128e0673a09cbc7a68b8485f5d19030633
+Subproject commit 3138e5068eeeb374c39c3d3b05b482b84d1f6e9c

aux/broctl

@@ -1 +1 @@
-Subproject commit 66793ec3c602439e235bee705b654aefb7ac8dec
+Subproject commit 756eb3e5bd63a830cfb0fab3ab6a41115f02c05b

cmake

@@ -1 +1 @@
-Subproject commit e7a46cb82ee10aa522c4d88115baf10181277d20
+Subproject commit 58c64e663ca9f035f7741775acefce1f6c8d1ed3

configure

@@ -32,6 +32,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --enable-perftools     force use of Google perftools on non-Linux systems
                            (automatically on when perftools is present on Linux)
     --enable-perftools-debug use Google's perftools for debugging
+    --enable-jemalloc      link against jemalloc
     --enable-ruby          build ruby bindings for broccoli (deprecated)
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
@@ -54,6 +55,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
   Optional Packages in Non-Standard Locations:
     --with-geoip=PATH      path to the libGeoIP install root
     --with-perftools=PATH  path to Google Perftools install root
+    --with-jemalloc=PATH   path to jemalloc install root
     --with-python=PATH     path to Python interpreter
     --with-python-lib=PATH path to libpython
     --with-python-inc=PATH path to Python headers
@@ -105,6 +107,7 @@ append_cache_entry BRO_ETC_INSTALL_DIR  PATH   $prefix/etc
 append_cache_entry ENABLE_DEBUG         BOOL   false
 append_cache_entry ENABLE_PERFTOOLS     BOOL   false
 append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
+append_cache_entry ENABLE_JEMALLOC      BOOL false
 append_cache_entry BinPAC_SKIP_INSTALL  BOOL   true
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
 append_cache_entry INSTALL_AUX_TOOLS    BOOL   true
@@ -160,6 +163,9 @@ while [ $# -ne 0 ]; do
             append_cache_entry ENABLE_PERFTOOLS     BOOL   true
             append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL   true
             ;;
+        --enable-jemalloc)
+            append_cache_entry ENABLE_JEMALLOC     BOOL   true
+            ;;
         --disable-broccoli)
             append_cache_entry INSTALL_BROCCOLI     BOOL   false
             ;;
@@ -214,6 +220,10 @@ while [ $# -ne 0 ]; do
         --with-perftools=*)
             append_cache_entry GooglePerftools_ROOT_DIR PATH $optarg
             ;;
+        --with-jemalloc=*)
+            append_cache_entry JEMALLOC_ROOT_DIR    PATH    $optarg
+            append_cache_entry ENABLE_JEMALLOC      BOOL    true
+            ;;
         --with-python=*)
             append_cache_entry PYTHON_EXECUTABLE    PATH    $optarg
             ;;

doc/broids/index.rst

@@ -15,9 +15,9 @@ conditions specific to your particular case.
 In the following sections, we present a few examples of common uses of
 Bro as an IDS.
 
-------------------------------------------------
+-------------------------------------------------
 Detecting an FTP Brute-force Attack and Notifying
-------------------------------------------------
+-------------------------------------------------
 
 For the purpose of this exercise, we define FTP brute-forcing as too many
 rejected usernames and passwords occurring from a single address.  We

doc/index.rst

@@ -17,6 +17,8 @@ Introduction Section
 
 .. 
 
+.. _using-bro:
+
 Using Bro Section
 =================
 

doc/quickstart/index.rst

@@ -292,9 +292,10 @@ tweak the most basic options.  Here's some suggestions on what to explore next:
 
 * We only looked at how to change options declared in the notice framework,
   there's many more options to look at in other script packages.
-* Continue reading with :ref:`using-bro` chapter which goes into more
+* Continue reading with :ref:`Using Bro <using-bro>` chapter which goes
-  depth on working with Bro; then look at :ref:`writing-scripts` for
+  into more depth on working with Bro; then look at
-  learning how to start writing your own scripts.
+  :ref:`writing-scripts` for learning how to start writing your own
+  scripts.
 * Look at the scripts in ``$PREFIX/share/bro/policy`` for further ones
   you may want to load; you can browse their documentation at the
   :ref:`overview of script packages <script-packages>`.

doc/scripting/index.rst

@@ -345,13 +345,13 @@ keyword.  Unlike globals, constants can only be set or altered at
 parse time if the ``&redef`` attribute has been used.  Afterwards (in
 runtime) the constants are unalterable.  In most cases, re-definable
 constants are used in Bro scripts as containers for configuration
-options.  For example, the configuration option to log password
+options.  For example, the configuration option to log passwords
 decrypted from HTTP streams is stored in
-``HTTP::default_capture_password`` as shown in the stripped down
+:bro:see:`HTTP::default_capture_password` as shown in the stripped down
 excerpt from :doc:`/scripts/base/protocols/http/main.bro` below.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro
-   :lines: 8-10,19-21,120
+   :lines: 9-11,20-22,121
 
 Because the constant was declared with the ``&redef`` attribute, if we
 needed to turn this option on globally, we could do so by adding the

scripts/base/init-bare.bro

@@ -39,6 +39,14 @@ type count_set: set[count];
 ##    directly and then remove this alias.
 type index_vec: vector of count;
 
+## A vector of any, used by some builtin functions to store a list of varying
+## types.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type any_vec: vector of any;
+
 ## A vector of strings.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions
@@ -1035,13 +1043,6 @@ const rpc_timeout = 24 sec &redef;
 ## means "forever", which resists evasion, but can lead to state accrual.
 const frag_timeout = 0.0 sec &redef;
 
-## Time window for reordering packets. This is used for dealing with timestamp
-## discrepancy between multiple packet sources.
-##
-## .. note:: Setting this can have a major performance impact as now packets
-##    need to be potentially copied and buffered.
-const packet_sort_window = 0 usecs &redef;
-
 ## If positive, indicates the encapsulation header size that should
 ## be skipped. This applies to all packets.
 const encap_hdr_size = 0 &redef;

scripts/base/protocols/dns/main.bro

@@ -181,10 +181,9 @@ function log_unmatched_msgs_queue(q: Queue::Queue)
 function log_unmatched_msgs(msgs: PendingMessages)
 	{
 	for ( trans_id in msgs )
-		{
 		log_unmatched_msgs_queue(msgs[trans_id]);
-		delete msgs[trans_id];
+
-		}
+	msgs = PendingMessages();
 	}
 
 function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
@@ -360,7 +359,15 @@ event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qcla
 	# Note: I'm ignoring the name type for now.  Not sure if this should be
 	#       worked into the query/response in some fashion.
 	if ( c$id$resp_p == 137/udp )
+		{
 		query = decode_netbios_name(query);
+		if ( c$dns$qtype_name == "SRV" )
+			{
+			# The SRV RFC used the ID used for NetBios Status RRs.
+			# So if this is NetBios Name Service we name it correctly.
+			c$dns$qtype_name = "NBSTAT";
+			}
+		}
 	c$dns$query = query;
 	}
 
@@ -421,9 +428,9 @@ event dns_WKS_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
 	hook DNS::do_reply(c, msg, ans, "");
 	}
 
-event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
+event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer, target: string, priority: count, weight: count, p: count) &priority=5
 	{
-	hook DNS::do_reply(c, msg, ans, "");
+	hook DNS::do_reply(c, msg, ans, target);
 	}
 
 # TODO: figure out how to handle these

scripts/base/protocols/http/dpd.sig

@@ -1,6 +1,8 @@
+# List of HTTP headers pulled from:
+#   http://annevankesteren.nl/2007/10/http-methods
 signature dpd_http_client {
   ip-proto == tcp
-  payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/
+  payload /^[[:space:]]*(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH|BCOPY|BDELETE|BMOVE|BPROPFIND|BPROPPATCH|NOTIFY|POLL|SUBSCRIBE|UNSUBSCRIBE|X-MS-ENUMATTS|RPC_OUT_DATA|RPC_IN_DATA)[[:space:]]*/
   tcp-state originator
 }
 

scripts/base/protocols/http/main.bro

@@ -4,6 +4,7 @@
 
 @load base/utils/numbers
 @load base/utils/files
+@load base/frameworks/tunnels
 
 module HTTP;
 
@@ -217,6 +218,17 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p
 		c$http$info_code = code;
 		c$http$info_msg = reason;
 		}
+
+	if ( c$http?$method && c$http$method == "CONNECT" && code == 200 )
+		{
+		# Copy this conn_id and set the orig_p to zero because in the case of CONNECT
+		# proxies there will be potentially many source ports since a new proxy connection
+		# is established for each proxied connection.  We treat this as a singular
+		# "tunnel".
+		local tid = copy(c$id);
+		tid$orig_p = 0/tcp;
+		Tunnel::register([$cid=tid, $tunnel_type=Tunnel::HTTP]);
+		}
 	}
 
 event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=5

scripts/base/protocols/ssl/consts.bro

@@ -89,6 +89,8 @@ export {
 		[13175] = "origin_bound_certificates",
 		[13180] = "encrypted_client_certificates",
 		[30031] = "channel_id",
+		[30032] = "channel_id_new",
+		[35655] = "padding",
 		[65281] = "renegotiation_info"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
 	

scripts/base/protocols/ssl/main.bro

@@ -31,6 +31,14 @@ export {
 		## to each connection.  It is not used for logging since it's a
 		## meaningless arbitrary number.
 		analyzer_id:      count            &optional;
+
+		## Flag to indicate if this ssl session has been established
+		## succesfully, or if it was aborted during the handshake.
+		established:  bool &log &default=F;
+
+		## Flag to indicate if this record already has been logged, to
+		## prevent duplicates.
+		logged:  bool  &default=F;
 	};
 
 	## The default root CA bundle.  By default, the mozilla-ca-list.bro
@@ -99,9 +107,13 @@ function undelay_log(info: Info, token: string)
 
 function log_record(info: Info)
 	{
+	if ( info$logged )
+		return;
+
 	if ( ! info?$delay_tokens || |info$delay_tokens| == 0 )
 		{
 		Log::write(SSL::LOG, info);
+		info$logged = T;
 		}
 	else
 		{
@@ -118,11 +130,14 @@ function log_record(info: Info)
 		}
 	}
 
-function finish(c: connection)
+# remove_analyzer flag is used to prevent disabling analyzer for finished
+# connections.
+function finish(c: connection, remove_analyzer: bool)
 	{
 	log_record(c$ssl);
-	if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
+	if ( remove_analyzer && disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
 		disable_analyzer(c$id, c$ssl$analyzer_id);
+		delete c$ssl$analyzer_id;
 	}
 
 event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) &priority=5
@@ -160,23 +175,33 @@ event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priori
 event ssl_established(c: connection) &priority=7
 	{
 	set_session(c);
+	c$ssl$established = T;
 	}
 
 event ssl_established(c: connection) &priority=-5
 	{
-	finish(c);
+	finish(c, T);
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c?$ssl )
+		# called in case a SSL connection that has not been established terminates
+		finish(c, F);
 	}
 
 event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=5
 	{
-	# Check by checking for existence of c$ssl record.
+	if ( atype == Analyzer::ANALYZER_SSL ) 
-	if ( c?$ssl && atype == Analyzer::ANALYZER_SSL )
+		{
+		set_session(c);
 		c$ssl$analyzer_id = aid;
 		}
+	}
 
 event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
                          reason: string) &priority=5
 	{
 	if ( c?$ssl )
-		finish(c);
+		finish(c, T);
 	}

src/CMakeLists.txt

@@ -293,7 +293,6 @@ set(bro_SRCS
     OpaqueVal.cc
     OSFinger.cc
     PacketFilter.cc
-    PacketSort.cc
     PersistenceSerializer.cc
     PktSrc.cc
     PolicyFile.cc

src/DebugLogger.h

@@ -33,10 +33,12 @@ enum DebugStream {
 	NUM_DBGS // Has to be last
 };
 
-#define DBG_LOG(args...) debug_logger.Log(args)
+#define DBG_LOG(stream, args...) \
-#define DBG_LOG_VERBOSE(args...) \
+	if ( debug_logger.IsEnabled(stream) ) \
-	if ( debug_logger.IsVerbose() ) \
+		debug_logger.Log(stream, args)
-		debug_logger.Log(args)
+#define DBG_LOG_VERBOSE(stream, args...) \
+	if ( debug_logger.IsVerbose() && debug_logger.IsEnabled(stream) ) \
+		debug_logger.Log(stream, args)
 #define DBG_PUSH(stream) debug_logger.PushIndent(stream)
 #define DBG_POP(stream) debug_logger.PopIndent(stream)
 

src/Expr.cc

@@ -3819,7 +3819,9 @@ VectorConstructorExpr::VectorConstructorExpr(ListExpr* constructor_list,
 		if ( constructor_list->Exprs().length() == 0 )
 			{
 			// vector().
-			SetType(new ::VectorType(base_type(TYPE_ANY)));
+			// By default, assign VOID type here. A vector with
+			// void type set is seen as an unspecified vector.
+			SetType(new ::VectorType(base_type(TYPE_VOID)));
 			return;
 			}
 

src/Net.cc

@@ -27,7 +27,6 @@
 #include "Reporter.h"
 #include "Net.h"
 #include "Anon.h"
-#include "PacketSort.h"
 #include "Serializer.h"
 #include "PacketDumper.h"
 
@@ -58,8 +57,6 @@ double bro_start_network_time;	// timestamp of first packet
 double last_watchdog_proc_time = 0.0;	// value of above during last watchdog
 bool terminating = false;	// whether we're done reading and finishing up
 
-PacketSortGlobalPQ* packet_sorter = 0;
-
 const struct pcap_pkthdr* current_hdr = 0;
 const u_char* current_pkt = 0;
 int current_dispatched = 0;
@@ -286,9 +283,6 @@ void net_init(name_list& interfaces, name_list& readfiles,
 
 	init_ip_addr_anonymizers();
 
-	if ( packet_sort_window > 0 )
-		packet_sorter = new PacketSortGlobalPQ();
-
 	sessions = new NetSessions();
 
 	if ( do_watchdog )
@@ -313,7 +307,7 @@ void expire_timers(PktSrc* src_ps)
 
 void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps, PacketSortElement* pkt_elem)
+			PktSrc* src_ps)
 	{
 	if ( ! bro_start_network_time )
 		bro_start_network_time = t;
@@ -351,7 +345,7 @@ void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			}
 		}
 
-	sessions->DispatchPacket(t, hdr, pkt, hdr_size, src_ps, pkt_elem);
+	sessions->DispatchPacket(t, hdr, pkt, hdr_size, src_ps);
 	mgr.Drain();
 
 	if ( sp )
@@ -367,62 +361,11 @@ void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 	current_pktsrc = 0;
 	}
 
-int process_packet_sorter(double latest_packet_time)
-	{
-	if ( ! packet_sorter )
-		return 0;
-
-	double min_t = latest_packet_time - packet_sort_window;
-
-	int num_pkts_dispatched = 0;
-	PacketSortElement* pkt_elem;
-
-	// Dispatch packets in the packet_sorter until timestamp min_t.
-	// It's possible that zero or multiple packets are dispatched.
-	while ( (pkt_elem = packet_sorter->RemoveMin(min_t)) != 0 )
-		{
-		net_packet_dispatch(pkt_elem->TimeStamp(),
-			pkt_elem->Hdr(), pkt_elem->Pkt(),
-			pkt_elem->HdrSize(), pkt_elem->Src(),
-			pkt_elem);
-		++num_pkts_dispatched;
-		delete pkt_elem;
-		}
-
-	return num_pkts_dispatched;
-	}
-
-void net_packet_arrival(double t, const struct pcap_pkthdr* hdr,
-			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps)
-	{
-	if ( packet_sorter )
-		{
-		// Note that when we enable packet sorter, there will
-		// be a small window between the time packet arrives
-		// to Bro and when it is processed ("dispatched").  We
-		// define network_time to be the latest timestamp for
-		// packets *dispatched* so far (usually that's the
-		// timestamp of the current packet).
-
-		// Add the packet to the packet_sorter.
-		packet_sorter->Add(
-			new PacketSortElement(src_ps, t, hdr, pkt, hdr_size));
-
-		// Do we have any packets to dispatch from packet_sorter?
-		process_packet_sorter(t);
-		}
-	else
-		// Otherwise we dispatch the packet immediately
-		net_packet_dispatch(t, hdr, pkt, hdr_size, src_ps, 0);
-	}
-
 void net_run()
 	{
 	set_processing_status("RUNNING", "net_run");
 
 	while ( io_sources.Size() ||
-		(packet_sorter && ! packet_sorter->Empty()) ||
 		(BifConst::exit_only_after_terminate && ! terminating) )
 		{
 		double ts;
@@ -445,14 +388,12 @@ void net_run()
 		current_iosrc = src;
 
 		if ( src )
-			src->Process();	// which will call net_packet_arrival()
+			src->Process();	// which will call net_packet_dispatch()
 
 		else if ( reading_live && ! pseudo_realtime)
 			{ // live but  no source is currently active
 			double ct = current_time();
-			if ( packet_sorter && ! packet_sorter->Empty() )
+			if ( ! net_is_processing_suspended() )
-				process_packet_sorter(ct);
-			else if ( ! net_is_processing_suspended() )
 				{
 				// Take advantage of the lull to get up to
 				// date on timers and events.
@@ -462,15 +403,6 @@ void net_run()
 				}
 			}
 
-		else if ( packet_sorter && ! packet_sorter->Empty() )
-			{
-			// We are no longer reading live; done with all the
-			// sources.
-			// Drain packets remaining in the packet sorter.
-			process_packet_sorter(
-				network_time + packet_sort_window + 1000000);
-			}
-
 		else if ( (have_pending_timers || using_communication) &&
 			  ! pseudo_realtime )
 			{
@@ -581,7 +513,6 @@ void net_delete()
 	set_processing_status("TERMINATING", "net_delete");
 
 	delete sessions;
-	delete packet_sorter;
 
 	for ( int i = 0; i < NUM_ADDR_ANONYMIZATION_METHODS; ++i )
 		delete ip_anonymizer[i];

src/Net.h

@@ -20,7 +20,7 @@ extern void net_run();
 extern void net_get_final_stats();
 extern void net_finish(int drain_events);
 extern void net_delete();	// Reclaim all memory, etc.
-extern void net_packet_arrival(double t, const struct pcap_pkthdr* hdr,
+extern void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
 			PktSrc* src_ps);
 extern int net_packet_match(BPF_Program* fp, const u_char* pkt,

src/NetVar.cc

@@ -153,8 +153,6 @@ int table_incremental_step;
 
 RecordType* packet_type;
 
-double packet_sort_window;
-
 double connection_status_update_interval;
 
 StringVal* state_dir;
@@ -475,8 +473,6 @@ void init_net_var()
 
 	packet_type = internal_type("packet")->AsRecordType();
 
-	packet_sort_window = opt_internal_double("packet_sort_window");
-
 	orig_addr_anonymization = opt_internal_int("orig_addr_anonymization");
 	resp_addr_anonymization = opt_internal_int("resp_addr_anonymization");
 	other_addr_anonymization = opt_internal_int("other_addr_anonymization");

src/NetVar.h

@@ -156,8 +156,6 @@ extern int table_incremental_step;
 
 extern RecordType* packet_type;
 
-extern double packet_sort_window;
-
 extern int orig_addr_anonymization, resp_addr_anonymization;
 extern int other_addr_anonymization;
 extern TableVal* preserve_orig_addr;

src/PacketSort.cc

@@ -1,364 +0,0 @@
-#include "IP.h"
-#include "PacketSort.h"
-
-const bool DEBUG_packetsort = false;
-
-PacketSortElement::PacketSortElement(PktSrc* arg_src,
-			double arg_timestamp, const struct pcap_pkthdr* arg_hdr,
-			const u_char* arg_pkt, int arg_hdr_size)
-	{
-	src = arg_src;
-	timestamp = arg_timestamp;
-	hdr = *arg_hdr;
-	hdr_size = arg_hdr_size;
-
-	pkt = new u_char[hdr.caplen];
-	memcpy(pkt, arg_pkt, hdr.caplen);
-
-	is_tcp = 0;
-	ip_hdr = 0;
-	tcp_flags = 0;
-	endp = 0;
-	payload_length = 0;
-	key = 0;
-
-	// Now check if it is a "parsable" TCP packet.
-	uint32 caplen = hdr.caplen;
-	uint32 tcp_offset;
-
-	if ( caplen >= sizeof(struct ip) + hdr_size )
-		{
-		const struct ip* ip = (const struct ip*) (pkt + hdr_size);
-		if ( ip->ip_v == 4 )
-			ip_hdr = new IP_Hdr(ip, false);
-		else if ( ip->ip_v == 6 && (caplen >= sizeof(struct ip6_hdr) + hdr_size) )
-			ip_hdr = new IP_Hdr((const struct ip6_hdr*) ip, false, caplen - hdr_size);
-		else
-			// Weird will be generated later in NetSessions::NextPacket.
-			return;
-
-		if ( ip_hdr->NextProto() == IPPROTO_TCP &&
-		      // Note: can't sort fragmented packets
-		     ( ! ip_hdr->IsFragment() ) )
-			{
-			tcp_offset = hdr_size + ip_hdr->HdrLen();
-			if ( caplen >= tcp_offset + sizeof(struct tcphdr) )
-				{
-				const struct tcphdr* tp = (const struct tcphdr*)
-							(pkt + tcp_offset);
-
-				id.src_addr = ip_hdr->SrcAddr();
-				id.dst_addr = ip_hdr->DstAddr();
-				id.src_port = tp->th_sport;
-				id.dst_port = tp->th_dport;
-				id.is_one_way = 0;
-
-				endp = addr_port_canon_lt(id.src_addr,
-							  id.src_port,
-							  id.dst_addr,
-							  id.dst_port) ? 0 : 1;
-
-				seq[endp] = ntohl(tp->th_seq);
-
-				if ( tp->th_flags & TH_ACK )
-					seq[1-endp] = ntohl(tp->th_ack);
-				else
-					seq[1-endp] = 0;
-
-				tcp_flags = tp->th_flags;
-
-				// DEBUG_MSG("%.6f: %u, %u\n", timestamp, seq[0], seq[1]);
-
-				payload_length = ip_hdr->PayloadLen() - tp->th_off * 4;
-
-				key = BuildConnIDHashKey(id);
-
-				is_tcp = 1;
-				}
-			}
-		}
-
-	if ( DEBUG_packetsort && ! is_tcp )
-		DEBUG_MSG("%.6f non-TCP packet\n", timestamp);
-	}
-
-PacketSortElement::~PacketSortElement()
-	{
-	delete [] pkt;
-	delete ip_hdr;
-	delete key;
-	}
-
-int PacketSortPQ::Timestamp_Cmp(PacketSortElement* a, PacketSortElement* b)
-	{
-	double d = a->timestamp - b->timestamp;
-
-	if ( d > 0 ) return 1;
-	else if ( d < 0 ) return -1;
-	else return 0;
-	}
-
-int PacketSortPQ::UpdatePQ(PacketSortElement* prev_e, PacketSortElement* new_e)
-	{
-	int index = prev_e->pq_index[pq_level];
-
-	new_e->pq_index[pq_level] = index;
-	pq[index] = new_e;
-
-	if ( Cmp(prev_e, new_e) > 0 )
-		return FixUp(new_e, index);
-	else
-		{
-		FixDown(new_e, index);
-		return index == 0;
-		}
-	}
-
-int PacketSortPQ::AddToPQ(PacketSortElement* new_e)
-	{
-	int index = pq.size();
-
-	new_e->pq_index[pq_level] = index;
-	pq.push_back(new_e);
-
-	return FixUp(new_e, index);
-	}
-
-int PacketSortPQ::RemoveFromPQ(PacketSortElement* prev_e)
-	{
-	if ( pq.size() > 1 )
-		{
-		PacketSortElement* new_e = pq[pq.size() - 1];
-		pq.pop_back();
-		return UpdatePQ(prev_e, new_e);		
-		}
-	else
-		{
-		pq.pop_back();
-		return 1;
-		}
-	}
-
-void PacketSortPQ::Assign(int k, PacketSortElement* e)
-	{
-	pq[k] = e;
-	e->pq_index[pq_level] = k;
-	}
-
-PacketSortConnPQ::~PacketSortConnPQ()
-	{
-	// Delete elements only in ConnPQ (not in GlobalPQ) to avoid
-	// double delete.
-	for ( int i = 0; i < (int) pq.size(); ++i )
-		{
-		delete pq[i];
-		pq[i] = 0;
-		}
-	}
-
-int PacketSortConnPQ::Cmp(PacketSortElement* a, PacketSortElement* b)
-	{
-	// Note: here we do not distinguish between packets without
-	// an ACK and packets with seq/ack of 0. The later will sorted
-	// only by their timestamps.
-
-	if ( a->seq[0] && b->seq[0] && a->seq[0] != b->seq[0] )
-		return (a->seq[0] > b->seq[0]) ? 1 : -1;
-
-	else if ( a->seq[1] && b->seq[1] && a->seq[1] != b->seq[1] )
-		return (a->seq[1] > b->seq[1]) ? 1 : -1;
-
-	else
-		return Timestamp_Cmp(a, b);
-	}
-
-int PacketSortPQ::FixUp(PacketSortElement* e, int k)
-	{
-	if ( k == 0 )
-		{
-		Assign(0, e);
-		return 1;
-		}
-
-	int parent = (k-1) / 2;
-	if ( Cmp(pq[parent], e) > 0 )
-		{
-		Assign(k, pq[parent]);
-		return FixUp(e, parent);
-		}
-	else
-		{
-		Assign(k, e);
-		return 0;
-		}
-	}
-
-void PacketSortPQ::FixDown(PacketSortElement* e, int k)
-	{
-	uint32 kid = k * 2 + 1;
-
-	if ( kid >= pq.size() )
-		{
-		Assign(k, e);
-		return;
-		}
-
-	if ( kid + 1 < pq.size() && Cmp(pq[kid], pq[kid+1]) > 0 )
-		++kid;
-
-	if ( Cmp(e, pq[kid]) > 0 )
-		{
-		Assign(k, pq[kid]);
-		FixDown(e, kid);
-		}
-	else
-		Assign(k, e);
-	}
-
-
-int PacketSortConnPQ::Add(PacketSortElement* e)
-	{
-#if 0
-	int endp = e->endp;
-	uint32 end_seq = e->seq[endp] + e->payload_length;
-
-	int p = 1 - endp;
-	if ( (e->tcp_flags & TH_RST) && ! (e->tcp_flags & TH_ACK) )
-		{
-		DEBUG_MSG("%.6f %c: %u -> %u\n",
-			  e->TimeStamp(), (p == endp) ? 'S' : 'A',
-			  e->seq[p], next_seq[p]);
-		e->seq[p] = next_seq[p];
-		}
-
-	if ( end_seq > next_seq[endp] )
-		next_seq[endp] = end_seq;
-#endif
-
-	return AddToPQ(e);
-	}
-
-void PacketSortConnPQ::UpdateDeliveredSeq(int endp, int seq, int len, int ack)
-	{
-	if ( delivered_seq[endp] == 0 || delivered_seq[endp] == seq )
-		delivered_seq[endp] = seq + len;
-	if ( ack > delivered_seq[1 - endp] )
-		delivered_seq[endp] = ack;
-	}
-
-bool PacketSortConnPQ::IsContentGapSafe(PacketSortElement* e)
-	{
-	int ack = e->seq[1 - e->endp];
-	return ack <= delivered_seq[1 - e->endp];
-	}
-
-int PacketSortConnPQ::Remove(PacketSortElement* e)
-	{
-	int ret = RemoveFromPQ(e);
-	UpdateDeliveredSeq(e->endp, e->seq[e->endp], e->payload_length,
-				e->seq[1 - e->endp]);
-	return ret;
-	}
-
-static void DeleteConnPQ(void* p)
-	{
-	delete (PacketSortConnPQ*) p;
-	}
-
-PacketSortGlobalPQ::PacketSortGlobalPQ()
-	{
-	pq_level = GLOBAL_PQ;
-	conn_pq_table.SetDeleteFunc(DeleteConnPQ);
-	}
-
-PacketSortGlobalPQ::~PacketSortGlobalPQ()
-	{
-	// Destruction of PacketSortConnPQ will delete all conn_pq's.
-	}
-
-int PacketSortGlobalPQ::Add(PacketSortElement* e)
-	{
-	if ( e->is_tcp )
-		{
-		// TCP packets are sorted by sequence numbers
-		PacketSortConnPQ* conn_pq = FindConnPQ(e);
-		PacketSortElement* prev_min = conn_pq->Min();
-
-		if ( conn_pq->Add(e) )
-			{
-			ASSERT(conn_pq->Min() != prev_min);
-
-			if ( prev_min )
-				return UpdatePQ(prev_min, e);
-			else
-				return AddToPQ(e);
-			}
-
-		else
-			{
-			ASSERT(conn_pq->Min() == prev_min);
-			return 0;
-			}
-		}
-	else
-		return AddToPQ(e);
-	}
-
-PacketSortElement* PacketSortGlobalPQ::RemoveMin(double timestamp)
-	{
-	PacketSortElement* e = Min();
-
-	if ( ! e )
-		return 0;
-
-	if ( e->is_tcp )
-		{
-		PacketSortConnPQ* conn_pq = FindConnPQ(e);
-
-#if 0
-		// Note: the content gap safety check does not work
-		// because we remove the state for a connection once
-		// it has no packet in the priority queue.
-
-		// Do not deliver e if it arrives later than timestamp,
-		// and is not content-gap-safe.
-		if ( e->timestamp > timestamp &&
-		     ! conn_pq->IsContentGapSafe(e) )
-			return 0;
-#else
-		if ( e->timestamp > timestamp )
-			return 0;
-#endif
-
-		conn_pq->Remove(e);
-		PacketSortElement* new_e = conn_pq->Min();
-
-		if ( new_e )
-			UpdatePQ(e, new_e);
-		else
-			{
-			RemoveFromPQ(e);
-			conn_pq_table.Remove(e->key);
-			delete conn_pq;
-			}
-		}
-	else
-		RemoveFromPQ(e);
-
-	return e;
-	}
-
-PacketSortConnPQ* PacketSortGlobalPQ::FindConnPQ(PacketSortElement* e)
-	{
-	if ( ! e->is_tcp )
-		reporter->InternalError("cannot find a connection for an invalid id");
-
-	PacketSortConnPQ* pq = (PacketSortConnPQ*) conn_pq_table.Lookup(e->key);
-	if ( ! pq )
-		{
-		pq = new PacketSortConnPQ();
-		conn_pq_table.Insert(e->key, pq);
-		}
-
-	return pq;
-	}

src/PacketSort.h

@@ -1,132 +0,0 @@
-#ifndef packetsort_h
-#define packetsort_h
-
-// Timestamps can be imprecise and even inconsistent among packets
-// from different sources. This class tries to guess a "correct"
-// order by looking at TCP sequence numbers.
-//
-// In particular, it tries to eliminate "false" content gaps.
-
-#include "Dict.h"
-#include "Conn.h"
-
-enum {
-	CONN_PQ,
-	GLOBAL_PQ,
-	NUM_OF_PQ_LEVEL,
-};
-
-class PktSrc;
-
-class PacketSortElement {
-public:
-	PacketSortElement(PktSrc* src, double timestamp,
-				const struct pcap_pkthdr* hdr,
-				const u_char* pkt, int hdr_size);
-	~PacketSortElement();
-
-	PktSrc* Src() const			{ return src; }
-	double TimeStamp() const		{ return timestamp; }
-	const struct pcap_pkthdr* Hdr() const	{ return &hdr; }
-	const u_char* Pkt() const		{ return pkt; }
-	int HdrSize() const			{ return hdr_size; }
-	const IP_Hdr* IPHdr() const		{ return ip_hdr; }
-
-protected:
-	PktSrc* src;
-	double timestamp;
-	struct pcap_pkthdr hdr;
-	u_char* pkt;
-	int hdr_size;
-
-	IP_Hdr* ip_hdr;
-	int is_tcp;
-	ConnID id;
-	uint32 seq[2];	// indexed by endpoint
-	int tcp_flags;
-	int endp;	// 0 or 1
-	int payload_length;
-
-	HashKey* key;
-
-	int pq_index[NUM_OF_PQ_LEVEL];
-
-	friend class PacketSortPQ;
-	friend class PacketSortConnPQ;
-	friend class PacketSortGlobalPQ;
-};
-
-class PacketSortPQ {
-public:
-	PacketSortPQ()
-		{ pq_level = -1; }
-	virtual ~PacketSortPQ() {}
-
-	PacketSortElement* Min() const	{ return (pq.size() > 0) ? pq[0] : 0; }
-
-protected:
-	virtual int Cmp(PacketSortElement* a, PacketSortElement* b) = 0;
-	int Timestamp_Cmp(PacketSortElement* a, PacketSortElement* b);
-
-	int UpdatePQ(PacketSortElement* prev_e, PacketSortElement* new_e);
-	int AddToPQ(PacketSortElement* e);
-	int RemoveFromPQ(PacketSortElement* e);
-
-	void Assign(int k, PacketSortElement* e);
-	int FixUp(PacketSortElement* e, int k);
-	void FixDown(PacketSortElement* e, int k);
-
-	vector<PacketSortElement*> pq;
-	int pq_level;
-};
-
-// Sort by sequence numbers within a connection
-class PacketSortConnPQ : public PacketSortPQ {
-public:
-	PacketSortConnPQ()
-		{
-		pq_level = CONN_PQ;
-		delivered_seq[0] = delivered_seq[1] = 0;
-		}
-	~PacketSortConnPQ();
-
-	int Add(PacketSortElement* e);
-
-	int Remove(PacketSortElement* e);
-
-	bool IsContentGapSafe(PacketSortElement* e);
-
-protected:
-	int Cmp(PacketSortElement* a, PacketSortElement* b);
-	void UpdateDeliveredSeq(int endp, int seq, int len, int ack);
-
-	int delivered_seq[2];
-};
-
-declare(PDict, PacketSortConnPQ);
-
-// Sort by timestamps.
-class PacketSortGlobalPQ : public PacketSortPQ {
-public:
-	PacketSortGlobalPQ();
-	~PacketSortGlobalPQ();
-
-	int Add(PacketSortElement* e);
-
-	int Empty() const { return conn_pq_table.Length() == 0; }
-
-	// Returns the next packet to dispatch if it arrives earlier than the
-	// given timestamp, otherwise returns 0.
-	// The packet, if to be returned, is also removed from the
-	// priority queue.
-	PacketSortElement* RemoveMin(double timestamp);
-
-protected:
-	int Cmp(PacketSortElement* a, PacketSortElement* b)
-		{ return Timestamp_Cmp(a, b); }
-	PacketSortConnPQ* FindConnPQ(PacketSortElement* e);
-
-	PDict(PacketSortConnPQ) conn_pq_table;
-};
-
-#endif

src/PktSrc.cc

@@ -317,13 +317,13 @@ void PktSrc::Process()
 	if ( pseudo_realtime )
 		{
 		current_pseudo = CheckPseudoTime();
-		net_packet_arrival(current_pseudo, &hdr, data, pkt_hdr_size, this);
+		net_packet_dispatch(current_pseudo, &hdr, data, pkt_hdr_size, this);
 		if ( ! first_wallclock )
 			first_wallclock = current_time(true);
 		}
 
 	else
-		net_packet_arrival(current_timestamp, &hdr, data, pkt_hdr_size, this);
+		net_packet_dispatch(current_timestamp, &hdr, data, pkt_hdr_size, this);
 
 	data = 0;
 	}

src/RemoteSerializer.cc

@@ -1466,7 +1466,7 @@ void RemoteSerializer::Process()
 		current_pkt = p->pkt;
 		current_pktsrc = 0;
 		current_iosrc = this;
-		sessions->NextPacket(p->time, p->hdr, p->pkt, p->hdr_size, 0);
+		sessions->NextPacket(p->time, p->hdr, p->pkt, p->hdr_size);
 		mgr.Drain();
 
 		current_hdr = 0;	// done with these

src/Sessions.cc

@@ -30,7 +30,6 @@
 #include "Discard.h"
 #include "RuleMatcher.h"
 
-#include "PacketSort.h"
 #include "TunnelEncapsulation.h"
 
 #include "analyzer/Manager.h"
@@ -168,7 +167,7 @@ void NetSessions::Done()
 
 void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps, PacketSortElement* pkt_elem)
+			PktSrc* src_ps)
 	{
 	const struct ip* ip_hdr = 0;
 	const u_char* ip_data = 0;
@@ -186,14 +185,13 @@ void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 		hdr_size += encap_hdr_size;
 
 	if ( src_ps->FilterType() == TYPE_FILTER_NORMAL )
-		NextPacket(t, hdr, pkt, hdr_size, pkt_elem);
+		NextPacket(t, hdr, pkt, hdr_size);
 	else
 		NextPacketSecondary(t, hdr, pkt, hdr_size, src_ps);
 	}
 
 void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
-			     const u_char* const pkt, int hdr_size,
+			     const u_char* const pkt, int hdr_size)
-			     PacketSortElement* pkt_elem)
 	{
 	SegmentProfiler(segment_logger, "processing-packet");
 	if ( pkt_profiler )
@@ -206,17 +204,6 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
 	if ( record_all_packets )
 		DumpPacket(hdr, pkt);
 
-	if ( pkt_elem && pkt_elem->IPHdr() )
-		// Fast path for "normal" IP packets if an IP_Hdr is
-		// already extracted when doing PacketSort. Otherwise
-		// the code below tries to extract the IP header, the
-		// difference here is that header extraction in
-		// PacketSort does not generate Weird events.
-
-		DoNextPacket(t, hdr, pkt_elem->IPHdr(), pkt, hdr_size, 0);
-
-	else
-		{
 	// ### The following isn't really correct.  What we *should*
 	// do is understanding the different link layers in order to
 	// find the network-layer protocol ID.  That's a big
@@ -270,7 +257,6 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
 		Weird("unknown_packet_type", hdr, pkt);
 		return;
 		}
-		}
 
 	if ( dump_this_packet && ! record_all_packets )
 		DumpPacket(hdr, pkt);

src/Sessions.h

@@ -28,7 +28,6 @@ declare(PDict,FragReassembler);
 
 class Discarder;
 class PacketFilter;
-class PacketSortElement;
 
 namespace analyzer { namespace stepping_stone { class SteppingStoneManager; } }
 namespace analyzer { namespace arp { class ARP_Analyzer; } }
@@ -74,7 +73,7 @@ public:
 	// employing the packet sorter first.
 	void DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 			const u_char* const pkt, int hdr_size,
-			PktSrc* src_ps, PacketSortElement* pkt_elem);
+			PktSrc* src_ps);
 
 	void Done();	// call to drain events before destructing
 
@@ -220,8 +219,7 @@ protected:
 				uint8 tcp_flags, bool& flip_roles);
 
 	void NextPacket(double t, const struct pcap_pkthdr* hdr,
-			const u_char* const pkt, int hdr_size,
+			const u_char* const pkt, int hdr_size);
-			PacketSortElement* pkt_elem);
 
 	void NextPacketSecondary(double t, const struct pcap_pkthdr* hdr,
 			const u_char* const pkt, int hdr_size,

src/Type.cc

@@ -1626,6 +1626,23 @@ VectorType::~VectorType()
 	Unref(yield_type);
 	}
 
+BroType* VectorType::YieldType()
+	{
+	// Work around the fact that we use void internally to mark a vector
+	// as being unspecified. When looking at its yield type, we need to
+	// return any as that's what other code historically expects for type
+	// comparisions.
+	if ( IsUnspecifiedVector() )
+		{
+		BroType* ret = ::base_type(TYPE_ANY);
+		Unref(ret); // unref, because this won't be held by anyone.
+		assert(ret);
+		return ret;
+		}
+
+	return yield_type;
+	}
+
 int VectorType::MatchesIndex(ListExpr*& index) const
 	{
 	expr_list& el = index->Exprs();
@@ -1645,7 +1662,7 @@ int VectorType::MatchesIndex(ListExpr*& index) const
 
 bool VectorType::IsUnspecifiedVector() const
 	{
-	return yield_type->Tag() == TYPE_ANY;
+	return yield_type->Tag() == TYPE_VOID;
 	}
 
 IMPLEMENT_SERIAL(VectorType, SER_VECTOR_TYPE);

src/Type.h

@@ -585,7 +585,7 @@ class VectorType : public BroType {
 public:
 	VectorType(BroType* t);
 	virtual ~VectorType();
-	BroType* YieldType()	{ return yield_type; }
+	BroType* YieldType();
 
 	int MatchesIndex(ListExpr*& index) const;
 

src/Val.cc

@@ -1171,23 +1171,6 @@ ListVal::~ListVal()
 	Unref(type);
 	}
 
-const char* ListVal::IncludedInString(const char* str) const
-	{
-	if ( tag != TYPE_STRING )
-		Internal("non-string list in ListVal::IncludedInString");
-
-	loop_over_list(vals, i)
-		{
-		const char* vs = (const char*) (vals[i]->AsString()->Bytes());
-
-		const char* embedded = strstr(str, vs);
-		if ( embedded )
-			return embedded;
-		}
-
-	return 0;
-	}
-
 RE_Matcher* ListVal::BuildRE() const
 	{
 	if ( tag != TYPE_STRING )
@@ -3139,7 +3122,7 @@ bool VectorVal::DoUnserialize(UnserialInfo* info)
 	for ( int i = 0; i < len; ++i )
 		{
 		Val* v;
-		UNSERIALIZE_OPTIONAL(v, Val::Unserialize(info, TYPE_ANY));
+		UNSERIALIZE_OPTIONAL(v, Val::Unserialize(info, TYPE_ANY)); // accept any type
 		Assign(i, v);
 		}
 

src/Val.h

@@ -656,6 +656,8 @@ protected:
 	DECLARE_SERIAL(PatternVal);
 };
 
+// ListVals are mainly used to index tables that have more than one 
+// element in their index.
 class ListVal : public Val {
 public:
 	ListVal(TypeTag t);
@@ -669,13 +671,6 @@ public:
 	Val* Index(const int n)		{ return vals[n]; }
 	const Val* Index(const int n) const	{ return vals[n]; }
 
-	// Returns offset of where str includes one of the strings in this
-	// ListVal (which had better be a list of strings), nil if none.
-	//
-	// Assumes that all of the strings in the list are NUL-terminated
-	// and do not have any embedded NULs.
-	const char* IncludedInString(const char* str) const;
-
 	// Returns an RE_Matcher() that will match any string that
 	// includes embedded within it one of the patterns listed
 	// (as a string, e.g., "foo|bar") in this ListVal.

src/analyzer/Analyzer.cc

@@ -209,11 +209,11 @@ void Analyzer::NextPacket(int len, const u_char* data, bool is_orig, int seq,
 	if ( skip )
 		return;
 
-	// If we have support analyzers, we pass it to them.
+	SupportAnalyzer* next_sibling = FirstSupportAnalyzer(is_orig);
-	if ( is_orig && orig_supporters )
+
-		orig_supporters->NextPacket(len, data, is_orig, seq, ip, caplen);
+	if ( next_sibling )
-	else if ( ! is_orig && resp_supporters )
+		next_sibling->NextPacket(len, data, is_orig, seq, ip, caplen);
-		resp_supporters->NextPacket(len, data, is_orig, seq, ip, caplen);
+
 	else
 		{
 		try
@@ -232,11 +232,11 @@ void Analyzer::NextStream(int len, const u_char* data, bool is_orig)
 	if ( skip )
 		return;
 
-	// If we have support analyzers, we pass it to them.
+	SupportAnalyzer* next_sibling = FirstSupportAnalyzer(is_orig);
-	if ( is_orig && orig_supporters )
+
-		orig_supporters->NextStream(len, data, is_orig);
+	if ( next_sibling )
-	else if ( ! is_orig && resp_supporters )
+		next_sibling->NextStream(len, data, is_orig);
-		resp_supporters->NextStream(len, data, is_orig);
+
 	else
 		{
 		try
@@ -255,11 +255,11 @@ void Analyzer::NextUndelivered(int seq, int len, bool is_orig)
 	if ( skip )
 		return;
 
-	// If we have support analyzers, we pass it to them.
+	SupportAnalyzer* next_sibling = FirstSupportAnalyzer(is_orig);
-	if ( is_orig && orig_supporters )
+
-		orig_supporters->NextUndelivered(seq, len, is_orig);
+	if ( next_sibling )
-	else if ( ! is_orig && resp_supporters )
+		next_sibling->NextUndelivered(seq, len, is_orig);
-		resp_supporters->NextUndelivered(seq, len, is_orig);
+
 	else
 		{
 		try
@@ -278,11 +278,10 @@ void Analyzer::NextEndOfData(bool is_orig)
 	if ( skip )
 		return;
 
-	// If we have support analyzers, we pass it to them.
+	SupportAnalyzer* next_sibling = FirstSupportAnalyzer(is_orig);
-	if ( is_orig && orig_supporters )
+
-		orig_supporters->NextEndOfData(is_orig);
+	if ( next_sibling )
-	else if ( ! is_orig && resp_supporters )
+		next_sibling->NextEndOfData(is_orig);
-		resp_supporters->NextEndOfData(is_orig);
 	else
 		EndOfData(is_orig);
 	}
@@ -558,31 +557,17 @@ void Analyzer::AddSupportAnalyzer(SupportAnalyzer* analyzer)
 
 void Analyzer::RemoveSupportAnalyzer(SupportAnalyzer* analyzer)
 	{
-	SupportAnalyzer** head =
+	DBG_LOG(DBG_ANALYZER, "%s disabled %s support analyzer %s",
-		analyzer->IsOrig() ? &orig_supporters : &resp_supporters;
-
-	SupportAnalyzer* prev = 0;
-	SupportAnalyzer* s;
-	for ( s = *head; s && s != analyzer; prev = s, s = s->sibling )
-		;
-
-	if ( ! s )
-		return;
-
-	if ( prev )
-		prev->sibling = s->sibling;
-	else
-		*head = s->sibling;
-
-	DBG_LOG(DBG_ANALYZER, "%s removed support %s",
 			fmt_analyzer(this).c_str(),
 			analyzer->IsOrig() ? "originator" : "responder",
 			fmt_analyzer(analyzer).c_str());
 
-	if ( ! analyzer->finished )
+	// We mark the analyzer as being removed here, which will prevent it
-		analyzer->Done();
+	// from being used further. However, we don't actually delete it
-
+	// before the parent gets destroyed. While we woulc do that, it's a
-	delete analyzer;
+	// bit tricky to do at the right time and it doesn't seem worth the
+	// trouble.
+	analyzer->removing = true;
 	return;
 	}
 
@@ -596,6 +581,19 @@ bool Analyzer::HasSupportAnalyzer(Tag tag, bool orig)
 	return false;
 	}
 
+SupportAnalyzer* Analyzer::FirstSupportAnalyzer(bool orig)
+	{
+	SupportAnalyzer* sa = orig ? orig_supporters : resp_supporters;
+
+	if ( ! sa )
+		return 0;
+
+	if ( ! sa->Removing() )
+		return sa;
+
+	return sa->Sibling(true);
+	}
+
 void Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig,
 				int seq, const IP_Hdr* ip, int caplen)
 	{
@@ -782,16 +780,35 @@ void Analyzer::Weird(const char* name, const char* addl)
 	conn->Weird(name, addl);
 	}
 
+SupportAnalyzer* SupportAnalyzer::Sibling(bool only_active) const
+	{
+	if ( ! only_active )
+		return sibling;
+
+	SupportAnalyzer* next = sibling;
+	while ( next && next->Removing() )
+		next = next->sibling;
+
+	return next;
+	}
+
 void SupportAnalyzer::ForwardPacket(int len, const u_char* data, bool is_orig,
 					int seq, const IP_Hdr* ip, int caplen)
 	{
 	// We do not call parent's method, as we're replacing the functionality.
+
 	if ( GetOutputHandler() )
+		{
 		GetOutputHandler()->DeliverPacket(len, data, is_orig, seq,
 							ip, caplen);
-	else if ( sibling )
+		return;
+		}
+
+	SupportAnalyzer* next_sibling = Sibling(true);
+
+	if ( next_sibling )
 		// Pass to next in chain.
-		sibling->NextPacket(len, data, is_orig, seq, ip, caplen);
+		next_sibling->NextPacket(len, data, is_orig, seq, ip, caplen);
 	else
 		// Finished with preprocessing - now it's the parent's turn.
 		Parent()->DeliverPacket(len, data, is_orig, seq, ip, caplen);
@@ -800,12 +817,18 @@ void SupportAnalyzer::ForwardPacket(int len, const u_char* data, bool is_orig,
 void SupportAnalyzer::ForwardStream(int len, const u_char* data, bool is_orig)
 	{
 	// We do not call parent's method, as we're replacing the functionality.
-	if ( GetOutputHandler() )
-		GetOutputHandler()->DeliverStream(len, data, is_orig);
 
-	else if ( sibling )
+	if ( GetOutputHandler() )
+		{
+		GetOutputHandler()->DeliverStream(len, data, is_orig);
+		return;
+		}
+
+	SupportAnalyzer* next_sibling = Sibling(true);
+
+	if ( next_sibling )
 		// Pass to next in chain.
-		sibling->NextStream(len, data, is_orig);
+		next_sibling->NextStream(len, data, is_orig);
 	else
 		// Finished with preprocessing - now it's the parent's turn.
 		Parent()->DeliverStream(len, data, is_orig);
@@ -814,12 +837,18 @@ void SupportAnalyzer::ForwardStream(int len, const u_char* data, bool is_orig)
 void SupportAnalyzer::ForwardUndelivered(int seq, int len, bool is_orig)
 	{
 	// We do not call parent's method, as we're replacing the functionality.
-	if ( GetOutputHandler() )
-		GetOutputHandler()->Undelivered(seq, len, is_orig);
 
-	else if ( sibling )
+	if ( GetOutputHandler() )
+		{
+		GetOutputHandler()->Undelivered(seq, len, is_orig);
+		return;
+		}
+
+	SupportAnalyzer* next_sibling = Sibling(true);
+
+	if ( next_sibling )
 		// Pass to next in chain.
-		sibling->NextUndelivered(seq, len, is_orig);
+		next_sibling->NextUndelivered(seq, len, is_orig);
 	else
 		// Finished with preprocessing - now it's the parent's turn.
 		Parent()->Undelivered(seq, len, is_orig);

src/analyzer/Analyzer.h

@@ -587,7 +587,7 @@ protected:
 	void RemoveTimer(Timer* t);
 
 	/**
-	 * Returnsn true if the analyzer has associated an SupportAnalyzer of a given type.
+	 * Returns true if the analyzer has associated an SupportAnalyzer of a given type.
 	 *
 	 * @param tag The type to check for.
 	 *
@@ -595,6 +595,14 @@ protected:
 	 */
 	bool HasSupportAnalyzer(Tag tag, bool orig);
 
+	/**
+	 * Returns the first still active support analyzer for the given
+	 * direction, or null if none.
+	 *
+	 * @param orig True if asking about the originator side.
+	 */
+	SupportAnalyzer* FirstSupportAnalyzer(bool orig);
+
 	/**
 	 * Adds a a new child analyzer with the option whether to intialize
 	 * it. This is an internal method.
@@ -616,6 +624,12 @@ protected:
 	 */
 	void AppendNewChildren();
 
+	/**
+	 * Returns true if the analyzer has been flagged for removal and
+	 * shouldn't be used otherwise anymore.
+	 */
+	bool Removing() const	{ return removing; }
+
 private:
 	// Internal method to eventually delete a child analyzer that's
 	// already Done().
@@ -718,6 +732,14 @@ public:
 	 */
 	bool IsOrig() const 	{ return orig; }
 
+	/**
+	 * Returns the analyzer's next sibling, or null if none.
+	 *
+	 * only_active: If true, this will skip siblings that are still link
+	 * but flagged for removal.
+	 */
+	SupportAnalyzer* Sibling(bool only_active = false) const;
+
 	/**
 	* Passes packet input to the next sibling SupportAnalyzer if any, or
 	* on to the associated main analyzer if none. If however there's an
@@ -749,11 +771,6 @@ public:
 	*/
 	virtual void ForwardUndelivered(int seq, int len, bool orig);
 
-	/**
-	 * Returns the analyzer next sibling, or null if none.
-	 */
-	SupportAnalyzer* Sibling() const 	{ return sibling; }
-
 protected:
 	friend class Analyzer;
 

src/analyzer/protocol/arp/ARP.h

@@ -24,7 +24,10 @@
 #endif
 
 #include "NetVar.h"
-#include "PacketSort.h"
+
+extern "C" {
+#include <pcap.h>
+}
 
 namespace analyzer { namespace arp {
 

src/analyzer/protocol/dns/DNS.cc

@@ -208,6 +208,7 @@ int DNS_Interpreter::ParseAnswer(DNS_MsgInfo* msg,
 	int name_len = sizeof(name) - 1;
 
 	u_char* name_end = ExtractName(data, len, name, name_len, msg_start);
+
 	if ( ! name_end )
 		return 0;
 
@@ -275,7 +276,17 @@ int DNS_Interpreter::ParseAnswer(DNS_MsgInfo* msg,
 			break;
 
 		case TYPE_SRV:
+			if ( ntohs(analyzer->Conn()->RespPort()) == 137 )
+				{
+				// This is an NBSTAT (NetBIOS NODE STATUS) record.
+				// The SRV RFC reused the value that was already being
+				// used for this.
+				// We aren't parsing this yet.
+				status = 1;
+				}
+			else
 				status = ParseRR_SRV(msg, data, len, rdlength, msg_start);
+
 			break;
 
 		case TYPE_EDNS:
@@ -400,7 +411,9 @@ int DNS_Interpreter::ExtractLabel(const u_char*& data, int& len,
 		return 0;
 		}
 
-	if ( label_len > 63 )
+	if ( label_len > 63 &&
+		// NetBIOS name service look ups can use longer labels.
+		ntohs(analyzer->Conn()->RespPort()) != 137 )
 		{
 		analyzer->Weird("DNS_label_too_long");
 		return 0;
@@ -633,15 +646,24 @@ int DNS_Interpreter::ParseRR_SRV(DNS_MsgInfo* msg,
 	u_char* name_end = ExtractName(data, len, name, name_len, msg_start);
 	if ( ! name_end )
 		return 0;
-	*name_end = 0;	// terminate name so we can use it in snprintf()
 
 	if ( data - data_start != rdlength )
 		analyzer->Weird("DNS_RR_length_mismatch");
 
-	// The following is just a placeholder.
+	if ( dns_SRV_reply && ! msg->skip_event )
-	char buf[2048];
+		{
-	safe_snprintf(buf, sizeof(buf), "SRV %s priority=%d weight=%d port=%d",
+		val_list* vl = new val_list;
-		      name, priority, weight, port);
+		vl->append(analyzer->BuildConnVal());
+		vl->append(msg->BuildHdrVal());
+		vl->append(msg->BuildAnswerVal());
+		vl->append(new StringVal(new BroString(name, name_end - name, 1)));
+		vl->append(new Val(priority, TYPE_COUNT));
+		vl->append(new Val(weight, TYPE_COUNT));
+		vl->append(new Val(port, TYPE_COUNT));
+
+		analyzer->ConnectionEvent(dns_SRV_reply, vl);
+		}
+
 	return 1;
 	}
 

src/analyzer/protocol/dns/events.bif

@@ -392,6 +392,12 @@ event dns_TXT_reply%(c: connection, msg: dns_msg, ans: dns_answer, str: string%)
 ##
 ## ans: The type-independent part of the parsed answer record.
 ##
+## priority: Priority of the SRV response.
+##
+## weight: Weight of the SRV response.
+##
+## p: Port of the SRV response.
+##
 ## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
 ##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
 ##    dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end dns_full_request
@@ -399,7 +405,7 @@ event dns_TXT_reply%(c: connection, msg: dns_msg, ans: dns_answer, str: string%)
 ##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
 ##    dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout
 ##    dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth
-event dns_SRV_reply%(c: connection, msg: dns_msg, ans: dns_answer%);
+event dns_SRV_reply%(c: connection, msg: dns_msg, ans: dns_answer, target: string, priority: count, weight: count, p: count%);
 
 ## Generated on DNS reply resource records when the type of record is not one
 ## that Bro knows how to parse and generate another more specific specific

src/analyzer/protocol/http/HTTP.cc

@@ -889,6 +889,9 @@ HTTP_Analyzer::HTTP_Analyzer(Connection* conn)
 	reply_code = 0;
 	reply_reason_phrase = 0;
 
+	connect_request = false;
+	pia = 0;
+
 	content_line_orig = new tcp::ContentLine_Analyzer(conn, true);
 	AddSupportAnalyzer(content_line_orig);
 
@@ -945,6 +948,14 @@ void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig)
 	if ( TCP() && TCP()->IsPartial() )
 		return;
 
+	if ( pia )
+		{
+		// There will be a PIA instance if this connection has been identified
+		// as a connect proxy.
+		ForwardStream(len, data, is_orig);
+		return;
+		}
+
 	const char* line = reinterpret_cast<const char*>(data);
 	const char* end_of_line = line + len;
 
@@ -1055,6 +1066,32 @@ void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig)
 
 				HTTP_Reply();
 
+				if ( connect_request && reply_code == 200 )
+					{
+					pia = new pia::PIA_TCP(Conn());
+
+					if ( AddChildAnalyzer(pia) )
+						{
+						pia->FirstPacket(true, 0);
+						pia->FirstPacket(false, 0);
+
+						// This connection has transitioned to no longer
+						// being http and the content line support analyzers
+						// need to be removed.
+						RemoveSupportAnalyzer(content_line_orig);
+						RemoveSupportAnalyzer(content_line_resp);
+
+						return;
+						}
+
+					else
+						{
+						// Shouldn't really happen.
+						delete pia;
+						pia = 0;
+						}
+					}
+
 				InitHTTPMessage(content_line,
 						reply_message, is_orig,
 						ExpectReplyMessageBody(),
@@ -1390,6 +1427,12 @@ void HTTP_Analyzer::HTTP_Request()
 	{
 	ProtocolConfirmation();
 
+	const char* method = (const char*) request_method->AsString()->Bytes();
+	int method_len = request_method->AsString()->Len();
+
+	if ( strcasecmp_n(method_len, method, "CONNECT") == 0 )
+		connect_request = true;
+
 	if ( http_request )
 		{
 		val_list* vl = new val_list;

src/analyzer/protocol/http/HTTP.h

@@ -5,6 +5,7 @@
 
 #include "analyzer/protocol/tcp/TCP.h"
 #include "analyzer/protocol/tcp/ContentLine.h"
+#include "analyzer/protocol/pia/PIA.h"
 #include "analyzer/protocol/zip/ZIP.h"
 #include "analyzer/protocol/mime/MIME.h"
 #include "binpac_bro.h"
@@ -237,6 +238,9 @@ protected:
 	int connection_close;
 	int request_ongoing, reply_ongoing;
 
+	bool connect_request;
+	pia::PIA_TCP *pia;
+
 	Val* request_method;
 
 	// request_URI is in the original form (may contain '%<hex><hex>'

src/analyzer/protocol/ssl/ssl-analyzer.pac

@@ -142,6 +142,8 @@ refine connection SSL_Conn += {
 		%{
 		if ( ! version_ok(version) )
 			bro_analyzer()->ProtocolViolation(fmt("unsupported client SSL version 0x%04x", version));
+		else
+			bro_analyzer()->ProtocolConfirmation();
 
 		if ( ssl_client_hello )
 			{
@@ -180,8 +182,6 @@ refine connection SSL_Conn += {
 		%{
 		if ( ! version_ok(version) )
 			bro_analyzer()->ProtocolViolation(fmt("unsupported server SSL version 0x%04x", version));
-		else
-			bro_analyzer()->ProtocolConfirmation();
 
 		if ( ssl_server_hello )
 			{

src/probabilistic/top-k.bif

@@ -49,7 +49,7 @@ function topk_add%(handle: opaque of topk, value: any%): any
 ##
 ## .. bro:see:: topk_init topk_add topk_count topk_epsilon
 ##    topk_size topk_sum topk_merge topk_merge_prune
-function topk_get_top%(handle: opaque of topk, k: count%): any
+function topk_get_top%(handle: opaque of topk, k: count%): any_vec
 	%{
 	assert(handle);
 	probabilistic::TopkVal* h = (probabilistic::TopkVal*) handle;

src/types.bif

@@ -186,6 +186,7 @@ enum Type %{
 	TEREDO,
 	SOCKS,
 	GTPv1,
+	HTTP,
 %}
 
 type EncapsulatingConn: record;

testing/btest/Baseline/bifs.topk/out

@@ -79,3 +79,12 @@
 0
 8
 0
+0, c
+1, e
+2, d
+0, c
+1, e
+2, d
+0, c
+1, e
+2, d

testing/btest/Baseline/language.vector-unspecified/output

@@ -0,0 +1 @@
+[5, Hi, 127.0.0.1]

testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
-#open	2013-08-26-19-47-01
+#open	2014-03-04-22-24-11
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject	established
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string	bool
-1348168976.508038	CXWv6p3arKYeMETxOg	192.168.57.103	60108	192.168.57.101	2811	TLSv10	TLS_RSA_WITH_AES_256_CBC_SHA	-	-	CN=host/alpha,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348161979.000000	1379697979.000000	-	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid
+1348168976.508038	CXWv6p3arKYeMETxOg	192.168.57.103	60108	192.168.57.101	2811	TLSv10	TLS_RSA_WITH_AES_256_CBC_SHA	-	-	CN=host/alpha,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348161979.000000	1379697979.000000	-	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	T
-1348168976.551422	CjhGID4nQcgTWjvg4c	192.168.57.103	35391	192.168.57.101	55968	TLSv10	TLS_RSA_WITH_NULL_SHA	-	-	CN=932373381,CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348168676.000000	1348206441.000000	-	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid
+1348168976.551422	CjhGID4nQcgTWjvg4c	192.168.57.103	35391	192.168.57.101	55968	TLSv10	TLS_RSA_WITH_NULL_SHA	-	-	CN=932373381,CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348168676.000000	1348206441.000000	-	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	T
-#close	2013-08-26-19-47-01
+#close	2014-03-04-22-24-11

testing/btest/Baseline/scripts.base.protocols.http.http-connect/conn.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2014-02-13-03-37-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1078232251.833846	CXWv6p3arKYeMETxOg	79.26.245.236	3378	254.228.86.79	8240	tcp	http,smtp	6.722274	1685	223	SF	-	0	ShADadfF	14	2257	16	944	(empty)
+#close	2014-02-13-03-37-02

testing/btest/Baseline/scripts.base.protocols.http.http-connect/http.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#open	2014-02-13-03-37-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	orig_fuids	orig_mime_types	resp_fuids	resp_mime_types
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	vector[string]	vector[string]	vector[string]	vector[string]
+1078232252.284420	CXWv6p3arKYeMETxOg	79.26.245.236	3378	254.228.86.79	8240	1	CONNECT	-	mailin03.sul.t-online.de:25 /	-	-	0	0	200	Connection established	-	-	-	(empty)	-	-	-	-	-	-	-
+#close	2014-02-13-03-37-02

testing/btest/Baseline/scripts.base.protocols.http.http-connect/smtp.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	smtp
+#open	2014-02-13-03-37-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	helo	mailfrom	rcptto	date	from	to	reply_to	msg_id	in_reply_to	subject	x_originating_ip	first_received	second_received	last_reply	path	user_agent	fuids
+#types	time	string	addr	port	addr	port	count	string	string	table[string]	string	string	table[string]	string	string	string	string	addr	string	string	string	vector[addr]	string	vector[string]
+1078232255.642953	CXWv6p3arKYeMETxOg	79.26.245.236	3378	254.228.86.79	8240	1	208.191.73.21	<nhfjenna_neumann@lycos.com>	<thenightwatch@t-online.de>	Tue, 2 Mar 2004 13:57:49 +0100	Sybille Ostermann <nhfjenna_neumann@lycos.com>	thenightwatch@t-online.de	-	-	-	Hier sind die dicken Girls hemmungloser denn je.. grcu	-	from mail.iosphere.net (mail.iosphere.net [216.58.97.33]) by mail.netsync.net with esmtp; Mrz, 02 2004 12:55:34 -0700	-	250 Message accepted.	254.228.86.79,79.26.245.236,216.58.97.33	Microsoft Outlook Build 10.0.2616	FVS9k93PUgScEUCOjd
+#close	2014-02-13-03-37-02

testing/btest/Baseline/scripts.base.protocols.http.http-connect/tunnel.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	tunnel
+#open	2014-02-13-03-37-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
+#types	time	string	addr	port	addr	port	enum	enum
+1078232252.284420	-	79.26.245.236	0	254.228.86.79	8240	Tunnel::HTTP	Tunnel::DISCOVER
+#close	2014-02-13-03-37-02

testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
-#open	2013-08-26-19-04-21
+#open	2014-03-04-22-02-50
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject	established
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string	bool
-1335538392.319381	CXWv6p3arKYeMETxOg	192.168.1.105	62045	74.125.224.79	443	TLSv10	TLS_ECDHE_RSA_WITH_RC4_128_SHA	ssl.gstatic.com	-	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	1334102677.000000	1365639277.000000	-	-	-
+1335538392.319381	CXWv6p3arKYeMETxOg	192.168.1.105	62045	74.125.224.79	443	TLSv10	TLS_ECDHE_RSA_WITH_RC4_128_SHA	ssl.gstatic.com	-	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	1334102677.000000	1365639277.000000	-	-	-	T
-#close	2013-08-26-19-04-21
+#close	2014-03-04-22-02-50

testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2-handshake-failure/ssl.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open	2014-03-04-21-57-58
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject	established
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string	bool
+1393957586.786031	CXWv6p3arKYeMETxOg	192.168.4.149	53525	74.125.239.37	443	-	-	-	-	-	-	-	-	handshake_failure	-	-	F
+#close	2014-03-04-21-57-58

testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2/ssl.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
-#open	2013-08-26-19-04-22
+#open	2014-03-04-22-03-00
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject	established
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string	bool
-1357328848.549370	CXWv6p3arKYeMETxOg	10.0.0.80	56637	68.233.76.12	443	TLSv12	TLS_RSA_WITH_RC4_128_MD5	-	-	CN=*.taleo.net,OU=Comodo PremiumSSL Wildcard,OU=Web,O=Taleo Inc.,street=4140 Dublin Boulevard,street=Suite 400,L=Dublin,ST=CA,postalCode=94568,C=US	CN=COMODO High-Assurance Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB	1304467200.000000	1467676799.000000	-	-	-
+1357328848.549370	CXWv6p3arKYeMETxOg	10.0.0.80	56637	68.233.76.12	443	TLSv12	TLS_RSA_WITH_RC4_128_MD5	-	-	CN=*.taleo.net,OU=Comodo PremiumSSL Wildcard,OU=Web,O=Taleo Inc.,street=4140 Dublin Boulevard,street=Suite 400,L=Dublin,ST=CA,postalCode=94568,C=US	CN=COMODO High-Assurance Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB	1304467200.000000	1467676799.000000	-	-	-	T
-#close	2013-08-26-19-04-22
+#close	2014-03-04-22-03-00

testing/btest/Baseline/scripts.policy.misc.dump-events/all-events-no-args.log

@@ -4,24 +4,24 @@
 1170717505.366729 filter_change_tracking
 1170717505.366729 new_connection
 1170717505.548308 connection_established
+1170717505.549109 protocol_confirmation
 1170717505.549109 ssl_client_hello
-1170717505.734145 protocol_confirmation
 1170717505.734145 ssl_server_hello
 1170717505.735416 x509_certificate
 1170717505.735416 x509_certificate
 1170717505.934612 ssl_established
 1170717508.515696 new_connection
 1170717508.696747 connection_established
+1170717508.697180 protocol_confirmation
 1170717508.697180 ssl_client_hello
-1170717508.881857 protocol_confirmation
 1170717508.881857 ssl_server_hello
 1170717508.883051 x509_certificate
 1170717508.883051 x509_certificate
 1170717509.082241 ssl_established
 1170717511.541455 new_connection
 1170717511.722589 connection_established
+1170717511.722913 protocol_confirmation
 1170717511.722913 ssl_client_hello
-1170717511.908619 protocol_confirmation
 1170717511.908619 ssl_server_hello
 1170717511.909717 x509_certificate
 1170717511.909717 x509_certificate

testing/btest/bifs/topk.bro

@@ -148,7 +148,17 @@ event bro_init()
 	print topk_count(k3, "d");
 	print topk_epsilon(k3, "d");
 
+	local styped: vector of count;
+	styped = topk_get_top(k3, 3);
+	for ( i in styped )
+	print i, styped[i];
 
+	local anytyped: vector of any;
+	anytyped = topk_get_top(k3, 3);
+	for ( i in anytyped )
+		print i, anytyped[i];
 
-
+	local suntyped = topk_get_top(k3, 3);
+	for ( i in suntyped )
+		print i, suntyped[i];
 }

testing/btest/core/leaks/ayiya.test

@@ -5,4 +5,4 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/tunnels/ayiya3.trace
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30

testing/btest/core/leaks/bloomfilter.bro

@@ -5,7 +5,7 @@
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/wikipedia.trace %INPUT
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30
 
 function test_basic_bloom_filter()
   {

testing/btest/core/leaks/gridftp.test

@@ -5,7 +5,7 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/globus-url-copy.trace %INPUT
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30
 
 @load base/protocols/ftp/gridftp
 

testing/btest/core/leaks/gtp_opt_header.test

@@ -5,7 +5,7 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/tunnels/gtp/gtp6_gtp_0x32.pcap %INPUT >out
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30
 
 # Some GTPv1 headers have some optional fields totaling to a 4-byte extension
 # of the mandatory header.

testing/btest/core/leaks/http-connect.bro

@@ -0,0 +1,14 @@
+# Needs perftools support.
+#
+# @TEST-GROUP: leaks
+#
+# @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
+#
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -b -m -r $TRACES/http/connect-with-smtp.trace %INPUT
+# @TEST-EXEC: btest-bg-wait 15
+
+@load base/protocols/conn
+@load base/protocols/http
+@load base/protocols/smtp
+@load base/protocols/tunnels
+@load base/frameworks/dpd

testing/btest/core/leaks/input-reread.bro

@@ -6,13 +6,13 @@
 #
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: sleep 10
 # @TEST-EXEC: cp input2.log input.log
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: sleep 10
 # @TEST-EXEC: cp input3.log input.log
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: sleep 10
 # @TEST-EXEC: cp input4.log input.log
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: sleep 10
 # @TEST-EXEC: cp input5.log input.log
 # @TEST-EXEC: btest-bg-wait 30
 

testing/btest/core/leaks/teredo.bro

@@ -5,7 +5,7 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/tunnels/Teredo.pcap %INPUT >output
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30
 
 function print_teredo(name: string, outer: connection, inner: teredo_hdr)
 	{

testing/btest/language/vector-unspecified.bro

@@ -0,0 +1,11 @@
+# @TEST-EXEC: bro -b %INPUT >output 2>&1
+# @TEST-EXEC: btest-diff output
+
+# Test assignment behavior of unspecified vectors
+local a = vector();
+
+a[0] = 5;
+a[1] = "Hi";
+a[2] = 127.0.0.1;
+
+print a;

testing/btest/scripts/base/protocols/http/http-connect.bro

@@ -0,0 +1,13 @@
+# This tests that the HTTP analyzer handles HTTP CONNECT proxying correctly.
+#
+# @TEST-EXEC: bro -r $TRACES/http/connect-with-smtp.trace %INPUT
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff http.log
+# @TEST-EXEC: btest-diff smtp.log
+# @TEST-EXEC: btest-diff tunnel.log
+
+@load base/protocols/conn
+@load base/protocols/http
+@load base/protocols/smtp
+@load base/protocols/tunnels
+@load base/frameworks/dpd

testing/btest/scripts/base/protocols/ssl/tls-1.2-handshake-failure.test

@@ -0,0 +1,2 @@
+# @TEST-EXEC: bro -r $TRACES/tls-1.2-handshake-failure.trace %INPUT
+# @TEST-EXEC: btest-diff ssl.log