Merge remote-tracking branch 'origin/master' into topic/dnthayer/ticket1963

.update-changes.cfg

@@ -0,0 +1,19 @@
+
+function new_version_hook
+    {
+    # This can be used to automatically adapt version numbers in files,
+    # except it's not used for that currently.  Instead it updates the files
+    # which store the commit hashes associated with the version of external
+    # test suite repos to check out on a CI system.
+    version=$1
+
+    if [ -d testing/external/bro-testing ]; then
+        ( cd testing/external/bro-testing && git fetch origin && git rev-parse origin/master ) > testing/external/commit-hash.bro-testing
+        git add testing/external/commit-hash.bro-testing
+    fi
+
+    if [ -d testing/external/bro-testing-private ]; then
+        ( cd testing/external/bro-testing-private && git fetch origin && git rev-parse origin/master ) > testing/external/commit-hash.bro-testing-private
+        git add testing/external/commit-hash.bro-testing-private
+    fi
+    }

CHANGES

@@ -1,4 +1,107 @@
 
+2.5-906 | 2018-08-24 14:57:55 -0500
+
+  * Stabilize a cluster logging unit test (Jon Siwek, Corelight)
+
+2.5-905 | 2018-08-24 10:21:35 -0500
+
+  * Detect MaxMind DB changes and auto-reload (Jonathan Perkins, Corelight)
+
+2.5-903 | 2018-08-23 16:54:24 -0500
+
+  * Fix finding of kerberos and libmaxminddb CMake < 3.3 (Daniel Thayer)
+
+  * BIT-1885: fix "kill" threading message (Jon Siwek, Corelight)
+
+2.5-900 | 2018-08-23 15:18:48 -0500
+
+  * Improve readability of the Travis job log (Daniel Thayer)
+
+  * Fix tracking of DCE-RPC context identifier mappings
+
+    This adds previously-missing support for "Alter Context"
+    request/response PDUs (initial patch contributed by Mark Fernandez).
+
+    Also, context ID arguments were added to dce_rpc_bind, dce_rpc_request,
+    and dce_rpc_response in order to properly track what endpoint/operation
+    a given opnum maps to. (Jon Siwek, Corelight)
+
+2.5-897 | 2018-08-23 15:53:16 +0000
+
+ * BIT-1885: Fix input framework memory leak. For input threads that
+   get joined during run-time, messages could remain in the thread's
+   queue and leak. (Jon Siwek, Corelight)
+
+ * Increase timeout for a memleak test. (Jon Siwek, Corelight)
+
+2.5-894 | 2018-08-22 12:05:19 -0500
+
+  * Ensure external test repo hashes track origin/master (Jon Siwek, Corelight)
+
+2.5-892 | 2018-08-22 11:49:12 -0500
+
+  * Fix "unused CMake variable" configuration warnings (Jon Siwek, Corelight)
+
+2.5-890 | 2018-08-21 16:47:52 -0500
+
+  * Fix Travis CI script to checkout particular commits of external tests
+    (Jon Siwek, Corelight)
+
+  * Fix signed/unsigned comparison warning (Jon Siwek, Corelight)
+
+2.5-888 | 2018-08-21 15:54:56 -0500
+
+  * Add --with-broker configure option (Jon Siwek, Corelight)
+
+2.5-887 | 2018-08-21 14:54:12 -0500
+
+  * Change default snaplen to 9216 bytes to better accommodate jumbo frames
+    (Justin Azoff)
+
+2.5-884 | 2018-08-20 15:39:21 -0500
+
+  * Fix outdated documentation test baselines (Jon Siwek, Corelight)
+
+  * Add 'smtp_excessive_pending_cmds' weird (Jon Siwek, Corelight)
+
+  * Fix SMTP command string comparisons (Jon Siwek, Corelight)
+
+  * Improve handling of empty lines in several text protocol analyzers
+    (Jon Siwek, Corelight)
+
+  * Add rate-limiting sampling mechanism for weird events
+
+    The generation of weird events, by default, are now rate-limited
+    according to these tunable options:
+
+      - Weird::sampling_whitelist
+      - Weird::sampling_threshold
+      - Weird::sampling_rate
+      - Weird::sampling_duration
+
+    The new get_reporter_stats() BIF also allows one to query the
+    total number of weirds generated (pre-sampling) which the new
+    policy/misc/weird-stats.bro script uses periodically to populate
+    a weird_stats.log.
+
+    There's also new reporter BIFs to allow generating weirds from the
+    script-layer such that they go through the same, internal
+    rate-limiting/sampling mechanisms:
+
+      - Reporter::conn_weird
+      - Reporter::flow_weird
+      - Reporter::net_weird
+
+    Some of the code was adapted from previous work by Johanna Amann.
+    (Jon Siwek, Corelight)
+
+  * Teach timestamp canonifier about timestamps before ~2001
+    (Jon Siwek, Corelight)
+
+2.5-877 | 2018-08-20 14:58:58 -0500
+
+  * Remove the node-specific local-*.bro scripts (Daniel Thayer)
+
 2.5-875 | 2018-08-20 12:45:32 -0500
 
   * Improve diff-remove-abspath canonifier: collapse '/' sequences

CMakeLists.txt

@@ -95,14 +95,6 @@ FindRequiredPackage(OpenSSL)
 FindRequiredPackage(BIND)
 FindRequiredPackage(ZLIB)
 
-find_package(CAF COMPONENTS core io openssl)
-if (CAF_FOUND)
-    # e.g. if not using embedded CAF, then need to know where to look
-    # for CAF headers since that may differ from where Broker headers
-    # are found (and including a Broker header may pull in CAF headers).
-	include_directories(BEFORE ${CAF_INCLUDE_DIRS})
-endif ()
-
 if (NOT BINPAC_EXE_PATH AND
     EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt)
     add_subdirectory(aux/binpac)
@@ -142,7 +134,7 @@ include_directories(BEFORE
 
 set(USE_GEOIP false)
 find_package(LibMMDB)
-if (LibMMDB_FOUND)
+if (LIBMMDB_FOUND)
     set(USE_GEOIP true)
     include_directories(BEFORE ${LibMMDB_INCLUDE_DIR})
     list(APPEND OPTLIBS ${LibMMDB_LIBRARY})
@@ -151,7 +143,7 @@ endif ()
 set(USE_KRB5 false)
 if ( ${CMAKE_SYSTEM_NAME} MATCHES Linux )
   find_package(LibKrb5)
-  if (LibKrb5_FOUND)
+  if (LIBKRB5_FOUND)
      set(USE_KRB5 true)
      list(APPEND OPTLIBS ${LibKrb5_LIBRARY})
   endif ()
@@ -255,7 +247,11 @@ else ()
   set(BRO_CONFIG_BINPAC_ROOT_DIR ${BRO_ROOT_DIR})
 endif ()
 
-set(BRO_CONFIG_BROKER_ROOT_DIR ${BRO_ROOT_DIR})
+if ( BROKER_ROOT_DIR )
+  set(BRO_CONFIG_BROKER_ROOT_DIR ${BROKER_ROOT_DIR})
+else ()
+  set(BRO_CONFIG_BROKER_ROOT_DIR ${BRO_ROOT_DIR})
+endif ()
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/bro-config.in
                ${CMAKE_CURRENT_BINARY_DIR}/bro-config @ONLY)
@@ -267,10 +263,22 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/cmake DESTINATION share/bro
 ########################################################################
 ## Recurse on sub-directories
 
-add_subdirectory(aux/broker)
+if ( BROKER_ROOT_DIR )
-set(brodeps ${brodeps} broker)
+  find_package(Broker REQUIRED)
-include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/aux/broker
+  find_package(CAF COMPONENTS core io openssl REQUIRED)
-                           ${CMAKE_CURRENT_BINARY_DIR}/aux/broker)
+
+  set(brodeps ${brodeps} ${BROKER_LIBRARY} ${CAF_LIBRARIES})
+  include_directories(BEFORE ${BROKER_INCLUDE_DIR})
+else ()
+  add_subdirectory(aux/broker)
+
+  set(brodeps ${brodeps} broker)
+  include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/aux/broker
+                             ${CMAKE_CURRENT_BINARY_DIR}/aux/broker)
+endif ()
+
+# CAF headers aren't necessarily in same location as Broker headers and
+# inclusion of a Broker header may pull in CAF headers.
 include_directories(BEFORE ${CAF_INCLUDE_DIR_CORE})
 include_directories(BEFORE ${CAF_INCLUDE_DIR_IO})
 include_directories(BEFORE ${CAF_INCLUDE_DIR_OPENSSL})

NEWS

@@ -409,6 +409,7 @@ Changed Functionality
 - The string_to_pattern() built-in (and the now-deprecated merge_pattern()
   built-in) is no longer restricted to only be called at initialization time.
 
+
 - GeoIP Legacy Database support has been replaced with GeoIP2 MaxMind DB
   format support.
 
@@ -418,6 +419,34 @@ Changed Functionality
     after January 2, 2019.  It's also noted that all GeoIP Legacy databases
     may be discontinued as they are superseded by GeoIP2.
 
+- "Weird" events are now generally suppressed/sampled by default according to
+  some tunable parameters:
+
+  - Weird::sampling_whitelist
+  - Weird::sampling_threshold
+  - Weird::sampling_rate
+  - Weird::sampling_duration
+
+  Those options can be changed if one needs the previous behavior of
+  a "net_weird", "flow_weird", or "conn_weird" event being raised for
+  every single event.  Otherwise, there is a new weird_stats.log which
+  contains concise summaries of weird counts per type per time period
+  and the original weird.log may not differ much either, except in
+  the cases where a particular weird type exceeds the sampling threshold.
+
+- Improved DCE-RPC analysis via tracking of context identifier mappings
+
+  - These DCE-RPC events now contain an additional context-id argument:
+
+    - dce_rpc_bind
+    - dce_rpc_request
+    - dce_rpc_response
+
+  - Added new events:
+
+    - dce_rpc_alter_context
+    - dce_rpc_alter_context_resp
+
 Removed Functionality
 ---------------------
 
@@ -432,6 +461,11 @@ Removed Functionality
 - BroControl: The "IPv6Comm" and "ZoneID" options are no longer
   available (though Broker should be able to handle IPv6 automatically).
 
+- The node-specific ``site/local-*.bro`` scripts have been removed.
+
+- The default value of ``Pcap::snaplen`` changed from 8192 to 9216 bytes
+  to better accommodate jumbo frames.
+
 Deprecated Functionality
 ------------------------
 

VERSION

@@ -1 +1 @@
-2.5-875
+2.5-906

aux/bifcl

@@ -1 +1 @@
-Subproject commit e99152c00aad8f81c684a01bc4d40790a295f85c
+Subproject commit b0ba021208c0c5fe8d7c92412b919d6667f63961

aux/binpac

@@ -1 +1 @@
-Subproject commit 74cf55ace0de2bf061bbbf285ccf47cba122955f
+Subproject commit ff8c94964fccbf60abae401d03c9fb35a8894c16

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 53aae820242c02790089e384a9fe2d3174799ab1
+Subproject commit 9866d31e45f5e1467c0f1fd4a1cd9948b067ce43

aux/broccoli

@@ -1 +1 @@
-Subproject commit edf754ea6e89a84ad74eff69a454c5e285c4b81b
+Subproject commit 1898e81bba7c595108bc20c5148df7f0f6a2258f

aux/broctl

@@ -1 +1 @@
-Subproject commit 70a8b2e15105f4c238765a882151718162e46208
+Subproject commit 486bbb9d9ee7c66b55003e58f986d18e951902ec

aux/broker

@@ -1 +1 @@
-Subproject commit 2727afff785201168b33b0c7448c886643f10e08
+Subproject commit 18d56b70558db61f424dc90e5d010f57b75de459

cmake

@@ -1 +1 @@
-Subproject commit 4cc3e344cf2698010a46684d32a2907a943430e3
+Subproject commit 433676df0af266540b8a3d9216cfd9dbc1361b09

configure

@@ -69,8 +69,10 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-flex=PATH       path to flex executable
     --with-bison=PATH      path to bison executable
     --with-python=PATH     path to Python executable
-    --with-caf=PATH        path to C++ Actor Framework installation for using external version
+    --with-broker=PATH     path to Broker install root
-                           (a required Broker dependency)
+                           (Bro uses an embedded version by default)
+    --with-caf=PATH        path to C++ Actor Framework install root
+                           (a Broker dependency that is embedded by default)
 
   Optional Packages in Non-Standard Locations:
     --with-geoip=PATH      path to the libmaxminddb install root
@@ -142,8 +144,6 @@ append_cache_entry INSTALL_BROCTL       BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
 append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
 append_cache_entry DISABLE_PERFTOOLS    BOOL   false
-append_cache_entry DISABLE_RUBY_BINDINGS     BOOL   true
-append_cache_entry ENABLE_COVERAGE	BOOL   false
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -220,6 +220,7 @@ while [ $# -ne 0 ]; do
             append_cache_entry ENABLE_JEMALLOC     BOOL   true
             ;;
         --enable-broccoli)
+            append_cache_entry DISABLE_RUBY_BINDINGS     BOOL   true
             append_cache_entry INSTALL_BROCCOLI     BOOL   yes
             ;;
        --disable-broctl)
@@ -285,6 +286,9 @@ while [ $# -ne 0 ]; do
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE      PATH    $optarg
             ;;
+        --with-broker=*)
+            append_cache_entry BROKER_ROOT_DIR      PATH   $optarg
+            ;;
         --with-caf=*)
             append_cache_entry CAF_ROOT_DIR      PATH   $optarg
             ;;

doc/script-reference/log-files.rst

@@ -152,6 +152,8 @@ Miscellaneous
 +----------------------------+---------------------------------------+---------------------------------+
 | weird.log                  | Unexpected network-level activity     | :bro:type:`Weird::Info`         |
 +----------------------------+---------------------------------------+---------------------------------+
+| weird-stats.log            | Statistics about unexpected activity  | :bro:type:`WeirdStats::Info`    |
++----------------------------+---------------------------------------+---------------------------------+
 
 Bro Diagnostics
 ---------------

scripts/CMakeLists.txt

@@ -14,19 +14,3 @@ InstallPackageConfigFile(
     ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro
     ${BRO_SCRIPT_INSTALL_PATH}/site
     local.bro)
-InstallPackageConfigFile(
-    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro
-    ${BRO_SCRIPT_INSTALL_PATH}/site
-    local-manager.bro)
-InstallPackageConfigFile(
-    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-logger.bro
-    ${BRO_SCRIPT_INSTALL_PATH}/site
-    local-logger.bro)
-InstallPackageConfigFile(
-    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro
-    ${BRO_SCRIPT_INSTALL_PATH}/site
-    local-proxy.bro)
-InstallPackageConfigFile(
-    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro
-    ${BRO_SCRIPT_INSTALL_PATH}/site
-    local-worker.bro)

scripts/base/init-bare.bro

@@ -82,6 +82,13 @@ type addr_vec: vector of addr;
 ##    directly and then remove this alias.
 type table_string_of_string: table[string] of string;
 
+## A table of counts indexed by strings.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type table_string_of_count: table[string] of count;
+
 ## A set of file analyzer tags.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions
@@ -626,6 +633,17 @@ type BrokerStats: record {
 	num_ids_outgoing: count;
 };
 
+## Statistics about reporter messages and weirds.
+##
+## .. bro:see:: get_reporter_stats
+type ReporterStats: record {
+	## Number of total weirds encountered, before any rate-limiting.
+	weirds: count;
+	## Number of times each individual weird is encountered, before any
+	## rate-limiting is applied.
+	weirds_by_type:	table[string] of count;
+};
+
 ## Deprecated.
 ##
 ## .. todo:: Remove. It's still declared internally but doesn't seem  used anywhere
@@ -4795,7 +4813,7 @@ export {
 module Pcap;
 export {
 	## Number of bytes per packet to capture from live interfaces.
-	const snaplen = 8192 &redef;
+	const snaplen = 9216 &redef;
 
 	## Number of Mbytes to provide as buffer space when capturing from live
 	## interfaces.
@@ -4826,6 +4844,35 @@ export {
 	type Cluster::Pool: record {};
 }
 
+module Weird;
+export {
+	## Prevents rate-limiting sampling of any weirds named in the table.
+	const sampling_whitelist: set[string] &redef;
+
+	## How many weirds of a given type to tolerate before sampling begins.
+	## i.e. this many consecutive weirds of a given type will be allowed to
+	## raise events for script-layer handling before being rate-limited.
+	const sampling_threshold = 25 &redef;
+
+	## The rate-limiting sampling rate.  One out of every of this number of
+	## rate-limited weirds of a given type will be allowed to raise events
+	## for further script-layer handling.
+	const sampling_rate = 1000 &redef;
+
+	## How long a weird of a given type is allowed to keep state/counters in
+	## memory.  For "net" weirds an expiration timer starts per weird name when
+	## first initializing its counter.  For "flow" weirds an expiration timer
+	## starts once per src/dst IP pair for the first weird of any name.  For
+	## "conn" weirds, counters and expiration timers are kept for the duration
+	## of the connection for each named weird and reset when necessary.  e.g.
+	## if a "conn" weird by the name of "foo" is seen more than
+	## :bro:see:`Weird::sampling_threshold` times, then an expiration timer
+	## begins for "foo" and upon triggering will reset the counter for "foo"
+	## and unthrottle its rate-limiting until it once again exceeds the
+	## threshold.
+	const sampling_duration = 10min &redef;
+}
+
 module GLOBAL;
 
 ## Seed for hashes computed internally for probabilistic data structures. Using

scripts/base/protocols/dce-rpc/main.bro

@@ -37,6 +37,7 @@ export {
 	type State: record {
 		uuid       : string &optional;
 		named_pipe : string &optional;
+		ctx_to_uuid: table[count] of string &optional;
 	};
 
 	# This is to store the log and state information
@@ -100,11 +101,30 @@ function set_session(c: connection, fid: count)
 	set_state(c, state_x);
 	}
 
-event dce_rpc_bind(c: connection, fid: count, uuid: string, ver_major: count, ver_minor: count) &priority=5
+event dce_rpc_bind(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count) &priority=5
 	{
 	set_session(c, fid);
 
 	local uuid_str = uuid_to_string(uuid);
+
+	if ( ! c$dce_rpc_state?$ctx_to_uuid )
+		c$dce_rpc_state$ctx_to_uuid = table();
+
+	c$dce_rpc_state$ctx_to_uuid[ctx_id] = uuid_str;
+	c$dce_rpc_state$uuid = uuid_str;
+	c$dce_rpc$endpoint = uuid_endpoint_map[uuid_str];
+	}
+
+event dce_rpc_alter_context(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count) &priority=5
+	{
+	set_session(c, fid);
+
+	local uuid_str = uuid_to_string(uuid);
+
+	if ( ! c$dce_rpc_state?$ctx_to_uuid )
+		c$dce_rpc_state$ctx_to_uuid = table();
+
+	c$dce_rpc_state$ctx_to_uuid[ctx_id] = uuid_str;
 	c$dce_rpc_state$uuid = uuid_str;
 	c$dce_rpc$endpoint = uuid_endpoint_map[uuid_str];
 	}
@@ -120,7 +140,12 @@ event dce_rpc_bind_ack(c: connection, fid: count, sec_addr: string) &priority=5
 		}
 	}
 
-event dce_rpc_request(c: connection, fid: count, opnum: count, stub_len: count) &priority=5
+event dce_rpc_alter_context_resp(c: connection, fid: count) &priority=5
+	{
+	set_session(c, fid);
+	}
+
+event dce_rpc_request(c: connection, fid: count, ctx_id: count, opnum: count, stub_len: count) &priority=5
 	{
 	set_session(c, fid);
 
@@ -130,7 +155,7 @@ event dce_rpc_request(c: connection, fid: count, opnum: count, stub_len: count)
 		}
 	}
 
-event dce_rpc_response(c: connection, fid: count, opnum: count, stub_len: count) &priority=5
+event dce_rpc_response(c: connection, fid: count, ctx_id: count, opnum: count, stub_len: count) &priority=5
 	{
 	set_session(c, fid);
 
@@ -146,15 +171,26 @@ event dce_rpc_response(c: connection, fid: count, opnum: count, stub_len: count)
 			}
 		}
 
-	if ( c?$dce_rpc && c$dce_rpc?$endpoint )
+	if ( c?$dce_rpc )
 		{
-		c$dce_rpc$operation = operations[c$dce_rpc_state$uuid, opnum];
+		if ( c$dce_rpc?$endpoint )
-		if ( c$dce_rpc$ts != network_time() )
+			{
-			c$dce_rpc$rtt = network_time() - c$dce_rpc$ts;
+			c$dce_rpc$operation = operations[c$dce_rpc_state$uuid, opnum];
+			if ( c$dce_rpc$ts != network_time() )
+				c$dce_rpc$rtt = network_time() - c$dce_rpc$ts;
+			}
+
+		if ( c$dce_rpc_state?$ctx_to_uuid &&
+		     ctx_id in c$dce_rpc_state$ctx_to_uuid )
+			{
+			local u = c$dce_rpc_state$ctx_to_uuid[ctx_id];
+			c$dce_rpc$endpoint = uuid_endpoint_map[u];
+			c$dce_rpc$operation = operations[u, opnum];
+			}
 		}
 	}
 
-event dce_rpc_response(c: connection, fid: count, opnum: count, stub_len: count) &priority=-5
+event dce_rpc_response(c: connection, fid: count, ctx_id: count, opnum: count, stub_len: count) &priority=-5
 	{
 	if ( c?$dce_rpc )
 		{

scripts/base/protocols/http/main.bro

@@ -199,7 +199,7 @@ event http_request(c: connection, method: string, original_URI: string,
 	c$http$uri = unescaped_URI;
 
 	if ( method !in http_methods )
-		event conn_weird("unknown_HTTP_method", c, method);
+		Reporter::conn_weird("unknown_HTTP_method", c, method);
 	}
 
 event http_reply(c: connection, version: string, code: count, reason: string) &priority=5

scripts/base/protocols/sip/main.bro

@@ -168,7 +168,7 @@ event sip_request(c: connection, method: string, original_URI: string, version:
 	c$sip$uri = original_URI;
 
 	if ( method !in sip_methods )
-		event conn_weird("unknown_SIP_method", c, method);
+		Reporter::conn_weird("unknown_SIP_method", c, method);
 	}
 
 event sip_reply(c: connection, version: string, code: count, reason: string) &priority=5

scripts/base/protocols/smb/smb1-main.bro

@@ -309,7 +309,7 @@ event smb_pipe_request(c: connection, hdr: SMB1::Header, op_num: count)
 	if ( ! f?$uuid )
 		{
 		# TODO: figure out why this is happening.
-		event conn_weird("smb_pipe_request_missing_uuid", c, "");
+		Reporter::conn_weird("smb_pipe_request_missing_uuid", c, "");
 		return;
 		}
 	local arg = fmt("%s: %s",

scripts/base/protocols/ssl/main.bro

@@ -263,7 +263,7 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
 		{
 		c$ssl$server_name = names[0];
 		if ( |names| > 1 )
-			event conn_weird("SSL_many_server_names", c, cat(names));
+			Reporter::conn_weird("SSL_many_server_names", c, cat(names));
 		}
 	}
 

scripts/policy/misc/weird-stats.bro

@@ -0,0 +1,100 @@
+##! Log weird statistics.
+
+@load base/frameworks/sumstats
+@load base/frameworks/cluster
+
+module WeirdStats;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	## How often stats are reported.
+	const weird_stat_interval = 15min &redef;
+
+	type Info: record {
+		## Timestamp for the measurement.
+		ts: time &log;
+		## Name of the weird.
+		name: string &log;
+		## Number of times weird was seen since the last stats interval.
+		num_seen: count &log;
+	};
+
+	global log_weird_stats: event(rec: Info);
+}
+
+global this_epoch_weirds: table[string] of double;
+global last_epoch_weirds: table[string] of double;
+
+function weird_epoch_results(ts: time, key: SumStats::Key, result: SumStats::Result)
+	{
+	this_epoch_weirds[key$str]=result["weirds.encountered"]$sum;
+	}
+
+function weird_epoch_finished(ts: time)
+	{
+	for ( n in this_epoch_weirds )
+		{
+		local last_count: double = 0.0;
+
+		if ( n in last_epoch_weirds )
+			last_count = last_epoch_weirds[n];
+
+		local num_seen: double = this_epoch_weirds[n] - last_count;
+
+		if ( num_seen > 0.0 )
+			Log::write(LOG, Info($ts = ts, $name = n,
+			                     $num_seen = double_to_count(num_seen)));
+		}
+
+	last_epoch_weirds = this_epoch_weirds;
+	this_epoch_weirds = table();
+	}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(WeirdStats::LOG,
+	                   [$columns = Info, $ev = log_weird_stats,
+	                    $path="weird_stats"]);
+	local r1 = SumStats::Reducer($stream = "weirds.encountered",
+	                             $apply = set(SumStats::SUM));
+	SumStats::create([$name = "weirds.statistics",
+	                  $epoch = weird_stat_interval, $reducers = set(r1),
+	                  $epoch_result = weird_epoch_results,
+	                  $epoch_finished = weird_epoch_finished]);
+	}
+
+module SumStats;
+
+function observe_weird_stats()
+	{
+	local rs = get_reporter_stats();
+
+	for ( n in rs$weirds_by_type )
+		SumStats::observe("weirds.encountered", SumStats::Key($str = n),
+		                  SumStats::Observation($dbl=rs$weirds_by_type[n]+0.0));
+	}
+
+@if ( Cluster::is_enabled() )
+
+# I'm not sure if this is a hack or not: the manager will generate this
+# event at the end of its epoch so workers can handle it just in time to
+# generate the necessary stats.  Alternative may be workers generating the
+# stats individually/proactively in their own finish_epoch, but that may be
+# less synchronized?
+event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool) &priority=10
+	{
+	observe_weird_stats();
+	}
+
+@else
+
+event SumStats::finish_epoch(ss: SumStat) &priority=10
+	{
+	if ( ss$name != "weirds.statistics" )
+		return;
+
+	observe_weird_stats();
+	}
+
+@endif

scripts/site/local-logger.bro

@@ -1 +0,0 @@
-##! Local site policy loaded only by the logger if Bro is running as a cluster.

scripts/site/local-manager.bro

@@ -1 +0,0 @@
-##! Local site policy loaded only by the manager if Bro is running as a cluster.

scripts/site/local-proxy.bro

@@ -1 +0,0 @@
-##! Local site policy loaded only by the proxies if Bro is running as a cluster.

scripts/site/local-worker.bro

@@ -1 +0,0 @@
-##! Local site policy loaded only by the workers if Bro is running as a cluster.

scripts/test-all-policy.bro

@@ -54,6 +54,7 @@
 @load misc/profiling.bro
 @load misc/scan.bro
 @load misc/stats.bro
+@load misc/weird-stats.bro
 @load misc/trim-trace-file.bro
 @load protocols/conn/known-hosts.bro
 @load protocols/conn/known-services.bro

src/3rdparty

@@ -1 +1 @@
-Subproject commit 6cdefdd1d45465ca09aba9e05c7ca12e1484ccc0
+Subproject commit 7c95b51de202ac534b27dd721da5778b773dd614

src/Conn.cc

@@ -1071,3 +1071,28 @@ void Connection::CheckFlowLabel(bool is_orig, uint32 flow_label)
 	else
 		saw_first_resp_packet = 1;
 	}
+
+bool Connection::PermitWeird(const char* name, uint64 threshold, uint64 rate,
+                             double duration)
+	{
+	auto& state = weird_state[name];
+	++state.count;
+
+	if ( state.count < threshold )
+		return true;
+
+	if ( state.count == threshold )
+		state.sampling_start_time = network_time;
+	else
+		{
+		if ( network_time > state.sampling_start_time + duration )
+			{
+			state.sampling_start_time = 0;
+			state.count = 1;
+			return true;
+			}
+		}
+
+	auto num_above_threshold = state.count - threshold;
+	return num_above_threshold % rate == 0;
+	}

src/Conn.h

@@ -5,6 +5,9 @@
 
 #include <sys/types.h>
 
+#include <unordered_map>
+#include <string>
+
 #include "Dict.h"
 #include "Val.h"
 #include "Timer.h"
@@ -275,6 +278,9 @@ public:
 	uint32 GetOrigFlowLabel() { return orig_flow_label; }
 	uint32 GetRespFlowLabel() { return resp_flow_label; }
 
+	bool PermitWeird(const char* name, uint64 threshold, uint64 rate,
+	                 double duration);
+
 protected:
 
 	Connection()	{ persistent = 0; }
@@ -339,6 +345,14 @@ protected:
 	analyzer::pia::PIA* primary_PIA;
 
 	Bro::UID uid;	// Globally unique connection ID.
+
+	struct WeirdState {
+		WeirdState() { count = 0; sampling_start_time = 0; }
+		uint64 count = 0;
+		double sampling_start_time = 0;
+	};
+
+	std::unordered_map<std::string, WeirdState> weird_state;
 };
 
 class ConnectionTimer : public Timer {

src/Func.cc

@@ -722,6 +722,7 @@ void init_builtin_funcs()
 	FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType();
 	ThreadStats = internal_type("ThreadStats")->AsRecordType();
 	BrokerStats = internal_type("BrokerStats")->AsRecordType();
+	ReporterStats = internal_type("ReporterStats")->AsRecordType();
 
 	var_sizes = internal_type("var_sizes")->AsTableType();
 

src/Reporter.cc

@@ -10,6 +10,7 @@
 #include "NetVar.h"
 #include "Net.h"
 #include "Conn.h"
+#include "Timer.h"
 #include "plugin/Plugin.h"
 #include "plugin/Manager.h"
 
@@ -36,6 +37,11 @@ Reporter::Reporter()
 	warnings_to_stderr = true;
 	errors_to_stderr = true;
 
+	weird_count = 0;
+	weird_sampling_rate = 0;
+	weird_sampling_duration = 0;
+	weird_sampling_threshold = 0;
+
 	openlog("bro", 0, LOG_LOCAL5);
 	}
 
@@ -49,6 +55,24 @@ void Reporter::InitOptions()
 	info_to_stderr = internal_const_val("Reporter::info_to_stderr")->AsBool();
 	warnings_to_stderr = internal_const_val("Reporter::warnings_to_stderr")->AsBool();
 	errors_to_stderr = internal_const_val("Reporter::errors_to_stderr")->AsBool();
+	weird_sampling_rate = internal_const_val("Weird::sampling_rate")->AsCount();
+	weird_sampling_threshold = internal_const_val("Weird::sampling_threshold")->AsCount();
+	weird_sampling_duration = internal_const_val("Weird::sampling_duration")->AsInterval();
+	auto wl_val = internal_const_val("Weird::sampling_whitelist")->AsTableVal();
+	auto wl_table = wl_val->AsTable();
+
+	HashKey* k;
+	IterCookie* c = wl_table->InitForIteration();
+	TableEntryVal* v;
+
+	while ( (v = wl_table->NextEntry(k, c)) )
+		{
+		auto index = wl_val->RecoverIndex(k);
+		string key = index->Index(0)->AsString()->CheckString();
+		weird_sampling_whitelist.emplace(move(key));
+		Unref(index);
+		delete k;
+		}
 	}
 
 void Reporter::Info(const char* fmt, ...)
@@ -221,23 +245,121 @@ void Reporter::WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const cha
 	delete vl;
 	}
 
+void Reporter::UpdateWeirdStats(const char* name)
+	{
+	++weird_count;
+	++weird_count_by_type[name];
+	}
+
+class NetWeirdTimer : public Timer {
+public:
+	NetWeirdTimer(double t, const char* name, double timeout)
+	: Timer(t + timeout, TIMER_NET_WEIRD_EXPIRE), weird_name(name)
+		{}
+
+	void Dispatch(double t, int is_expire) override
+		{ reporter->ResetNetWeird(weird_name); }
+
+	std::string weird_name;
+};
+
+class FlowWeirdTimer : public Timer {
+public:
+	using IPPair = std::pair<IPAddr, IPAddr>;
+
+	FlowWeirdTimer(double t, IPPair p, double timeout)
+	: Timer(t + timeout, TIMER_FLOW_WEIRD_EXPIRE), endpoints(p)
+		{}
+
+	void Dispatch(double t, int is_expire) override
+		{ reporter->ResetFlowWeird(endpoints.first, endpoints.second); }
+
+	IPPair endpoints;
+};
+
+void Reporter::ResetNetWeird(const std::string& name)
+	{
+	net_weird_state.erase(name);
+	}
+
+void Reporter::ResetFlowWeird(const IPAddr& orig, const IPAddr& resp)
+	{
+	flow_weird_state.erase(std::make_pair(orig, resp));
+	}
+
+bool Reporter::PermitNetWeird(const char* name)
+	{
+	auto& count = net_weird_state[name];
+	++count;
+
+	if ( count == 1 )
+		timer_mgr->Add(new NetWeirdTimer(network_time, name,
+		                                 weird_sampling_duration));
+
+	if ( count < weird_sampling_threshold )
+		return true;
+
+	auto num_above_threshold = count - weird_sampling_threshold;
+	return num_above_threshold % weird_sampling_rate == 0;
+	}
+
+bool Reporter::PermitFlowWeird(const char* name,
+                               const IPAddr& orig, const IPAddr& resp)
+	{
+	auto endpoints = std::make_pair(orig, resp);
+	auto& map = flow_weird_state[endpoints];
+
+	if ( map.empty() )
+		timer_mgr->Add(new FlowWeirdTimer(network_time, endpoints,
+		                                  weird_sampling_duration));
+
+	auto& count = map[name];
+	++count;
+
+	if ( count < weird_sampling_threshold )
+		return true;
+
+	auto num_above_threshold = count - weird_sampling_threshold;
+	return num_above_threshold % weird_sampling_rate == 0;
+	}
+
 void Reporter::Weird(const char* name)
 	{
+	UpdateWeirdStats(name);
+
+	if ( ! WeirdOnSamplingWhiteList(name) )
+		{
+		if ( ! PermitNetWeird(name) )
+			return;
+		}
+
 	WeirdHelper(net_weird, 0, 0, "%s", name);
 	}
 
 void Reporter::Weird(Connection* conn, const char* name, const char* addl)
 	{
-	WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name);
+	UpdateWeirdStats(name);
-	}
 
-void Reporter::Weird(Val* conn_val, const char* name, const char* addl)
+	if ( ! WeirdOnSamplingWhiteList(name) )
-	{
+		{
-	WeirdHelper(conn_weird, conn_val, addl, "%s", name);
+		if ( ! conn->PermitWeird(name, weird_sampling_threshold,
+		                         weird_sampling_rate, weird_sampling_duration) )
+			return;
+		}
+
+	WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name);
 	}
 
 void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name)
 	{
+	UpdateWeirdStats(name);
+
+	if ( ! WeirdOnSamplingWhiteList(name) )
+		{
+		if ( ! PermitFlowWeird(name, orig, resp) )
+			 return;
+		}
+
 	WeirdFlowHelper(orig, resp, "%s", name);
 	}
 

src/Reporter.h

@@ -7,6 +7,10 @@
 
 #include <list>
 #include <utility>
+#include <string>
+#include <map>
+#include <unordered_set>
+#include <unordered_map>
 
 #include "util.h"
 #include "EventHandler.h"
@@ -36,6 +40,11 @@ protected:
 
 class Reporter {
 public:
+	using IPPair = std::pair<IPAddr, IPAddr>;
+	using WeirdCountMap = std::unordered_map<std::string, uint64>;
+	using WeirdFlowMap = std::map<IPPair, WeirdCountMap>;
+	using WeirdSet = std::unordered_set<std::string>;
+
 	Reporter();
 	~Reporter();
 
@@ -76,7 +85,6 @@ public:
 	// that may lead to incorrectly processing a connnection.
 	void Weird(const char* name);	// Raises net_weird().
 	void Weird(Connection* conn, const char* name, const char* addl = "");	// Raises conn_weird().
-	void Weird(Val* conn_val, const char* name, const char* addl = "");	// Raises conn_weird().
 	void Weird(const IPAddr& orig, const IPAddr& resp, const char* name);	// Raises flow_weird().
 
 	// Syslog a message. This methods does nothing if we're running
@@ -120,6 +128,30 @@ public:
 	// Signals that we're done processing an error handler event.
 	void EndErrorHandler()	{ --in_error_handler; }
 
+	/**
+	 * Reset/cleanup state tracking for a "net" weird.
+	 */
+	void ResetNetWeird(const std::string& name);
+
+	/**
+	 * Reset/cleanup state tracking for a "flow" weird.
+	 */
+	void ResetFlowWeird(const IPAddr& orig, const IPAddr& resp);
+
+	/**
+	 * Return the total number of weirds generated (counts weirds before
+	 * any rate-limiting occurs).
+	 */
+	uint64 GetWeirdCount() const
+		{ return weird_count; }
+
+	/**
+	 * Return number of weirds generated per weird type/name (counts weirds
+	 * before any rate-limiting occurs).
+	 */
+	const WeirdCountMap& GetWeirdsByType() const
+		{ return weird_count_by_type; }
+
 private:
 	void DoLog(const char* prefix, EventHandlerPtr event, FILE* out,
 		   Connection* conn, val_list* addl, bool location, bool time,
@@ -129,6 +161,11 @@ private:
 	// contain format specifiers
 	void WeirdHelper(EventHandlerPtr event, Val* conn_val, const char* addl, const char* fmt_name, ...) __attribute__((format(printf, 5, 6)));;
 	void WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const char* fmt_name, ...) __attribute__((format(printf, 4, 5)));;
+	void UpdateWeirdStats(const char* name);
+	inline bool WeirdOnSamplingWhiteList(const char* name)
+		{ return weird_sampling_whitelist.find(name) != weird_sampling_whitelist.end(); }
+	bool PermitNetWeird(const char* name);
+	bool PermitFlowWeird(const char* name, const IPAddr& o, const IPAddr& r);
 
 	int errors;
 	bool via_events;
@@ -138,6 +175,18 @@ private:
 	bool errors_to_stderr;
 
 	std::list<std::pair<const Location*, const Location*> > locations;
+
+	uint64 weird_count;
+	WeirdCountMap weird_count_by_type;
+
+	WeirdCountMap net_weird_state;
+	WeirdFlowMap flow_weird_state;
+
+	WeirdSet weird_sampling_whitelist;
+	uint64 weird_sampling_threshold;
+	uint64 weird_sampling_rate;
+	double weird_sampling_duration;
+
 };
 
 extern Reporter* reporter;

src/Timer.cc

@@ -18,12 +18,14 @@ const char* TimerNames[] = {
 	"ConnectionStatusUpdateTimer",
 	"DNSExpireTimer",
 	"FileAnalysisInactivityTimer",
+	"FlowWeirdTimer",
 	"FragTimer",
 	"IncrementalSendTimer",
 	"IncrementalWriteTimer",
 	"InterconnTimer",
 	"IPTunnelInactivityTimer",
 	"NetbiosExpireTimer",
+	"NetWeirdTimer",
 	"NetworkTimer",
 	"NTPExpireTimer",
 	"ProfileTimer",

src/Timer.h

@@ -23,12 +23,14 @@ enum TimerType {
 	TIMER_CONN_STATUS_UPDATE,
 	TIMER_DNS_EXPIRE,
 	TIMER_FILE_ANALYSIS_INACTIVITY,
+	TIMER_FLOW_WEIRD_EXPIRE,
 	TIMER_FRAG,
 	TIMER_INCREMENTAL_SEND,
 	TIMER_INCREMENTAL_WRITE,
 	TIMER_INTERCONN,
 	TIMER_IP_TUNNEL_INACTIVITY,
 	TIMER_NB_EXPIRE,
+	TIMER_NET_WEIRD_EXPIRE,
 	TIMER_NETWORK,
 	TIMER_NTP_EXPIRE,
 	TIMER_PROFILE,

src/analyzer/protocol/dce-rpc/dce_rpc-analyzer.pac

@@ -54,6 +54,7 @@ refine connection DCE_RPC_Conn += {
 			BifEvent::generate_dce_rpc_bind(bro_analyzer(),
 			                                bro_analyzer()->Conn(),
 			                                fid,
+			                                ${req.id},
 			                                bytestring_to_val(${req.abstract_syntax.uuid}),
 			                                ${req.abstract_syntax.ver_major},
 			                                ${req.abstract_syntax.ver_minor});
@@ -62,6 +63,22 @@ refine connection DCE_RPC_Conn += {
 		return true;
 		%}
 
+	function process_dce_rpc_alter_context(req: ContextRequest): bool
+		%{
+		if ( dce_rpc_alter_context )
+			{
+			BifEvent::generate_dce_rpc_alter_context(bro_analyzer(),
+			                                         bro_analyzer()->Conn(),
+			                                         fid,
+			                                         ${req.id},
+			                                         bytestring_to_val(${req.abstract_syntax.uuid}),
+			                                         ${req.abstract_syntax.ver_major},
+			                                         ${req.abstract_syntax.ver_minor});
+			}
+
+		return true;
+		%}
+
 	function process_dce_rpc_bind_ack(bind: DCE_RPC_Bind_Ack): bool
 		%{
 		if ( dce_rpc_bind_ack )
@@ -86,6 +103,17 @@ refine connection DCE_RPC_Conn += {
 		return true;
 		%}
 
+	function process_dce_rpc_alter_context_resp(bind: DCE_RPC_AlterContext_Resp): bool
+		%{
+		if ( dce_rpc_alter_context_resp )
+			{
+			BifEvent::generate_dce_rpc_alter_context_resp(bro_analyzer(),
+			                                              bro_analyzer()->Conn(),
+			                                              fid);
+			}
+		return true;
+		%}
+
 	function process_dce_rpc_request(req: DCE_RPC_Request): bool
 		%{
 		if ( dce_rpc_request )
@@ -93,6 +121,7 @@ refine connection DCE_RPC_Conn += {
 			BifEvent::generate_dce_rpc_request(bro_analyzer(),
 			                                   bro_analyzer()->Conn(),
 			                                   fid,
+			                                   ${req.context_id},
 			                                   ${req.opnum},
 			                                   ${req.stub}.length());
 			}
@@ -109,6 +138,7 @@ refine connection DCE_RPC_Conn += {
 			BifEvent::generate_dce_rpc_response(bro_analyzer(),
 			                                    bro_analyzer()->Conn(),
 			                                    fid,
+			                                    ${resp.context_id},
 			                                    get_cont_id_opnum_map(${resp.context_id}),
 			                                    ${resp.stub}.length());
 			}
@@ -127,13 +157,20 @@ refine typeattr DCE_RPC_Header += &let {
 };
 
 refine typeattr ContextRequest += &let {
-	proc = $context.connection.process_dce_rpc_bind(this);
+	proc = case ptype of {
+		DCE_RPC_BIND          -> $context.connection.process_dce_rpc_bind(this);
+		DCE_RPC_ALTER_CONTEXT -> $context.connection.process_dce_rpc_alter_context(this);
+	};
 };
 
 refine typeattr DCE_RPC_Bind_Ack += &let {
 	proc = $context.connection.process_dce_rpc_bind_ack(this);
 };
 
+refine typeattr DCE_RPC_AlterContext_Resp += &let {
+	proc = $context.connection.process_dce_rpc_alter_context_resp(this);
+};
+
 refine typeattr DCE_RPC_Request += &let {
 	proc = $context.connection.process_dce_rpc_request(this);
 };

src/analyzer/protocol/dce-rpc/dce_rpc-protocol.pac

@@ -71,7 +71,7 @@ type Syntax = record {
 	ver_minor : uint16;
 };
 
-type ContextRequest = record {
+type ContextRequest(ptype: uint8) = record {
 	id                : uint16;
 	num_syntaxes      : uint8;
 	reserved          : padding[1];
@@ -85,11 +85,11 @@ type ContextReply = record {
 	syntax            : Syntax;
 };
 
-type ContextList(is_request: bool) = record {
+type ContextList(is_request: bool, ptype: uint8) = record {
 	num_contexts   : uint8;
 	reserved       : padding[3];
 	req_reply      : case is_request of {
-		true  -> request_contexts : ContextRequest[num_contexts];
+		true  -> request_contexts : ContextRequest(ptype)[num_contexts];
 		false -> reply_contexts   : ContextReply[num_contexts];
 	};
 };
@@ -98,7 +98,7 @@ type DCE_RPC_Bind = record {
 	max_xmit_frag  : uint16;
 	max_recv_frag  : uint16;
 	assoc_group_id : uint32;
-	context_list   : ContextList(1);
+	context_list   : ContextList(1, DCE_RPC_BIND);
 };
 
 type DCE_RPC_Bind_Ack = record {
@@ -108,7 +108,7 @@ type DCE_RPC_Bind_Ack = record {
 	sec_addr_length : uint16;
 	sec_addr        : bytestring &length=sec_addr_length;
 	pad             : padding align 4;
-	contexts        : ContextList(0);
+	contexts        : ContextList(0, DCE_RPC_BIND_ACK);
 };
 
 type DCE_RPC_Request(h: DCE_RPC_Header) = record {
@@ -136,15 +136,17 @@ type DCE_RPC_AlterContext = record {
 	max_xmit_frag  : uint16;
 	max_recv_frag  : uint16;
 	assoc_group_id : uint32;
-	contexts       : ContextList(0);
+	context_list   : ContextList(1, DCE_RPC_ALTER_CONTEXT);
 };
 
 type DCE_RPC_AlterContext_Resp = record {
-	max_xmit_frag  : uint16;
+	max_xmit_frag   : uint16;
-	max_recv_frag  : uint16;
+	max_recv_frag   : uint16;
-	assoc_group_id : uint32;
+	assoc_group_id  : uint32;
-	sec_addr_len   : uint16;
+	sec_addr_length : uint16;
-	contexts       : ContextList(0);
+	sec_addr        : bytestring &length=sec_addr_length;
+	pad             : padding align 4;
+	contexts        : ContextList(0, DCE_RPC_ALTER_CONTEXT_RESP);
 };
 
 type DCE_RPC_Body(header: DCE_RPC_Header) = case header.PTYPE of {
@@ -152,9 +154,8 @@ type DCE_RPC_Body(header: DCE_RPC_Header) = case header.PTYPE of {
 	DCE_RPC_BIND_ACK           -> bind_ack      : DCE_RPC_Bind_Ack;
 	DCE_RPC_REQUEST            -> request       : DCE_RPC_Request(header);
 	DCE_RPC_RESPONSE           -> response      : DCE_RPC_Response;
-	# TODO: Something about the two following structures isn't being handled correctly.
+	DCE_RPC_ALTER_CONTEXT      -> alter_context : DCE_RPC_AlterContext;
-	#DCE_RPC_ALTER_CONTEXT      -> alter_context : DCE_RPC_AlterContext;
+	DCE_RPC_ALTER_CONTEXT_RESP -> alter_resp    : DCE_RPC_AlterContext_Resp;
-	#DCE_RPC_ALTER_CONTEXT_RESP -> alter_resp    : DCE_RPC_AlterContext_Resp;
 	default                    -> other         : bytestring &restofdata;
 };
 

src/analyzer/protocol/dce-rpc/events.bif

@@ -25,6 +25,8 @@ event dce_rpc_message%(c: connection, is_orig: bool, fid: count, ptype_id: count
 ##      message. Zero will be used if the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` was
 ##      not transported over a pipe.
 ##
+## ctx_id: The context identifier of the data representation.
+##
 ## uuid: The string interpretted uuid of the endpoint being requested.
 ##
 ## ver_major: The major version of the endpoint being requested.
@@ -32,7 +34,28 @@ event dce_rpc_message%(c: connection, is_orig: bool, fid: count, ptype_id: count
 ## ver_minor: The minor version of the endpoint being requested.
 ##
 ## .. bro:see:: dce_rpc_message dce_rpc_bind_ack dce_rpc_request dce_rpc_response
-event dce_rpc_bind%(c: connection, fid: count, uuid: string, ver_major: count, ver_minor: count%);
+event dce_rpc_bind%(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count%);
+
+## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` alter context request message.
+## Since RPC offers the ability for a client to request connections to multiple endpoints, this event can occur
+## multiple times for a single RPC message.
+##
+## c: The connection.
+##
+## fid: File ID of the PIPE that carried the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)`
+##      message. Zero will be used if the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` was
+##      not transported over a pipe.
+##
+## ctx_id: The context identifier of the data representation.
+##
+## uuid: The string interpretted uuid of the endpoint being requested.
+##
+## ver_major: The major version of the endpoint being requested.
+##
+## ver_minor: The minor version of the endpoint being requested.
+##
+## .. bro:see:: dce_rpc_message dce_rpc_bind dce_rpc_bind_ack dce_rpc_request dce_rpc_response dce_rpc_alter_context_resp
+event dce_rpc_alter_context%(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count%);
 
 ## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` bind request ack message.
 ##
@@ -47,6 +70,17 @@ event dce_rpc_bind%(c: connection, fid: count, uuid: string, ver_major: count, v
 ## .. bro:see:: dce_rpc_message dce_rpc_bind dce_rpc_request dce_rpc_response
 event dce_rpc_bind_ack%(c: connection, fid: count, sec_addr: string%);
 
+## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` alter context response message.
+##
+## c: The connection.
+##
+## fid: File ID of the PIPE that carried the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)`
+##      message. Zero will be used if the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` was
+##      not transported over a pipe.
+##
+## .. bro:see:: dce_rpc_message dce_rpc_bind dce_rpc_bind_ack dce_rpc_request dce_rpc_response dce_rpc_alter_context
+event dce_rpc_alter_context_resp%(c: connection, fid: count%);
+
 ## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` request message.
 ##
 ## c: The connection.
@@ -55,12 +89,14 @@ event dce_rpc_bind_ack%(c: connection, fid: count, sec_addr: string%);
 ##      message. Zero will be used if the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` was
 ##      not transported over a pipe.
 ##
+## ctx_id: The context identifier of the data representation.
+##
 ## opnum: Number of the RPC operation.
 ##
 ## stub_len: Length of the data for the request.
 ##
 ## .. bro:see:: dce_rpc_message dce_rpc_bind dce_rpc_bind_ack dce_rpc_response
-event dce_rpc_request%(c: connection, fid: count, opnum: count, stub_len: count%);
+event dce_rpc_request%(c: connection, fid: count, ctx_id: count, opnum: count, stub_len: count%);
 
 ## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` response message.
 ##
@@ -70,9 +106,11 @@ event dce_rpc_request%(c: connection, fid: count, opnum: count, stub_len: count%
 ##      message. Zero will be used if the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` was
 ##      not transported over a pipe.
 ##
+## ctx_id: The context identifier of the data representation.
+###
 ## opnum: Number of the RPC operation.
 ##
 ## stub_len: Length of the data for the response.
 ##
 ## .. bro:see:: dce_rpc_message dce_rpc_bind dce_rpc_bind_ack dce_rpc_request
-event dce_rpc_response%(c: connection, fid: count, opnum: count, stub_len: count%);
+event dce_rpc_response%(c: connection, fid: count, ctx_id: count, opnum: count, stub_len: count%);

src/analyzer/protocol/finger/Finger.cc

@@ -41,6 +41,9 @@ void Finger_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig
 	const char* line = (const char*) data;
 	const char* end_of_line = line + length;
 
+	if ( length == 0 )
+		return;
+
 	if ( is_orig )
 		{
 

src/analyzer/protocol/ftp/FTP.cc

@@ -69,6 +69,10 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 	const char* line = (const char*) data;
 	const char* end_of_line = line + length;
 
+	if ( length == 0 )
+		// Could emit "ftp empty request/reply" weird, but maybe not worth it.
+		return;
+
 	val_list* vl = new val_list;
 	vl->append(BuildConnVal());
 

src/analyzer/protocol/ident/Ident.cc

@@ -56,6 +56,9 @@ void Ident_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig)
 	if ( TCP() )
 		s = is_orig ? TCP()->Orig() : TCP()->Resp();
 
+	if ( length == 0 )
+		return;
+
 	if ( is_orig )
 		{
 		if ( ! ident_request )

src/analyzer/protocol/pop3/POP3.cc

@@ -132,6 +132,9 @@ static string trim_whitespace(const char* in)
 
 void POP3_Analyzer::ProcessRequest(int length, const char* line)
 	{
+	if ( length == 0 )
+		return;
+
 	if ( waitingForAuthentication )
 		{
 		++authLines;

src/analyzer/protocol/smtp/SMTP.cc

@@ -269,7 +269,9 @@ void SMTP_Analyzer::ProcessLine(int length, const char* line, bool orig)
 			if ( smtp_request )
 				{
 				int data_len = end_of_line - line;
-				RequestEvent(cmd_len, cmd, data_len, line);
+
+				if ( cmd_len > 0 || data_len > 0 )
+					RequestEvent(cmd_len, cmd, data_len, line);
 				}
 
 			if ( cmd_code != SMTP_CMD_END_OF_DATA )
@@ -379,7 +381,17 @@ void SMTP_Analyzer::NewCmd(const int cmd_code)
 		if ( first_cmd < 0 )
 			first_cmd = cmd_code;
 		else
+			{
+			auto constexpr max_pending_cmd_q_size = 1000;
+
+			if ( pending_cmd_q.size() == max_pending_cmd_q_size )
+				{
+				Weird("smtp_excessive_pending_cmds");
+				pending_cmd_q.clear();
+				}
+
 			pending_cmd_q.push_back(cmd_code);
+			}
 		}
 	else
 		first_cmd = cmd_code;
@@ -805,12 +817,22 @@ void SMTP_Analyzer::UpdateState(const int cmd_code, const int reply_code, bool o
 #endif
 	}
 
+static bool istrequal(const char* s, const char* cmd, int s_len)
+	{
+	int cmd_len = strlen(cmd);
+
+	if ( cmd_len != s_len )
+		return false;
+
+	return strncasecmp(s, cmd, s_len) == 0;
+	}
+
 void SMTP_Analyzer::ProcessExtension(int ext_len, const char* ext)
 	{
 	if ( ! ext )
 		return;
 
-	if ( ! strncasecmp(ext, "PIPELINING", ext_len) )
+	if ( istrequal(ext, "PIPELINING", ext_len) )
 		pipelining = 1;
 	}
 
@@ -820,11 +842,11 @@ int SMTP_Analyzer::ParseCmd(int cmd_len, const char* cmd)
 		return -1;
 
 	// special case because we cannot define our usual macros with "-"
-	if ( strncmp(cmd, "X-ANONYMOUSTLS", cmd_len) == 0 )
+	if ( istrequal(cmd, "X-ANONYMOUSTLS", cmd_len) )
 		return SMTP_CMD_X_ANONYMOUSTLS;
 
 	for ( int code = SMTP_CMD_EHLO; code < SMTP_CMD_LAST; ++code )
-		if ( ! strncasecmp(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) )
+		if ( istrequal(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) )
 			return code;
 
 	return -1;

src/bro.bif

@@ -3598,6 +3598,8 @@ function lookup_hostname%(host: string%) : addr_set
 
 %%{
 #ifdef USE_GEOIP
+#include <chrono>
+
 extern "C" {
 #include <maxminddb.h>
 #include <sys/stat.h>
@@ -3608,17 +3610,24 @@ extern "C" {
 
 class MMDB {
 public:
-	MMDB(const char* filename);
+	MMDB(const char* filename, struct stat info);
 
 	~MMDB();
 
 	MMDB_lookup_result_s Lookup(const struct sockaddr* const sa);
+	bool StaleDB();
+	const char* Filename();
 
 private:
 	MMDB_s mmdb;
+	struct stat file_info;
+	bool lookup_error;
+	std::chrono::time_point<std::chrono::steady_clock> last_check;
 };
 
-MMDB::MMDB(const char *filename)
+MMDB::MMDB(const char* filename, struct stat info)
+	: file_info{info}, lookup_error{false},
+	  last_check{std::chrono::steady_clock::now()}
 	{
 	int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb);
 
@@ -3640,12 +3649,50 @@ MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa)
 
 	if ( MMDB_SUCCESS != mmdb_error )
 		{
+		lookup_error = true;
 		throw std::runtime_error(MMDB_strerror(mmdb_error));
 		}
 
 	return result;
 	}
 
+// Check to see if the Maxmind DB should be closed and reopened.  This will
+// happen if there was a lookup error or if the mmap'd file has been replaced
+// by an external process.
+bool MMDB::StaleDB()
+	{
+	struct stat buf;
+	using Clock = std::chrono::steady_clock;
+	std::chrono::time_point<Clock> now = Clock::now();
+
+	if ( lookup_error )
+		return true;
+
+	// Only perform stat once per 5 minutes.
+	using Min = std::chrono::minutes;
+	if ( std::chrono::duration_cast<Min>(now - last_check).count() < 5 )
+		return false;
+
+	last_check = now;
+
+	if ( 0 != stat(mmdb.filename, &buf) )
+		return true;
+
+	if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime )
+		{
+		reporter->Info("Inode change detected for MaxMind DB [%s]",
+		               mmdb.filename);
+		return true;
+		}
+
+	return false;
+	}
+
+const char* MMDB::Filename()
+	{
+	return mmdb.filename;
+	}
+
 std::unique_ptr<MMDB> mmdb_loc;
 std::unique_ptr<MMDB> mmdb_asn;
 
@@ -3662,11 +3709,11 @@ static bool mmdb_open(const char* filename, bool asn)
 		{
 		if ( asn )
 			{
-			mmdb_asn.reset(new MMDB(filename));
+			mmdb_asn.reset(new MMDB(filename, buf));
 			}
 		else
 			{
-			mmdb_loc.reset(new MMDB(filename));
+			mmdb_loc.reset(new MMDB(filename, buf));
 			}
 		}
 
@@ -3690,6 +3737,24 @@ static bool mmdb_open_asn(const char* filename)
 	return mmdb_open(filename, true);
 	}
 
+static void mmdb_check_loc()
+	{
+	if ( mmdb_loc && mmdb_loc->StaleDB() )
+		{
+		reporter->Info("Closing stale MaxMind DB [%s]", mmdb_loc->Filename());
+		mmdb_loc.release();
+		}
+	}
+
+static void mmdb_check_asn()
+	{
+	if ( mmdb_asn && mmdb_asn->StaleDB() )
+		{
+		reporter->Info("Closing stale MaxMind DB [%s]", mmdb_asn->Filename());
+		mmdb_asn.release();
+		}
+	}
+
 static bool mmdb_lookup(const IPAddr& addr, MMDB_lookup_result_s& result,
                         bool asn)
 	{
@@ -3873,6 +3938,7 @@ function lookup_location%(a: addr%) : geo_location
 	RecordVal* location = new RecordVal(geo_location);
 
 #ifdef USE_GEOIP
+	mmdb_check_loc();
 	if ( ! mmdb_loc )
 		{
 		if ( ! mmdb_try_open_loc() )
@@ -3950,6 +4016,7 @@ function lookup_location%(a: addr%) : geo_location
 function lookup_asn%(a: addr%) : count
 	%{
 #ifdef USE_GEOIP
+	mmdb_check_asn();
 	if ( ! mmdb_asn )
 		{
 		if ( ! mmdb_try_open_asn() )

src/reporter.bif

@@ -71,3 +71,44 @@ function Reporter::fatal%(msg: string%): bool
 	reporter->PopLocation();
 	return new Val(1, TYPE_BOOL);
 	%}
+
+## Generates a "net" weird.
+##
+## name: the name of the weird.
+##
+## Returns: Always true.
+function Reporter::net_weird%(name: string%): bool
+	%{
+	reporter->Weird(name->CheckString());
+	return new Val(1, TYPE_BOOL);
+	%}
+
+## Generates a "flow" weird.
+##
+## name: the name of the weird.
+##
+## orig: the originator host associated with the weird.
+##
+## resp: the responder host associated with the weird.
+##
+## Returns: Always true.
+function Reporter::flow_weird%(name: string, orig: addr, resp: addr%): bool
+	%{
+	reporter->Weird(orig->AsAddr(), resp->AsAddr(), name->CheckString());
+	return new Val(1, TYPE_BOOL);
+	%}
+
+## Generates a "conn" weird.
+##
+## name: the name of the weird.
+##
+## c: the connection associated with the weird.
+##
+## addl: additional information to accompany the weird.
+##
+## Returns: Always true.
+function Reporter::conn_weird%(name: string, c: connection, addl: string &default=""%): bool
+	%{
+	reporter->Weird(c, name->CheckString(), addl->CheckString());
+	return new Val(1, TYPE_BOOL);
+	%}

src/stats.bif

@@ -16,6 +16,7 @@ RecordType* ThreadStats;
 RecordType* TimerStats;
 RecordType* FileAnalysisStats;
 RecordType* BrokerStats;
+RecordType* ReporterStats;
 %%}
 
 ## Returns packet capture statistics. Statistics include the number of
@@ -35,6 +36,7 @@ RecordType* BrokerStats;
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_net_stats%(%): NetStats
 	%{
 	uint64 recv = 0;
@@ -83,6 +85,7 @@ function get_net_stats%(%): NetStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_conn_stats%(%): ConnStats
 	%{
 	RecordVal* r = new RecordVal(ConnStats);
@@ -133,6 +136,7 @@ function get_conn_stats%(%): ConnStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_proc_stats%(%): ProcStats
 	%{
 	struct rusage ru;
@@ -189,6 +193,7 @@ function get_proc_stats%(%): ProcStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_event_stats%(%): EventStats
 	%{
 	RecordVal* r = new RecordVal(EventStats);
@@ -215,6 +220,7 @@ function get_event_stats%(%): EventStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_reassembler_stats%(%): ReassemblerStats
 	%{
 	RecordVal* r = new RecordVal(ReassemblerStats);
@@ -243,6 +249,7 @@ function get_reassembler_stats%(%): ReassemblerStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_dns_stats%(%): DNSStats
 	%{
 	RecordVal* r = new RecordVal(DNSStats);
@@ -276,6 +283,7 @@ function get_dns_stats%(%): DNSStats
 ##              get_reassembler_stats
 ##              get_thread_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_timer_stats%(%): TimerStats
 	%{
 	RecordVal* r = new RecordVal(TimerStats);
@@ -303,6 +311,7 @@ function get_timer_stats%(%): TimerStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_file_analysis_stats%(%): FileAnalysisStats
 	%{
 	RecordVal* r = new RecordVal(FileAnalysisStats);
@@ -330,6 +339,7 @@ function get_file_analysis_stats%(%): FileAnalysisStats
 ##              get_reassembler_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_thread_stats%(%): ThreadStats
 	%{
 	RecordVal* r = new RecordVal(ThreadStats);
@@ -355,6 +365,7 @@ function get_thread_stats%(%): ThreadStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_gap_stats%(%): GapStats
 	%{
 	RecordVal* r = new RecordVal(GapStats);
@@ -386,6 +397,7 @@ function get_gap_stats%(%): GapStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_matcher_stats%(%): MatcherStats
 	%{
 	RecordVal* r = new RecordVal(MatcherStats);
@@ -423,6 +435,7 @@ function get_matcher_stats%(%): MatcherStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_broker_stats%(%): BrokerStats
 	%{
 	RecordVal* r = new RecordVal(BrokerStats);
@@ -441,3 +454,39 @@ function get_broker_stats%(%): BrokerStats
 
 	return r;
 	%}
+
+## Returns statistics about reporter messages and weirds.
+##
+## Returns: A record with reporter statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
+##              get_broker_stats
+function get_reporter_stats%(%): ReporterStats
+	%{
+	RecordVal* r = new RecordVal(ReporterStats);
+	int n = 0;
+
+	TableVal* weirds_by_type = new TableVal(internal_type("table_string_of_count")->AsTableType());
+
+	for ( auto& kv : reporter->GetWeirdsByType() )
+		{
+		Val* weird = new StringVal(kv.first);
+		weirds_by_type->Assign(weird, new Val(kv.second, TYPE_COUNT));
+		Unref(weird);
+		}
+
+	r->Assign(n++, new Val(reporter->GetWeirdCount(), TYPE_COUNT));
+	r->Assign(n++, weirds_by_type);
+
+	return r;
+	%}

src/threading/Manager.cc

@@ -160,6 +160,7 @@ void Manager::Process()
 	for ( all_thread_list::iterator i = to_delete.begin(); i != to_delete.end(); i++ )
 		{
 		BasicThread* t = *i;
+		t->WaitForStop();
 
 		all_threads.remove(t);
 

src/threading/MsgThread.cc

@@ -21,6 +21,8 @@ public:
 		network_time(network_time) { }
 
 	virtual bool Process()	{
+		if ( Object()->child_finished )
+			return true;
 		bool result = Object()->OnFinish(network_time);
 		Object()->Finished();
 		return result;
@@ -90,7 +92,13 @@ public:
 	KillMeMessage(MsgThread* thread)
 		: OutputMessage<MsgThread>("ReporterMessage", thread) 	{}
 
-	virtual bool Process()	{ thread_mgr->KillThread(Object()); return true; }
+	virtual bool Process()
+		{
+		Object()->SignalStop();
+		Object()->WaitForStop();
+		thread_mgr->KillThread(Object());
+		return true;
+		}
 };
 
 #ifdef DEBUG

src/threading/MsgThread.h

@@ -181,6 +181,7 @@ protected:
 	friend class HeartbeatMessage;
 	friend class FinishMessage;
 	friend class FinishedMessage;
+	friend class KillMeMessage;
 
 	/**
 	 * Pops a message sent by the child from the child-to-main queue.

testing/btest/Baseline/bifs.dump_current_packet/1.hex

@@ -0,0 +1,9 @@
+00000000  d4 c3 b2 a1 02 00 04 00  00 00 00 00 00 00 00 00  |................|
+00000010  00 24 00 00 01 00 00 00  1f ad 83 4d 17 79 01 00  |.$.........M.y..|
+00000020  57 00 00 00 57 00 00 00  01 00 5e 00 00 fb 00 30  |W...W.....^....0|
+00000030  48 bd 3e c4 08 00 45 00  00 49 00 00 40 00 ff 11  |H.>...E..I..@...|
+00000040  30 4f 8d 8e dc ca e0 00  00 fb 14 e9 14 e9 00 35  |0O.............5|
+00000050  7f 62 00 00 00 00 00 01  00 00 00 00 00 00 06 67  |.b.............g|
+00000060  65 6d 69 6e 69 09 5f 73  66 74 70 2d 73 73 68 04  |emini._sftp-ssh.|
+00000070  5f 74 63 70 05 6c 6f 63  61 6c 00 00 21 00 01     |_tcp.local..!..|
+0000007f

testing/btest/Baseline/bifs.dump_current_packet/2.hex

@@ -0,0 +1,17 @@
+00000000  d4 c3 b2 a1 02 00 04 00  00 00 00 00 00 00 00 00  |................|
+00000010  00 24 00 00 01 00 00 00  1f ad 83 4d f4 7a 01 00  |.$.........M.z..|
+00000020  d5 00 00 00 d5 00 00 00  33 33 00 00 00 fb 00 17  |........33......|
+00000030  f2 d7 cf 65 86 dd 60 00  00 00 00 9f 11 ff fe 80  |...e..`.........|
+00000040  00 00 00 00 00 00 02 17  f2 ff fe d7 cf 65 ff 02  |.............e..|
+00000050  00 00 00 00 00 00 00 00  00 00 00 00 00 fb 14 e9  |................|
+00000060  14 e9 00 9f 44 e9 00 00  84 00 00 00 00 01 00 00  |....D...........|
+00000070  00 04 06 67 65 6d 69 6e  69 09 5f 73 66 74 70 2d  |...gemini._sftp-|
+00000080  73 73 68 04 5f 74 63 70  05 6c 6f 63 61 6c 00 00  |ssh._tcp.local..|
+00000090  21 80 01 00 00 00 78 00  0f 00 00 00 00 00 16 06  |!.....x.........|
+000000a0  67 65 6d 69 6e 69 c0 22  c0 39 00 1c 80 01 00 00  |gemini.".9......|
+000000b0  00 78 00 10 fe 80 00 00  00 00 00 00 02 17 f2 ff  |.x..............|
+000000c0  fe d7 cf 65 c0 39 00 01  80 01 00 00 00 78 00 04  |...e.9.......x..|
+000000d0  8d 8e dc 32 c0 0c 00 2f  80 01 00 00 00 78 00 09  |...2.../.....x..|
+000000e0  c0 0c 00 05 00 00 80 00  40 c0 39 00 2f 80 01 00  |........@.9./...|
+000000f0  00 00 78 00 08 c0 39 00  04 40 00 00 08           |..x...9..@...|
+000000fd

testing/btest/Baseline/core.pcap.dumper/output

@@ -1,4 +1,4 @@
 2c2
 < 00000010  ff ff 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |...........Ns...|
 ---
-> 00000010  00 20 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |. .........Ns...|
+> 00000010  00 24 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |.$.........Ns...|

testing/btest/Baseline/core.reporter-weird-sampling/output

@@ -0,0 +1,351 @@
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird

testing/btest/Baseline/coverage.find-bro-logs/out

@@ -55,4 +55,5 @@ traceroute
 tunnel
 unified2
 weird
+weird_stats
 x509

testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output

@@ -36,7 +36,7 @@ export {
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
 	## allows you to require a minimum detection rate.
-	const notice_threshold = 10 &redef;
+	option notice_threshold = 10;
 }
 
 function do_mhr_lookup(hash: string, fi: Notice::FileInfo)

testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output

@@ -27,5 +27,5 @@ export {
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
 	## allows you to require a minimum detection rate.
-	const notice_threshold = 10 &redef;
+	option notice_threshold = 10;
 }

testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster-error/manager-reporter.log

@@ -1,12 +1,2 @@
-#separator \x09
+1535139819.649067	Reporter::INFO	qux	/home/jon/projects/bro/bro/testing/btest/.tmp/scripts.base.frameworks.logging.field-extension-cluster-error/field-extension-cluster-error.bro, line XX
-#set_separator	,
+1535139821.906059	bah	manager-1	0.000000	Reporter::INFO	qux	/home/jon/projects/bro/bro/testing/btest/.tmp/scripts.base.frameworks.logging.field-extension-cluster-error/field-extension-cluster-error.bro, line XX
-#empty_field	(empty)
-#unset_field	-
-#path	reporter
-#open	2018-04-27-22-48-04
-#fields	_write_ts	_stream	_system_name	ts	level	message	location
-#types	time	string	string	time	enum	string	string
-1524869284.624934	reporter	manager-1	0.000000	Reporter::WARNING	WriterFrontend broker/Log::WRITER_ASCII expected 9 fields in write, got 6. Skipping line.	(empty)
-1524869284.679015	reporter	manager-1	0.000000	Reporter::WARNING	WriterFrontend cluster/Log::WRITER_ASCII expected 6 fields in write, got 3. Skipping line.	(empty)
-1524869299.534389	reporter	manager-1	0.000000	Reporter::INFO	received termination signal	(empty)
-#close	2018-04-27-22-48-19

testing/btest/Baseline/scripts.base.protocols.dce-rpc.context/dce_rpc.log

@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	dce_rpc
+#open	2018-08-23-19-54-19
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	rtt	named_pipe	endpoint	operation
+#types	time	string	addr	port	addr	port	interval	string	string	string
+1347446180.330312	CHhAvVGS1DHFjwGM9	192.168.122.145	55614	192.168.122.3	1024	0.005544	\\PIPE\\drsuapi	drsuapi	DRSBind
+1347446180.336310	CHhAvVGS1DHFjwGM9	192.168.122.145	55614	192.168.122.3	1024	0.000788	\\PIPE\\drsuapi	drsuapi	DRSCrackNames
+1347446180.369165	CHhAvVGS1DHFjwGM9	192.168.122.145	55614	192.168.122.3	1024	0.000580	\\PIPE\\drsuapi	drsuapi	DRSUnbind
+#close	2018-08-23-19-54-19

testing/btest/Baseline/scripts.base.protocols.dce-rpc.context/out

@@ -0,0 +1,15 @@
+dce_rpc_bind :: fid  == 0
+dce_rpc_bind :: ctx_id  == 0
+dce_rpc_bind :: uuid == e3514235-4b06-11d1-ab04-00c04fc2dcd2
+dce_rpc_bind :: fid  == 0
+dce_rpc_bind :: ctx_id  == 1
+dce_rpc_bind :: uuid == e3514235-4b06-11d1-ab04-00c04fc2dcd2
+dce_rpc_bind :: fid  == 0
+dce_rpc_bind :: ctx_id  == 2
+dce_rpc_bind :: uuid == e3514235-4b06-11d1-ab04-00c04fc2dcd2
+dce_rpc_bind_ack :: fid      == 0
+dce_rpc_bind_ack :: sec_addr == \PIPE\drsuapi
+dce_rpc_alter_context :: fid  == 0
+dce_rpc_alter_context :: ctx_id  == 0
+dce_rpc_alter_context :: uuid == e3514235-4b06-11d1-ab04-00c04fc2dcd2
+dce_rpc_alter_context_resp :: fid      == 0

testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log

@@ -0,0 +1,13 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird_stats
+#open	2018-07-26-23-11-27
+#fields	ts	name	num_seen
+#types	time	string	count
+1532646687.827249	weird3	1
+1532646687.827249	weird2	1000
+1532646687.827249	weird1	2000
+1532646692.877464	weird1	2
+#close	2018-07-26-23-11-34

testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log

@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird_stats
+#open	2018-07-27-00-20-35
+#fields	ts	name	num_seen
+#types	time	string	count
+1532650834.978616	my_weird	1000
+1532650840.011592	my_weird	2000
+1532650845.043367	my_weird	10
+#close	2018-07-27-00-20-47

testing/btest/bifs/dump_current_packet.bro

@@ -1,6 +1,12 @@
 # @TEST-EXEC: bro -b -r $TRACES/wikipedia.trace %INPUT
-# @TEST-EXEC: btest-diff 1.pcap
+# @TEST-EXEC: hexdump -C 1.pcap >1.hex
-# @TEST-EXEC: btest-diff 2.pcap
+# @TEST-EXEC: hexdump -C 2.pcap >2.hex
+# @TEST-EXEC: btest-diff 1.hex
+# @TEST-EXEC: btest-diff 2.hex
+
+# Note that the hex output will contain global pcap header information,
+# including Bro's snaplen setting (so maybe check that out in the case
+# you are reading this message due to this test failing in the future).
 
 global i: count = 0;
 

testing/btest/core/leaks/input-raw.bro

@@ -6,9 +6,9 @@
 #
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT 
-# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 8 || (btest-bg-wait -k 1 && false)
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 60 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cat input2.log >> input.log
-# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got6 8 || (btest-bg-wait -k 1 && false)
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got6 15 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cat input3.log >> input.log
 # @TEST-EXEC: btest-bg-wait 60
 

testing/btest/core/pcap/dumper.bro

@@ -2,4 +2,8 @@
 # @TEST-EXEC: hexdump -C $TRACES/workshop_2011_browse.trace >1
 # @TEST-EXEC: hexdump -C dump >2
 # @TEST-EXEC: diff 1 2 >output || true
+
+# Note that we're diff'ing the diff because there is an expected
+# difference in the pcaps: namely, the snaplen setting stored in the
+# global pcap header.
 # @TEST-EXEC: btest-diff output

testing/btest/core/reporter-weird-sampling.bro

@@ -0,0 +1,55 @@
+# @TEST-EXEC: bro -b -r $TRACES/http/bro.org.pcap %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+redef Weird::sampling_duration = 5sec;
+redef Weird::sampling_threshold = 10;
+redef Weird::sampling_rate = 10;
+redef Weird::sampling_whitelist = set("whitelisted_net_weird",
+                                      "whitelisted_flow_weird",
+                                      "whitelisted_conn_weird");
+
+event conn_weird(name: string, c: connection, addl: string)
+	{
+	print "conn_weird", name;
+	}
+
+event flow_weird(name: string, src: addr, dst: addr)
+	{
+	print "flow_weird", name;
+	}
+
+event net_weird(name: string)
+	{
+	print "net_weird", name;
+	}
+
+event gen_weirds(c: connection)
+	{
+	local num = 30;
+
+	while ( num != 0 )
+		{
+		Reporter::net_weird("my_net_weird");
+		Reporter::flow_weird("my_flow_weird", c$id$orig_h, c$id$resp_h);
+		Reporter::conn_weird("my_conn_weird", c);
+
+		Reporter::net_weird("whitelisted_net_weird");
+		Reporter::flow_weird("whitelisted_flow_weird", c$id$orig_h, c$id$resp_h);
+		Reporter::conn_weird("whitelisted_conn_weird", c);
+		--num;
+		}
+	}
+
+global did_one_connection = F;
+
+event new_connection(c: connection)
+	{
+	if ( did_one_connection )
+		return;
+
+	did_one_connection = T;
+	event gen_weirds(c);             # should permit 10 + 2 of each "my" weird
+	schedule 2sec { gen_weirds(c) }; # should permit 3 of each "my" weird
+	schedule 7sec { gen_weirds(c) }; # should permit 10 + 2 of each "my" weird
+	# Total of 27 "my" weirds of each type and 90 of each "whitelisted" type
+	}

testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest

@@ -36,7 +36,7 @@ export {
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
 	## allows you to require a minimum detection rate.
-	const notice_threshold = 10 &redef;
+	option notice_threshold = 10;
 }
 
 function do_mhr_lookup(hash: string, fi: Notice::FileInfo)

testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest

@@ -27,5 +27,5 @@ export {
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
 	## allows you to require a minimum detection rate.
-	const notice_threshold = 10 &redef;
+	option notice_threshold = 10;
 }

testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro

@@ -3,8 +3,9 @@
 # @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT"
 # @TEST-EXEC: btest-bg-run worker-1  "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/wikipedia.trace %INPUT"
 # @TEST-EXEC: btest-bg-wait 20
-# @TEST-EXEC: cat manager-1/reporter.log | grep -v "reporter/" > manager-reporter.log
+# @TEST-EXEC: grep qux manager-1/reporter.log   | sed 's#line ..#line XX#g'  > manager-reporter.log
-# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-canonifier | grep -v ^# | $SCRIPTS/diff-sort" btest-diff manager-reporter.log
+# @TEST-EXEC: grep qux manager-1/reporter-2.log | sed 's#line ..*#line XX#g' >> manager-reporter.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-canonifier | $SCRIPTS/diff-remove-abspath | grep -v ^# | $SCRIPTS/diff-sort" btest-diff manager-reporter.log
 
 
 @TEST-START-FILE cluster-layout.bro
@@ -35,7 +36,7 @@ type Extension: record {
 function add_extension(path: string): Extension
 	{
 	return Extension($write_ts    = network_time(),
-	                 $stream      = path,
+	                 $stream      = "bah",
 	                 $system_name = peer_description);
 	}
 
@@ -56,16 +57,31 @@ event slow_death()
 
 event kill_worker()
 	{
+	Reporter::info("qux");
 	Broker::publish("death", slow_death);
 	}
 
 event bro_init()
 	{
 	if ( Cluster::node == "worker-1" )
+		{
 		Broker::subscribe("death");
+		suspend_processing();
+		}
+	}
 
+event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
+	{
 	if ( Cluster::node == "manager-1" )
-		schedule 13sec { kill_worker() };
+		{
+		schedule 2sec { kill_worker() };
+		}
+
+	if ( Cluster::node == "worker-1" )
+		{
+		continue_processing();
+		Reporter::info("qux");
+		}
 	}
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)

testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro

@@ -57,13 +57,19 @@ event kill_worker()
 event bro_init()
 	{
 	if ( Cluster::node == "worker-1" )
+		{
+		suspend_processing();
 		Broker::subscribe("death");
+		}
 	}
 
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	if ( Cluster::node == "manager-1" )
 		schedule 2sec { kill_worker() };
+
+	if ( Cluster::node == "worker-1" )
+		continue_processing();
 	}
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)

testing/btest/scripts/base/protocols/dce-rpc/context.bro

@@ -0,0 +1,31 @@
+# @TEST-EXEC: bro -b -C -r $TRACES/dce-rpc/cs_window7-join_stream092.pcap %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff dce_rpc.log
+
+@load base/protocols/dce-rpc
+
+event dce_rpc_bind(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count) &priority=5
+	{
+	print fmt("dce_rpc_bind :: fid  == %s", fid);
+	print fmt("dce_rpc_bind :: ctx_id  == %s", ctx_id);
+	print fmt("dce_rpc_bind :: uuid == %s", uuid_to_string(uuid));
+	}
+
+event dce_rpc_alter_context(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count) &priority=5
+	{
+	print fmt("dce_rpc_alter_context :: fid  == %s", fid);
+	print fmt("dce_rpc_alter_context :: ctx_id  == %s", ctx_id);
+	print fmt("dce_rpc_alter_context :: uuid == %s", uuid_to_string(uuid));
+	}
+
+
+event dce_rpc_bind_ack(c: connection, fid: count, sec_addr: string) &priority=5
+	{
+	print fmt("dce_rpc_bind_ack :: fid      == %s", fid);
+	print fmt("dce_rpc_bind_ack :: sec_addr == %s", sec_addr);
+	}
+
+event dce_rpc_alter_context_resp(c: connection, fid: count) &priority=5
+	{
+	print fmt("dce_rpc_alter_context_resp :: fid      == %s", fid);
+	}

testing/btest/scripts/policy/misc/weird-stats-cluster.bro

@@ -0,0 +1,93 @@
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
+# @TEST-EXEC: btest-bg-wait 20
+
+# @TEST-EXEC: btest-diff manager-1/weird_stats.log
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"],
+	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"],
+};
+@TEST-END-FILE
+
+@load misc/weird-stats
+
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
+redef Log::enable_local_logging = T;
+redef Log::default_rotation_interval = 0secs;
+redef WeirdStats::weird_stat_interval = 5secs;
+
+event terminate_me()
+	{
+	terminate();
+	}
+
+event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
+	{
+	terminate();
+	}
+
+event ready_again()
+	{
+	Reporter::net_weird("weird1");
+
+	if ( Cluster::node == "worker-2" )
+		{
+		schedule 5secs { terminate_me() };
+		}
+	}
+
+event ready_for_data()
+	{
+	local n = 0;
+
+	if ( Cluster::node == "worker-1" )
+		{
+		while ( n < 1000 )
+			{
+			Reporter::net_weird("weird1");
+			++n;
+			}
+
+		Reporter::net_weird("weird3");
+		}
+	else if ( Cluster::node == "worker-2" )
+		{
+		while ( n < 1000 )
+			{
+			Reporter::net_weird("weird1");
+			Reporter::net_weird("weird2");
+			++n;
+			}
+		}
+
+	schedule 5secs { ready_again() };
+	}
+
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+event bro_init()
+	{
+	Broker::auto_publish(Cluster::worker_topic, ready_for_data);
+	}
+
+global peer_count = 0;
+
+event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
+	{
+	++peer_count;
+
+	if ( peer_count == 2 )
+		event ready_for_data();
+	}
+
+@endif

testing/btest/scripts/policy/misc/weird-stats.bro

@@ -0,0 +1,32 @@
+# @TEST-EXEC: btest-bg-run bro bro %INPUT
+# @TEST-EXEC: btest-bg-wait 20
+# @TEST-EXEC: btest-diff bro/weird_stats.log
+
+@load misc/weird-stats.bro
+
+redef exit_only_after_terminate = T;
+redef WeirdStats::weird_stat_interval = 5sec;
+
+event die()
+	{
+	terminate();
+	}
+
+event gen_weirds(n: count, done: bool &default = F)
+	{
+	while ( n != 0 )
+		{
+		Reporter::net_weird("my_weird");
+		--n;
+		}
+
+	if ( done )
+		schedule 5sec { die() };
+	}
+
+event bro_init()
+	{
+	event gen_weirds(1000);
+	schedule 7.5sec { gen_weirds(2000) } ;
+	schedule 12.5sec { gen_weirds(10, T) } ;
+	}

testing/external/commit-hash.bro-testing

@@ -0,0 +1 @@
+489007e9bd035777cd1d33a9e1db9857449f805b

testing/external/commit-hash.bro-testing-private

@@ -0,0 +1 @@
+1917c0c602b18aa52150c7a940e9038a4382aa44

testing/scripts/diff-remove-timestamps

@@ -9,5 +9,5 @@ else
    sed="sed -E"
 fi
 
-$sed 's/(0\.000000)|([0-9]{10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \
+$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \
 $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'

testing/scripts/travis-job

@@ -198,10 +198,24 @@ run() {
         make init
     fi
 
+    if [ -d bro-testing ]; then
+        commit=`cat commit-hash.bro-testing`
+        echo "Checking out $commit"
+        ( cd bro-testing && git checkout -q $commit )
+    fi
+
+    echo
+
     if [ ! -d bro-testing-private ]; then
         get_private_tests
     fi
 
+    if [ -d bro-testing-private ]; then
+        commit=`cat commit-hash.bro-testing-private`
+        echo "Checking out $commit"
+        ( cd bro-testing-private && git checkout -q $commit )
+    fi
+
     echo
     echo "Running external tests ##############################################"
     echo