Merge branch 'master' into topic/jsiwek/broxygen

CHANGES

@@ -1,4 +1,48 @@
 
+2.2-9 | 2013-11-18 14:03:21 -0800
+
+  * Update local.bro for Bro >= 2.2. The commented out Notice::policy
+    example didn't work anymore. (Daniel Thayer)
+
+2.2-6 | 2013-11-15 07:05:15 -0800
+
+  * Make "install-example-configs" target use DESTDIR. (Jon Siwek)
+    
+2.2-5 | 2013-11-11 13:47:54 -0800
+
+  * Fix the irc_reply event for certain server message types. (Seth
+    Hall)
+
+  * Fixed Segmentation fault in SQLite Writer. (Jon Crussell)
+    
+2.2 | 2013-11-07 10:25:50 -0800
+
+  * Release 2.2.
+
+  * Removing location information from ssh.log in external tests.
+    (Robin Sommer)
+
+2.2-beta-199 | 2013-11-07 00:36:46 -0800
+
+  * Fixing warnings during doc build. (Robin Sommer)
+
+2.2-beta-198 | 2013-11-06 22:54:30 -0800
+
+  * Update docs and tests for a recent change to detect-MHR.bro
+    (Daniel Thayer)
+
+  * Update tests and baselines for sumstats docs. (Daniel Thayer)
+
+2.2-beta-194 | 2013-11-06 14:39:50 -0500
+
+  * Remove resp_size from the ssh log. Refactor when we write out to
+    the log a bit. Geodata now works reliably. (Vlad Grigorescu)
+
+  * Update VirusTotal URL to work with changes to their website and
+    changed it to a redef. (Vlad Grigorescu)
+
+  * Added a document for the SumStats framework. (Seth Hall)
+
 2.2-beta-184 | 2013-11-03 22:53:42 -0800
 
   * Remove swig-ruby from required packages section of install doc.

NEWS

@@ -4,8 +4,8 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file
 (note that submodules, such as BroControl and Broccoli, come with
 their own ``CHANGES``.)
 
-Bro 2.2 Beta
+Bro 2.2
-============
+=======
 
 New Functionality
 -----------------

VERSION

@@ -1 +1 @@
-2.2-beta-184
+2.2-9

aux/binpac

@@ -1 +1 @@
-Subproject commit 0f20a50afacb68154b4035b6da63164d154093e4
+Subproject commit 54b321009b750268526419bdbd841f421c839313

aux/bro-aux

@@ -1 +1 @@
-Subproject commit ce8a9733c4f8c4c8e8b2e9e440acfcf985f39cd8
+Subproject commit ebf9c0d88ae8230845b91f15755156f93ff21aa8

aux/broccoli

@@ -1 +1 @@
-Subproject commit 02e5c1e1f993ef0fea3e2a59c34df9f40839e398
+Subproject commit 17ec437752837fb4214abfb0a2da49df74668d5d

aux/broctl

@@ -1 +1 @@
-Subproject commit aeb8501a50dcf33c53e7fe776b6e333327c11861
+Subproject commit 6e01d6972f02d68ee82d05f392d1a00725595b7f

aux/btest

@@ -1 +1 @@
-Subproject commit cfc8fe7ddf5ba3a9f957d1d5a98e9cfe1e9692ac
+Subproject commit 26c3136d56493017bc33c5a2f22ae393d585c2d9

cmake

@@ -1 +1 @@
-Subproject commit d902e23fd14624eb9caf0b4a0e693014bf5bd684
+Subproject commit e7a46cb82ee10aa522c4d88115baf10181277d20

doc/frameworks/index.rst

@@ -13,4 +13,5 @@ Frameworks
    logging
    notice
    signatures
+   sumstats
 

doc/frameworks/sumstats-countconns.bro

@@ -0,0 +1,36 @@
+@load base/frameworks/sumstats
+
+event connection_established(c: connection)
+	{
+	# Make an observation!
+	# This observation is global so the key is empty.
+	# Each established connection counts as one so the observation is always 1.
+	SumStats::observe("conn established", 
+	                  SumStats::Key(), 
+	                  SumStats::Observation($num=1));
+	}
+
+event bro_init()
+	{
+	# Create the reducer.
+	# The reducer attaches to the "conn established" observation stream
+	# and uses the summing calculation on the observations.
+	local r1 = SumStats::Reducer($stream="conn established", 
+	                             $apply=set(SumStats::SUM));
+
+	# Create the final sumstat.
+	# We give it an arbitrary name and make it collect data every minute.
+	# The reducer is then attached and a $epoch_result callback is given 
+	# to finally do something with the data collected.
+	SumStats::create([$name = "counting connections",
+	                  $epoch = 1min,
+	                  $reducers = set(r1),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	# This is the body of the callback that is called when a single 
+	                  	# result has been collected.  We are just printing the total number
+	                  	# of connections that were seen.  The $sum field is provided as a 
+	                  	# double type value so we need to use %f as the format specifier.
+	                  	print fmt("Number of connections established: %.0f", result["conn established"]$sum);
+	                  	}]);
+	}

doc/frameworks/sumstats-toy-scan.bro

@@ -0,0 +1,45 @@
+@load base/frameworks/sumstats
+
+# We use the connection_attempt event to limit our observations to those
+# which were attempted and not successful.
+event connection_attempt(c: connection)
+	{
+	# Make an observation!
+	# This observation is about the host attempting the connection.
+	# Each established connection counts as one so the observation is always 1.
+	SumStats::observe("conn attempted", 
+	                  SumStats::Key($host=c$id$orig_h), 
+	                  SumStats::Observation($num=1));
+	}
+
+event bro_init()
+	{
+	# Create the reducer.
+	# The reducer attaches to the "conn attempted" observation stream
+	# and uses the summing calculation on the observations. Keep
+	# in mind that there will be one result per key (connection originator).
+	local r1 = SumStats::Reducer($stream="conn attempted", 
+	                             $apply=set(SumStats::SUM));
+
+	# Create the final sumstat.
+	# This is slightly different from the last example since we're providing
+	# a callback to calculate a value to check against the threshold with 
+	# $threshold_val.  The actual threshold itself is provided with $threshold.
+	# Another callback is provided for when a key crosses the threshold.
+	SumStats::create([$name = "finding scanners",
+	                  $epoch = 5min,
+	                  $reducers = set(r1),
+	                  # Provide a threshold.
+	                  $threshold = 5.0,
+	                  # Provide a callback to calculate a value from the result
+	                  # to check against the threshold field.
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return result["conn attempted"]$sum;
+	                  	},
+	                  # Provide a callback for when a key crosses the threshold.
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum);
+	                  	}]);
+	}

doc/frameworks/sumstats.rst

@@ -0,0 +1,102 @@
+==================
+Summary Statistics
+==================
+
+.. rst-class:: opening
+
+    Measuring aspects of network traffic is an extremely common task in Bro.
+    Bro provides data structures which make this very easy as well in
+    simplistic cases such as size limited trace file processing. In
+    real-world deployments though, there are difficulties that arise from
+    clusterization (many processes sniffing traffic) and unbounded data sets
+    (traffic never stops). The Summary Statistics (otherwise referred to as
+    SumStats) framework aims to define a mechanism for consuming unbounded
+    data sets and making them measurable in practice on large clustered and
+    non-clustered Bro deployments.
+
+.. contents::
+
+Overview
+========
+
+The Sumstat processing flow is broken into three pieces. Observations, where
+some aspect of an event is observed and fed into the Sumstats framework.
+Reducers, where observations are collected and measured, typically by taking
+some sort of summary statistic measurement like average or variance (among
+others). Sumstats, where reducers have an epoch (time interval) that their
+measurements are performed over along with callbacks for monitoring thresholds
+or viewing the collected and measured data.
+
+Terminology
+===========
+
+    Observation
+
+        A single point of data. Observations have a few components of their
+        own. They are part of an arbitrarily named observation stream, they
+        have a key that is something the observation is about, and the actual
+        observation itself.
+
+    Reducer
+
+        Calculations are applied to an observation stream here to reduce the
+        full unbounded set of observations down to a smaller representation.
+        Results are collected within each reducer per-key so care must be
+        taken to keep the total number of keys tracked down to a reasonable
+        level.
+
+    Sumstat
+
+        The final definition of a Sumstat where one or more reducers is
+        collected over an interval, also known as an epoch. Thresholding can
+        be applied here along with a callback in the event that a threshold is
+        crossed. Additionally, a callback can be provided to access each
+        result (per-key) at the end of each epoch.
+
+Examples
+========
+
+These examples may seem very simple to an experienced Bro script developer and
+they're intended to look that way. Keep in mind that these scripts will work
+on small single process Bro instances as well as large many-worker clusters.
+The complications from dealing with flow based load balancing can be ignored
+by developers writing scripts that use Sumstats due to its built-in cluster
+transparency.
+
+Printing the number of connections
+----------------------------------
+
+Sumstats provides a simple way of approaching the problem of trying to count
+the number of connections over a given time interval.  Here is a script with
+inline documentation that does this with the Sumstats framework:
+
+.. btest-include:: ${DOC_ROOT}/frameworks/sumstats-countconns.bro
+
+When run on a sample PCAP file from the Bro test suite, the following output
+is created:
+
+.. btest:: sumstats-countconns
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/workshop_2011_browse.trace ${DOC_ROOT}/frameworks/sumstats-countconns.bro
+
+
+Toy scan detection
+------------------
+
+Taking the previous example even further, we can implement a simple detection
+to demonstrate the thresholding functionality.  This example is a toy to
+demonstrate how thresholding works in Sumstats and is not meant to be a
+real-world functional example, that is left to the
+:doc:`/scripts/policy/misc/scan` script that is included with Bro.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro
+
+Let's see if there are any hosts that crossed the threshold in a PCAP file
+containing a host running nmap:
+
+.. btest:: sumstats-toy-scan
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/nmap-vsn.trace ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro
+
+It seems the host running nmap was detected!
+

doc/scripting/index.rst

@@ -66,7 +66,7 @@ are ensuring the Files framework, the Notice framework and the script to hash al
 been loaded by Bro.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
-   :lines: 10-31
+   :lines: 10-36
 
 The export section redefines an enumerable constant that describes the
 type of notice we will generate with the Notice framework.  Bro
@@ -87,7 +87,7 @@ Up until this point, the script has merely done some basic setup.  With the next
 the script starts to define instructions to take in a given event.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
-   :lines: 33-57
+   :lines: 38-62
 
 The workhorse of the script is contained in the event handler for
 ``file_hash``.  The :bro:see:`file_hash` event allows scripts to access

doc/scripts/notices.rst

@@ -0,0 +1,8 @@
+
+.. Not nice but I don't find a way to link to the notice index
+.. directly from the upper level TOC tree.
+
+Notices
+=======
+
+See the `Bro Notice Index <../bro-noticeindex.html>`_.

scripts/base/frameworks/packet-filter/cluster.bro

@@ -1,4 +1,7 @@
 
+@load base/frameworks/cluster
+@load ./main
+
 module PacketFilter;
 
 event remote_connection_handshake_done(p: event_peer) &priority=3

scripts/base/protocols/ssh/main.bro

@@ -37,12 +37,6 @@ export {
 		client:          string       &log &optional;
 		## Software string from the server.
 		server:          string       &log &optional;
-		## Amount of data returned from the server. This is currently
-		## the only measure of the success heuristic and it is logged to
-		## assist analysts looking at the logs to make their own
-		## determination about the success on a case-by-case basis.
-		resp_size:       count        &log &default=0;
-
 		## Indicate if the SSH session is done being watched.
 		done:            bool         &default=F;
 	};
@@ -107,10 +101,10 @@ function check_ssh_connection(c: connection, done: bool)
 		# this matches the conditions for a failed login.  Failed
 		# logins are only detected at connection state removal.
 
-		if ( # Require originators to have sent at least 50 bytes.
+		if ( # Require originators and responders to have sent at least 50 bytes.
-		     c$orig$size > 50 &&
+		     c$orig$size > 50 && c$resp$size > 50 &&
 		     # Responders must be below 4000 bytes.
-		     c$resp$size < 4000 &&
+		     c$resp$size < authentication_data_size &&
 		     # Responder must have sent fewer than 40 packets.
 		     c$resp$num_pkts < 40 &&
 		     # If there was a content gap we can't reliably do this heuristic.
@@ -122,7 +116,7 @@ function check_ssh_connection(c: connection, done: bool)
 			event SSH::heuristic_failed_login(c);
 			}
 
-		if ( c$resp$size > authentication_data_size )
+		if ( c$resp$size >= authentication_data_size )
 			{
 			c$ssh$status = "success";
 			event SSH::heuristic_successful_login(c);
@@ -132,7 +126,7 @@ function check_ssh_connection(c: connection, done: bool)
 		{
 		# If this connection is still being tracked, then it's possible
 		# to watch for it to be a successful connection.
-		if ( c$resp$size > authentication_data_size )
+		if ( c$resp$size >= authentication_data_size )
 			{
 			c$ssh$status = "success";
 			event SSH::heuristic_successful_login(c);
@@ -150,8 +144,6 @@ function check_ssh_connection(c: connection, done: bool)
 	# after detection is done.
 	c$ssh$done=T;
 
-	Log::write(SSH::LOG, c$ssh);
-
 	if ( skip_processing_after_detection )
 		{
 		# Stop watching this connection, we don't care about it anymore.
@@ -161,10 +153,24 @@ function check_ssh_connection(c: connection, done: bool)
 	}
 
 
+event heuristic_successful_login(c: connection) &priority=-5
+	{
+	Log::write(SSH::LOG, c$ssh);
+	}
+
+event heuristic_failed_login(c: connection) &priority=-5
+	{
+	Log::write(SSH::LOG, c$ssh);
+	}
+
 event connection_state_remove(c: connection) &priority=-5
 	{
 	if ( c?$ssh )
+		{
 		check_ssh_connection(c, T);
+		if ( c$ssh$status == "undetermined" )
+			Log::write(SSH::LOG, c$ssh);
+		}
 	}
 
 event ssh_watcher(c: connection)

scripts/policy/frameworks/files/detect-MHR.bro

@@ -23,6 +23,11 @@ export {
 	                         /application\/jar/ |
 	                         /video\/mp4/ &redef;
 
+	## The Match notice has a sub message with a URL where you can get more
+	## information about the file. The %s will be replaced with the SHA-1
+	## hash of the file.
+	const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef;
+
 	## The malware hash registry runs each malware sample through several
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
@@ -48,7 +53,7 @@ event file_hash(f: fa_file, kind: string, hash: string)
 				if ( mhr_detect_rate >= notice_threshold )
 					{
 					local message = fmt("Malware Hash Registry Detection rate: %d%%  Last seen: %s", mhr_detect_rate, readable_first_detected);
-					local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
+					local virustotal_url = fmt(match_sub_url, hash);
 					NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
 					}
 				}

scripts/policy/protocols/ssh/geo-data.bro

@@ -24,21 +24,29 @@ export {
 	const watched_countries: set[string] = {"RO"} &redef;
 }
 
+function get_location(c: connection): geo_location
+	{
+	local lookup_ip = (c$ssh$direction == OUTBOUND) ? c$id$resp_h : c$id$orig_h;
+	return lookup_location(lookup_ip);
+	}
+
 event SSH::heuristic_successful_login(c: connection) &priority=5
 	{
-	local location: geo_location;
-	location = (c$ssh$direction == OUTBOUND) ? 
-		lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
-	
 	# Add the location data to the SSH record.
-	c$ssh$remote_location = location;
+	c$ssh$remote_location = get_location(c);
 	
-	if ( location?$country_code && location$country_code in watched_countries )
+	if ( c$ssh$remote_location?$country_code && c$ssh$remote_location$country_code in watched_countries )
 		{
 		NOTICE([$note=Watched_Country_Login,
 		        $conn=c,
 		        $msg=fmt("SSH login %s watched country: %s", 
 		                 (c$ssh$direction == OUTBOUND) ? "to" : "from", 
-		                 location$country_code)]);
+		                 c$ssh$remote_location$country_code)]);
 		}
 	}
+
+event SSH::heuristic_failed_login(c: connection) &priority=5
+	{
+	# Add the location data to the SSH record.
+	c$ssh$remote_location = get_location(c);
+	}

scripts/site/local.bro

@@ -30,10 +30,6 @@
 # This adds signatures to detect cleartext forward and reverse windows shells.
 @load-sigs frameworks/signatures/detect-windows-shells
 
-# Uncomment the following line to begin receiving (by default hourly) emails
-# containing all of your notices.
-# redef Notice::policy += { [$action = Notice::ACTION_ALARM, $priority = 0] };
-
 # Load all of the scripts that detect software in various protocols.
 @load protocols/ftp/software
 @load protocols/smtp/software

src/analyzer/protocol/irc/IRC.cc

@@ -164,7 +164,7 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig)
 	if ( code > 0 )
 		{
 		switch ( code ) {
-		// Ignore unimportant messages.
+		/*
 		case 1: // RPL_WELCOME
 		case 2: // RPL_YOURHOST
 		case 3: // RPL_CREATED
@@ -183,6 +183,7 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig)
 		case 376: // RPL_ENDOFMOTD
 		case 331: // RPL_NOTOPIC
 			break;
+		*/
 
 		// Count of users, services and servers in whole network.
 		case 251:

src/logging/writers/SQLite.cc

@@ -308,7 +308,7 @@ int SQLite::AddParams(Value* val, int pos)
 				if ( j > 0 )
 					desc.AddRaw(set_separator);
 
-				io->Describe(&desc, val->val.set_val.vals[j], fields[pos]->name);
+				io->Describe(&desc, val->val.set_val.vals[j], fields[pos-1]->name);
 				}
 
 		desc.RemoveEscapeSequence(set_separator);
@@ -330,7 +330,7 @@ int SQLite::AddParams(Value* val, int pos)
 				if ( j > 0 )
 					desc.AddRaw(set_separator);
 
-				io->Describe(&desc, val->val.vector_val.vals[j], fields[pos]->name);
+				io->Describe(&desc, val->val.vector_val.vals[j], fields[pos-1]->name);
 				}
 
 		desc.RemoveEscapeSequence(set_separator);

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-countconns_bro/output

@@ -0,0 +1,40 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+sumstats-countconns.bro
+
+@load base/frameworks/sumstats
+
+event connection_established(c: connection)
+	{
+	# Make an observation!
+	# This observation is global so the key is empty.
+	# Each established connection counts as one so the observation is always 1.
+	SumStats::observe("conn established", 
+	                  SumStats::Key(), 
+	                  SumStats::Observation($num=1));
+	}
+
+event bro_init()
+	{
+	# Create the reducer.
+	# The reducer attaches to the "conn established" observation stream
+	# and uses the summing calculation on the observations.
+	local r1 = SumStats::Reducer($stream="conn established", 
+	                             $apply=set(SumStats::SUM));
+
+	# Create the final sumstat.
+	# We give it an arbitrary name and make it collect data every minute.
+	# The reducer is then attached and a $epoch_result callback is given 
+	# to finally do something with the data collected.
+	SumStats::create([$name = "counting connections",
+	                  $epoch = 1min,
+	                  $reducers = set(r1),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	# This is the body of the callback that is called when a single 
+	                  	# result has been collected.  We are just printing the total number
+	                  	# of connections that were seen.  The $sum field is provided as a 
+	                  	# double type value so we need to use %f as the format specifier.
+	                  	print fmt("Number of connections established: %.0f", result["conn established"]$sum);
+	                  	}]);
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output

@@ -0,0 +1,49 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+sumstats-toy-scan.bro
+
+@load base/frameworks/sumstats
+
+# We use the connection_attempt event to limit our observations to those
+# which were attempted and not successful.
+event connection_attempt(c: connection)
+	{
+	# Make an observation!
+	# This observation is about the host attempting the connection.
+	# Each established connection counts as one so the observation is always 1.
+	SumStats::observe("conn attempted", 
+	                  SumStats::Key($host=c$id$orig_h), 
+	                  SumStats::Observation($num=1));
+	}
+
+event bro_init()
+	{
+	# Create the reducer.
+	# The reducer attaches to the "conn attempted" observation stream
+	# and uses the summing calculation on the observations. Keep
+	# in mind that there will be one result per key (connection originator).
+	local r1 = SumStats::Reducer($stream="conn attempted", 
+	                             $apply=set(SumStats::SUM));
+
+	# Create the final sumstat.
+	# This is slightly different from the last example since we're providing
+	# a callback to calculate a value to check against the threshold with 
+	# $threshold_val.  The actual threshold itself is provided with $threshold.
+	# Another callback is provided for when a key crosses the threshold.
+	SumStats::create([$name = "finding scanners",
+	                  $epoch = 5min,
+	                  $reducers = set(r1),
+	                  # Provide a threshold.
+	                  $threshold = 5.0,
+	                  # Provide a callback to calculate a value from the result
+	                  # to check against the threshold field.
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return result["conn attempted"]$sum;
+	                  	},
+	                  # Provide a callback for when a key crosses the threshold.
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum);
+	                  	}]);
+	}

testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output

@@ -27,6 +27,11 @@ export {
 	                         /application\/jar/ |
 	                         /video\/mp4/ &redef;
 
+	## The Match notice has a sub message with a URL where you can get more
+	## information about the file. The %s will be replaced with the SHA-1
+	## hash of the file.
+	const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef;
+
 	## The malware hash registry runs each malware sample through several
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
@@ -52,7 +57,7 @@ event file_hash(f: fa_file, kind: string, hash: string)
 				if ( mhr_detect_rate >= notice_threshold )
 					{
 					local message = fmt("Malware Hash Registry Detection rate: %d%%  Last seen: %s", mhr_detect_rate, readable_first_detected);
-					local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
+					local virustotal_url = fmt(match_sub_url, hash);
 					NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
 					}
 				}

testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output

@@ -18,6 +18,11 @@ export {
 	                         /application\/jar/ |
 	                         /video\/mp4/ &redef;
 
+	## The Match notice has a sub message with a URL where you can get more
+	## information about the file. The %s will be replaced with the SHA-1
+	## hash of the file.
+	const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef;
+
 	## The malware hash registry runs each malware sample through several
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold

testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output

@@ -20,7 +20,7 @@ event file_hash(f: fa_file, kind: string, hash: string)
 				if ( mhr_detect_rate >= notice_threshold )
 					{
 					local message = fmt("Malware Hash Registry Detection rate: %d%%  Last seen: %s", mhr_detect_rate, readable_first_detected);
-					local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
+					local virustotal_url = fmt(match_sub_url, hash);
 					NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
 					}
 				}

testing/btest/Baseline/doc.sphinx.sumstats-countconns/btest-doc.sphinx.sumstats-countconns#1

@@ -0,0 +1,9 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r workshop_2011_browse.trace sumstats-countconns.bro
+      Number of connections established: 6
+

testing/btest/Baseline/doc.sphinx.sumstats-toy-scan/btest-doc.sphinx.sumstats-toy-scan#1

@@ -0,0 +1,9 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r nmap-vsn.trace sumstats-toy-scan.bro
+      192.168.1.71 attempted 5 or more connections
+

testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.set/ssh.select

@@ -0,0 +1 @@
+CC,AA,BB

testing/btest/doc/sphinx/include-doc_frameworks_sumstats-countconns_bro.btest

@@ -0,0 +1,40 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+sumstats-countconns.bro
+
+@load base/frameworks/sumstats
+
+event connection_established(c: connection)
+	{
+	# Make an observation!
+	# This observation is global so the key is empty.
+	# Each established connection counts as one so the observation is always 1.
+	SumStats::observe("conn established", 
+	                  SumStats::Key(), 
+	                  SumStats::Observation($num=1));
+	}
+
+event bro_init()
+	{
+	# Create the reducer.
+	# The reducer attaches to the "conn established" observation stream
+	# and uses the summing calculation on the observations.
+	local r1 = SumStats::Reducer($stream="conn established", 
+	                             $apply=set(SumStats::SUM));
+
+	# Create the final sumstat.
+	# We give it an arbitrary name and make it collect data every minute.
+	# The reducer is then attached and a $epoch_result callback is given 
+	# to finally do something with the data collected.
+	SumStats::create([$name = "counting connections",
+	                  $epoch = 1min,
+	                  $reducers = set(r1),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	# This is the body of the callback that is called when a single 
+	                  	# result has been collected.  We are just printing the total number
+	                  	# of connections that were seen.  The $sum field is provided as a 
+	                  	# double type value so we need to use %f as the format specifier.
+	                  	print fmt("Number of connections established: %.0f", result["conn established"]$sum);
+	                  	}]);
+	}

testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest

@@ -0,0 +1,49 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+sumstats-toy-scan.bro
+
+@load base/frameworks/sumstats
+
+# We use the connection_attempt event to limit our observations to those
+# which were attempted and not successful.
+event connection_attempt(c: connection)
+	{
+	# Make an observation!
+	# This observation is about the host attempting the connection.
+	# Each established connection counts as one so the observation is always 1.
+	SumStats::observe("conn attempted", 
+	                  SumStats::Key($host=c$id$orig_h), 
+	                  SumStats::Observation($num=1));
+	}
+
+event bro_init()
+	{
+	# Create the reducer.
+	# The reducer attaches to the "conn attempted" observation stream
+	# and uses the summing calculation on the observations. Keep
+	# in mind that there will be one result per key (connection originator).
+	local r1 = SumStats::Reducer($stream="conn attempted", 
+	                             $apply=set(SumStats::SUM));
+
+	# Create the final sumstat.
+	# This is slightly different from the last example since we're providing
+	# a callback to calculate a value to check against the threshold with 
+	# $threshold_val.  The actual threshold itself is provided with $threshold.
+	# Another callback is provided for when a key crosses the threshold.
+	SumStats::create([$name = "finding scanners",
+	                  $epoch = 5min,
+	                  $reducers = set(r1),
+	                  # Provide a threshold.
+	                  $threshold = 5.0,
+	                  # Provide a callback to calculate a value from the result
+	                  # to check against the threshold field.
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return result["conn attempted"]$sum;
+	                  	},
+	                  # Provide a callback for when a key crosses the threshold.
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum);
+	                  	}]);
+	}

testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest

@@ -27,6 +27,11 @@ export {
 	                         /application\/jar/ |
 	                         /video\/mp4/ &redef;
 
+	## The Match notice has a sub message with a URL where you can get more
+	## information about the file. The %s will be replaced with the SHA-1
+	## hash of the file.
+	const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef;
+
 	## The malware hash registry runs each malware sample through several
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
@@ -52,7 +57,7 @@ event file_hash(f: fa_file, kind: string, hash: string)
 				if ( mhr_detect_rate >= notice_threshold )
 					{
 					local message = fmt("Malware Hash Registry Detection rate: %d%%  Last seen: %s", mhr_detect_rate, readable_first_detected);
-					local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
+					local virustotal_url = fmt(match_sub_url, hash);
 					NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
 					}
 				}

testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest

@@ -18,6 +18,11 @@ export {
 	                         /application\/jar/ |
 	                         /video\/mp4/ &redef;
 
+	## The Match notice has a sub message with a URL where you can get more
+	## information about the file. The %s will be replaced with the SHA-1
+	## hash of the file.
+	const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef;
+
 	## The malware hash registry runs each malware sample through several
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold

testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest

@@ -20,7 +20,7 @@ event file_hash(f: fa_file, kind: string, hash: string)
 				if ( mhr_detect_rate >= notice_threshold )
 					{
 					local message = fmt("Malware Hash Registry Detection rate: %d%%  Last seen: %s", mhr_detect_rate, readable_first_detected);
-					local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
+					local virustotal_url = fmt(match_sub_url, hash);
 					NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
 					}
 				}

testing/btest/doc/sphinx/sumstats-countconns.btest

@@ -0,0 +1 @@
+@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/workshop_2011_browse.trace ${DOC_ROOT}/frameworks/sumstats-countconns.bro

testing/btest/doc/sphinx/sumstats-toy-scan.btest

@@ -0,0 +1 @@
+@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/nmap-vsn.trace ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro

testing/btest/scripts/base/frameworks/logging/sqlite/set.bro

@@ -0,0 +1,50 @@
+#
+# Check if set works in last position (the describe call in sqlite.cc has a good
+# chance of being off by one if someone changes it).
+#
+# @TEST-REQUIRES: which sqlite3
+# @TEST-REQUIRES: has-writer SQLite
+# @TEST-GROUP: sqlite
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: sqlite3 ssh.sqlite 'select * from ssh' > ssh.select
+# @TEST-EXEC: btest-diff ssh.select
+#
+# Testing all possible types.
+
+redef LogSQLite::unset_field = "(unset)";
+
+module SSH;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		ss: set[string];
+	} &log;
+}
+
+function foo(i : count) : string
+	{
+	if ( i > 0 )
+		return "Foo";
+	else
+		return "Bar";
+	}
+
+event bro_init()
+{
+	Log::create_stream(SSH::LOG, [$columns=Log]);
+	Log::remove_filter(SSH::LOG, "default");
+
+	local filter: Log::Filter = [$name="sqlite", $path="ssh", $writer=Log::WRITER_SQLITE];
+	Log::add_filter(SSH::LOG, filter);
+
+	local empty_set: set[string];
+	local empty_vector: vector of string;
+
+	Log::write(SSH::LOG, [
+		$ss=set("AA", "BB", "CC")
+		]);
+}
+

testing/scripts/diff-canonifier-external

@@ -8,6 +8,10 @@ if [ "$1" == "capture_loss.log" ]; then
     addl="`dirname $0`/diff-remove-fractions"
 fi
 
+if [ "$1" == "ssh.log" ]; then
+    addl="`dirname $0`/diff-remove-fields remote_location"
+fi
+
 `dirname $0`/diff-remove-timestamps \
     | `dirname $0`/diff-remove-uids \
     | `dirname $0`/diff-remove-file-ids \

testing/scripts/diff-remove-fields

@@ -0,0 +1,34 @@
+#! /usr/bin/env bash
+#
+# A diff canonifier that removes all fields starting with a given
+# prefix.
+
+if [ $# != 1 ]; then
+   echo "usage: `basename $0` <field prefix>"
+   exit 1
+fi
+
+awk -v "PREFIX=$1" '
+
+BEGIN { FS="\t"; OFS="\t"; }
+
+/^#fields/ {
+    for ( i = 2; i <= NF; ++i )
+        {
+        if ( index($i, PREFIX) == 1 )
+            rem[i-1] = 1;
+        }
+     print;
+     next;
+}
+
+{
+    for ( i in rem )
+       # Mark that it iss set, but ignore content.
+       $i = "+";
+
+    print;
+}
+
+'
+