Layout tweaks for the sumstats code, and preliminary updates for NEWS.

The layout changes are mostly whitespace and some comment rewrapping.
No functional changes.

NEWS

@@ -126,6 +126,9 @@ Changed Functionality
 - Removed the byte_len() and length() bif functions. Use the "|...|"
   operator instead.
 
+- The SSH::Login notice has been superseded by an corresponding
+  intelligence framework observation (SSH::SUCCESSFUL_LOGIN).
+
 Bro 2.1
 -------
 
@@ -209,6 +212,27 @@ New Functionality
   outputs. We do not yet recommend them for production (but welcome
   feedback!)
 
+- Summary statistics framework. [Extend]
+
+- A number of new applications build on top of the summary statistics
+  framework:
+
+    * Scan detection: Detectors for port and address scans return. See
+      policy/misc/scan.bro.
+
+    * Tracerouter detector: policy/misc/detect-traceroute
+
+    * Web application detection/measurement: policy/misc/app-metrics.bro
+
+    * FTP brute-forcing detector: policy/protocols/ftp/detect-bruteforcing.bro
+
+    * HTTP-based SQL injection detector: policy/protocols/http/detect-sqli.bro
+      (existed before, but now ported to the new framework)
+
+    * SSH brute-forcing detector feeding the intelligence framework:
+      policy/protocols/ssh/detect-bruteforcing.bro
+
+
 
 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~

scripts/base/frameworks/sumstats/cluster.bro

@@ -10,49 +10,48 @@
 module SumStats;
 
 export {
-	## Allows a user to decide how large of result groups the 
+	## Allows a user to decide how large of result groups the workers should transmit
-	## workers should transmit values for cluster stats aggregation.
+	## values for cluster stats aggregation.
 	const cluster_send_in_groups_of = 50 &redef;
 
-	## The percent of the full threshold value that needs to be met 
+	## The percent of the full threshold value that needs to be met on a single worker
-	## on a single worker for that worker to send the value to its manager in
+	## for that worker to send the value to its manager in order for it to request a
-	## order for it to request a global view for that value.  There is no
+	## global view for that value.  There is no requirement that the manager requests
-	## requirement that the manager requests a global view for the key
+	## a global view for the key since it may opt not to if it requested a global view
-	## since it may opt not to if it requested a global view for the key
+	## for the key recently.
-	## recently.
 	const cluster_request_global_view_percent = 0.2 &redef;
 
 	## This is to deal with intermediate update overload.  A manager will only allow
-	## this many intermediate update requests to the workers to be inflight at 
+	## this many intermediate update requests to the workers to be inflight at any
-	## any given time.  Requested intermediate updates are currently thrown out
+	## given time.  Requested intermediate updates are currently thrown out and not
-	## and not performed.  In practice this should hopefully have a minimal effect.
+	## performed.  In practice this should hopefully have a minimal effect.
 	const max_outstanding_global_views = 10 &redef;
 
-	## Intermediate updates can cause overload situations on very large clusters.
+	## Intermediate updates can cause overload situations on very large clusters. This
-	## This option may help reduce load and correct intermittent problems.
+	## option may help reduce load and correct intermittent problems. The goal for this
-	## The goal for this option is also meant to be temporary.
+	## option is also meant to be temporary.
 	const enable_intermediate_updates = T &redef;
 
-	## Event sent by the manager in a cluster to initiate the 
+	## Event sent by the manager in a cluster to initiate the collection of values for
-	## collection of values for a sumstat.
+	## a sumstat.
 	global cluster_ss_request: event(uid: string, ssid: string);
 
-	## Event sent by nodes that are collecting sumstats after receiving
+	## Event sent by nodes that are collecting sumstats after receiving a request for
-	## a request for the sumstat from the manager.
+	## the sumstat from the manager.
 	global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
 
-	## This event is sent by the manager in a cluster to initiate the
+	## This event is sent by the manager in a cluster to initiate the collection of
-	## collection of a single key value from a sumstat.  It's typically
+	## a single key value from a sumstat.  It's typically used to get intermediate
-	## used to get intermediate updates before the break interval triggers
+	## updates before the break interval triggers to speed detection of a value
-	## to speed detection of a value crossing a threshold.
+	## crossing a threshold.
 	global cluster_key_request: event(uid: string, ssid: string, key: Key);
 
 	## This event is sent by nodes in response to a
 	## :bro:id:`SumStats::cluster_key_request` event.
 	global cluster_key_response: event(uid: string, ssid: string, key: Key, result: Result);
 
-	## This is sent by workers to indicate that they crossed the percent of the 
+	## This is sent by workers to indicate that they crossed the percent
-	## current threshold by the percentage defined globally in 
+	## of the current threshold by the percentage defined globally in
 	## :bro:id:`SumStats::cluster_request_global_view_percent`
 	global cluster_key_intermediate_response: event(ssid: string, key: SumStats::Key);
 

scripts/base/frameworks/sumstats/main.bro

@@ -81,6 +81,7 @@ export {
 	## SumStats represent an aggregation of reducers along with
 	## mechanisms to handle various situations like the epoch ending
 	## or thresholds being crossed.
+	##
 	## It's best to not access any global state outside
 	## of the variables given to the callbacks because there
 	## is no assurance provided as to where the callbacks
@@ -181,16 +182,17 @@ global result_store: table[string] of ResultTable = table();
 # Store of threshold information.
 global thresholds_store: table[string, Key] of bool = table();
 
-# This is called whenever
+# This is called whenever key values are updated and the new val is given as the
-# key values are updated and the new val is given as the `val` argument.
+# `val` argument. It's only prototyped here because cluster and non-cluster have
-# It's only prototyped here because cluster and non-cluster have separate 
+# separate  implementations.
-# implementations.
 global data_added: function(ss: SumStat, key: Key, result: Result);
 
 # Prototype the hook point for plugins to do calculations.
 global observe_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
+
 # Prototype the hook point for plugins to initialize any result values.
 global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
+
 # Prototype the hook point for plugins to merge Results.
 global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
 

scripts/base/frameworks/sumstats/plugins/sample.bro

@@ -10,10 +10,8 @@ export {
 	};
 
 	redef record ResultVal += {
-		## This is the queue where samples
+		## This is the queue where samples are maintained.  Use the
-		## are maintained.  Use the 
+		## :bro:see:`SumStats::get_samples` function to get a vector of the samples.
-		## :bro:see:`SumStats::get_samples` function
-		## to get a vector of the samples.
 		samples: Queue::Queue &optional;
 	};
 

scripts/policy/misc/detect-traceroute/main.bro

@@ -1,7 +1,7 @@
-##! This script detects large number of ICMP Time Exceeded messages heading
+##! This script detects a large number of ICMP Time Exceeded messages heading toward
-##! toward hosts that have sent low TTL packets.
+##! hosts that have sent low TTL packets. It generates a notice when the number of
-##! It generates a notice when the number of ICMP Time Exceeded 
+##! ICMP Time Exceeded messages for a source-destination pair exceeds a
-##! messages for a source-destination pair exceeds threshold
+##! threshold.
 @load base/frameworks/sumstats
 @load base/frameworks/signatures
 @load-sigs ./detect-low-ttls.sig

scripts/policy/misc/scan.bro

@@ -13,11 +13,12 @@ module Scan;
 
 export {
 	redef enum Notice::Type += {
-		## Address scans detect that a host appears to be scanning some number
+		## Address scans detect that a host appears to be scanning some number of
-		## of hosts on a single port. This notice is generated when more than 
+		## destinations on a single port. This notice is generated when more than
-		## :bro:id:`addr_scan_threshold` unique hosts are seen over the 
+		## :bro:id:`addr_scan_threshold` unique hosts are seen over the previous
-		## previous :bro:id:`addr_scan_interval` time range.
+		## :bro:id:`addr_scan_interval` time range.
 		Address_Scan,
+
 		## Port scans detect that an attacking host appears to be scanning a
 		## single victim host on several ports.  This notice is generated when
 		## an attacking host attempts to connect to :bro:id:`port_scan_threshold`
@@ -27,17 +28,19 @@ export {
 	};
 
 	## Failed connection attempts are tracked over this time interval for the address
-	## scan detection.  A higher interval will detect slower scanners, but may 
+	## scan detection.  A higher interval will detect slower scanners, but may also
-	## also yield more false positives.
+	## yield more false positives.
 	const addr_scan_interval = 5min &redef;
-	## Failed connection attempts are tracked over this time interval for the port 
+
-	## scan detection.  A higher interval will detect slower scanners, but may 
+	## Failed connection attempts are tracked over this time interval for the port scan
-	## also yield more false positives.
+	## detection.  A higher interval will detect slower scanners, but may also yield
+	## more false positives.
 	const port_scan_interval = 5min &redef;
 
 	## The threshold of a unique number of hosts a scanning host has to have failed
 	## connections with on a single port.
 	const addr_scan_threshold = 25 &redef;
+
 	## The threshold of a number of unique ports a scanning host has to have failed
 	## connections with on a single victim host.
 	const port_scan_threshold = 15 &redef;
@@ -147,9 +150,8 @@ function is_reverse_failed_conn(c: connection): bool
 ## Generated for an unsuccessful connection attempt. This
 ## event is raised when an originator unsuccessfully attempted
 ## to establish a connection. “Unsuccessful” is defined as at least
-## tcp_attempt_delay seconds having elapsed since the originator 
+## tcp_attempt_delay seconds having elapsed since the originator first sent a
-## first sent a connection establishment packet to the destination 
+## connection establishment packet to the destination without seeing a reply.
-## without seeing a reply.
 event connection_attempt(c: connection)
 	{
 	local is_reverse_scan = F;
@@ -159,9 +161,8 @@ event connection_attempt(c: connection)
 	add_sumstats(c$id, is_reverse_scan);
 	}
 
-## Generated for a rejected TCP connection. This event 
+## Generated for a rejected TCP connection. This event is raised when an originator
-## is raised when an originator attempted to setup a TCP 
+## attempted to setup a TCP connection but the responder replied with a RST packet
-## connection but the responder replied with a RST packet 
 ## denying it.
 event connection_rejected(c: connection)
 	{
@@ -172,9 +173,8 @@ event connection_rejected(c: connection)
 	add_sumstats(c$id, is_reverse_scan);
 	}
 
-## Generated when an endpoint aborted a TCP connection. 
+## Generated when an endpoint aborted a TCP connection. The event is raised when
-## The event is raised when one endpoint of an *established* 
+## one endpoint of an *established* TCP connection aborted by sending a RST packet.
-## TCP connection aborted by sending a RST packet.
 event connection_reset(c: connection)
 	{
 	if ( is_failed_conn(c) )

scripts/policy/protocols/ftp/detect-bruteforcing.bro

@@ -1,3 +1,5 @@
+##! FTP brute-forcing detector, triggering when too  many rejected usernames or
+##! failed passwords have occured from a single address.
 
 @load base/protocols/ftp
 @load base/frameworks/sumstats