Merge remote-tracking branch 'origin/topic/robin/metrics-merge' into topic/bernhard/topk

CHANGES

@@ -1,4 +1,18 @@
 
+2.1-394 | 2013-04-27 15:02:31 -0700
+
+  * Fixed a bug in the vulnerable software script and added a test.
+    (Seth Hall)
+
+  * Fix schedule statements used outside event handlers. Addresses
+    #974. (Jon Siwek)
+
+  * Fix record coercion for default inner record fields. Addresses
+    #973. (Jon Siwek)
+
+  * Add bytestring_to_count function to bro.bif. Addresses #968. (Yun
+    Zheng Hu)
+
 2.1-386 | 2013-03-22 12:41:50 -0700
 
   * Added reverse() function to strings.bif. (Yun Zheng Hu)

NEWS

@@ -126,6 +126,9 @@ Changed Functionality
 - Removed the byte_len() and length() bif functions. Use the "|...|"
   operator instead.
 
+- The SSH::Login notice has been superseded by an corresponding
+  intelligence framework observation (SSH::SUCCESSFUL_LOGIN).
+
 Bro 2.1
 -------
 
@@ -209,6 +212,27 @@ New Functionality
   outputs. We do not yet recommend them for production (but welcome
   feedback!)
 
+- Summary statistics framework. [Extend]
+
+- A number of new applications build on top of the summary statistics
+  framework:
+
+    * Scan detection: Detectors for port and address scans return. See
+      policy/misc/scan.bro.
+
+    * Tracerouter detector: policy/misc/detect-traceroute
+
+    * Web application detection/measurement: policy/misc/app-metrics.bro
+
+    * FTP brute-forcing detector: policy/protocols/ftp/detect-bruteforcing.bro
+
+    * HTTP-based SQL injection detector: policy/protocols/http/detect-sqli.bro
+      (existed before, but now ported to the new framework)
+
+    * SSH brute-forcing detector feeding the intelligence framework:
+      policy/protocols/ssh/detect-bruteforcing.bro
+
+
 
 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~

VERSION

@@ -1 +1 @@
-2.1-386
+2.1-394

aux/binpac

@@ -1 +1 @@
-Subproject commit 72d121ade5a37df83d3252646de51cb77ce69a89
+Subproject commit a4b8dd0b691c3f614537ad8471fc80a82ce7b2df

aux/broctl

@@ -1 +1 @@
-Subproject commit 2b35d0331366865fbf0119919cc9692d55c4538c
+Subproject commit 786b83664c6a15faeb153d118310526b7790deae

scripts/base/frameworks/sumstats/cluster.bro

@@ -10,49 +10,48 @@
 module SumStats;
 
 export {
-	## Allows a user to decide how large of result groups the 
-	## workers should transmit values for cluster stats aggregation.
+	## Allows a user to decide how large of result groups the workers should transmit
+	## values for cluster stats aggregation.
 	const cluster_send_in_groups_of = 50 &redef;
 
-	## The percent of the full threshold value that needs to be met 
-	## on a single worker for that worker to send the value to its manager in
-	## order for it to request a global view for that value.  There is no
-	## requirement that the manager requests a global view for the key
-	## since it may opt not to if it requested a global view for the key
-	## recently.
+	## The percent of the full threshold value that needs to be met on a single worker
+	## for that worker to send the value to its manager in order for it to request a
+	## global view for that value.  There is no requirement that the manager requests
+	## a global view for the key since it may opt not to if it requested a global view
+	## for the key recently.
 	const cluster_request_global_view_percent = 0.2 &redef;
 
 	## This is to deal with intermediate update overload.  A manager will only allow
-	## this many intermediate update requests to the workers to be inflight at 
-	## any given time.  Requested intermediate updates are currently thrown out
-	## and not performed.  In practice this should hopefully have a minimal effect.
+	## this many intermediate update requests to the workers to be inflight at any
+	## given time.  Requested intermediate updates are currently thrown out and not
+	## performed.  In practice this should hopefully have a minimal effect.
 	const max_outstanding_global_views = 10 &redef;
 
-	## Intermediate updates can cause overload situations on very large clusters.
-	## This option may help reduce load and correct intermittent problems.
-	## The goal for this option is also meant to be temporary.
+	## Intermediate updates can cause overload situations on very large clusters. This
+	## option may help reduce load and correct intermittent problems. The goal for this
+	## option is also meant to be temporary.
 	const enable_intermediate_updates = T &redef;
 
-	## Event sent by the manager in a cluster to initiate the 
-	## collection of values for a sumstat.
+	## Event sent by the manager in a cluster to initiate the collection of values for
+	## a sumstat.
 	global cluster_ss_request: event(uid: string, ssid: string);
 
-	## Event sent by nodes that are collecting sumstats after receiving
-	## a request for the sumstat from the manager.
+	## Event sent by nodes that are collecting sumstats after receiving a request for
+	## the sumstat from the manager.
 	global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
 
-	## This event is sent by the manager in a cluster to initiate the
-	## collection of a single key value from a sumstat.  It's typically
-	## used to get intermediate updates before the break interval triggers
-	## to speed detection of a value crossing a threshold.
+	## This event is sent by the manager in a cluster to initiate the collection of
+	## a single key value from a sumstat.  It's typically used to get intermediate
+	## updates before the break interval triggers to speed detection of a value
+	## crossing a threshold.
 	global cluster_key_request: event(uid: string, ssid: string, key: Key);
 
 	## This event is sent by nodes in response to a
 	## :bro:id:`SumStats::cluster_key_request` event.
 	global cluster_key_response: event(uid: string, ssid: string, key: Key, result: Result);
 
-	## This is sent by workers to indicate that they crossed the percent of the 
-	## current threshold by the percentage defined globally in 
+	## This is sent by workers to indicate that they crossed the percent
+	## of the current threshold by the percentage defined globally in
 	## :bro:id:`SumStats::cluster_request_global_view_percent`
 	global cluster_key_intermediate_response: event(ssid: string, key: SumStats::Key);
 

scripts/base/frameworks/sumstats/main.bro

@@ -81,6 +81,7 @@ export {
 	## SumStats represent an aggregation of reducers along with
 	## mechanisms to handle various situations like the epoch ending
 	## or thresholds being crossed.
+	##
 	## It's best to not access any global state outside
 	## of the variables given to the callbacks because there
 	## is no assurance provided as to where the callbacks
@@ -181,16 +182,17 @@ global result_store: table[string] of ResultTable = table();
 # Store of threshold information.
 global thresholds_store: table[string, Key] of bool = table();
 
-# This is called whenever
-# key values are updated and the new val is given as the `val` argument.
-# It's only prototyped here because cluster and non-cluster have separate 
-# implementations.
+# This is called whenever key values are updated and the new val is given as the
+# `val` argument. It's only prototyped here because cluster and non-cluster have
+# separate  implementations.
 global data_added: function(ss: SumStat, key: Key, result: Result);
 
 # Prototype the hook point for plugins to do calculations.
 global observe_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
+
 # Prototype the hook point for plugins to initialize any result values.
 global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
+
 # Prototype the hook point for plugins to merge Results.
 global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
 

scripts/base/frameworks/sumstats/plugins/sample.bro

@@ -10,10 +10,8 @@ export {
 	};
 
 	redef record ResultVal += {
-		## This is the queue where samples
-		## are maintained.  Use the 
-		## :bro:see:`SumStats::get_samples` function
-		## to get a vector of the samples.
+		## This is the queue where samples are maintained.  Use the
+		## :bro:see:`SumStats::get_samples` function to get a vector of the samples.
 		samples: Queue::Queue &optional;
 	};
 

scripts/policy/frameworks/software/vulnerable.bro

@@ -43,15 +43,6 @@ export {
 
 global internal_vulnerable_versions: table[string] of set[VulnerableVersionRange] = table();
 
-event Control::configuration_update()
-	{
-	internal_vulnerable_versions = table();
-
-	# Copy the const vulnerable versions into the global modifiable one.
-	for ( sw in vulnerable_versions )
-		internal_vulnerable_versions[sw] = vulnerable_versions[sw];
-	}
-
 function decode_vulnerable_version_range(vuln_sw: string): VulnerableVersionRange
 	{
 	# Create a max value with a dunce value only because the $max field
@@ -115,11 +106,27 @@ event grab_vulnerable_versions(i: count)
 		}
 	}
 
-event bro_init()
+function update_vulnerable_sw()
 	{
+	internal_vulnerable_versions = table();
+
+	# Copy the const vulnerable versions into the global modifiable one.
+	for ( sw in vulnerable_versions )
+		internal_vulnerable_versions[sw] = vulnerable_versions[sw];
+
 	event grab_vulnerable_versions(1);
 	}
 
+event bro_init() &priority=3
+	{
+	update_vulnerable_sw();
+	}
+
+event Control::configuration_update() &priority=3
+	{
+	update_vulnerable_sw();
+	}
+
 event log_software(rec: Info)
 	{
 	if ( rec$name !in internal_vulnerable_versions )

scripts/policy/misc/detect-traceroute/main.bro

@@ -1,7 +1,7 @@
-##! This script detects large number of ICMP Time Exceeded messages heading
-##! toward hosts that have sent low TTL packets.
-##! It generates a notice when the number of ICMP Time Exceeded 
-##! messages for a source-destination pair exceeds threshold
+##! This script detects a large number of ICMP Time Exceeded messages heading toward
+##! hosts that have sent low TTL packets. It generates a notice when the number of
+##! ICMP Time Exceeded messages for a source-destination pair exceeds a
+##! threshold.
 @load base/frameworks/sumstats
 @load base/frameworks/signatures
 @load-sigs ./detect-low-ttls.sig

scripts/policy/misc/scan.bro

@@ -13,11 +13,12 @@ module Scan;
 
 export {
 	redef enum Notice::Type += {
-		## Address scans detect that a host appears to be scanning some number
-		## of hosts on a single port. This notice is generated when more than 
-		## :bro:id:`addr_scan_threshold` unique hosts are seen over the 
-		## previous :bro:id:`addr_scan_interval` time range.
+		## Address scans detect that a host appears to be scanning some number of
+		## destinations on a single port. This notice is generated when more than
+		## :bro:id:`addr_scan_threshold` unique hosts are seen over the previous
+		## :bro:id:`addr_scan_interval` time range.
 		Address_Scan,
+
 		## Port scans detect that an attacking host appears to be scanning a
 		## single victim host on several ports.  This notice is generated when
 		## an attacking host attempts to connect to :bro:id:`port_scan_threshold`
@@ -27,17 +28,19 @@ export {
 	};
 
 	## Failed connection attempts are tracked over this time interval for the address
-	## scan detection.  A higher interval will detect slower scanners, but may 
-	## also yield more false positives.
+	## scan detection.  A higher interval will detect slower scanners, but may also
+	## yield more false positives.
 	const addr_scan_interval = 5min &redef;
-	## Failed connection attempts are tracked over this time interval for the port 
-	## scan detection.  A higher interval will detect slower scanners, but may 
-	## also yield more false positives.
+
+	## Failed connection attempts are tracked over this time interval for the port scan
+	## detection.  A higher interval will detect slower scanners, but may also yield
+	## more false positives.
 	const port_scan_interval = 5min &redef;
 
 	## The threshold of a unique number of hosts a scanning host has to have failed
 	## connections with on a single port.
 	const addr_scan_threshold = 25 &redef;
+
 	## The threshold of a number of unique ports a scanning host has to have failed
 	## connections with on a single victim host.
 	const port_scan_threshold = 15 &redef;
@@ -147,9 +150,8 @@ function is_reverse_failed_conn(c: connection): bool
 ## Generated for an unsuccessful connection attempt. This
 ## event is raised when an originator unsuccessfully attempted
 ## to establish a connection. “Unsuccessful” is defined as at least
-## tcp_attempt_delay seconds having elapsed since the originator 
-## first sent a connection establishment packet to the destination 
-## without seeing a reply.
+## tcp_attempt_delay seconds having elapsed since the originator first sent a
+## connection establishment packet to the destination without seeing a reply.
 event connection_attempt(c: connection)
 	{
 	local is_reverse_scan = F;
@@ -159,9 +161,8 @@ event connection_attempt(c: connection)
 	add_sumstats(c$id, is_reverse_scan);
 	}
 
-## Generated for a rejected TCP connection. This event 
-## is raised when an originator attempted to setup a TCP 
-## connection but the responder replied with a RST packet 
+## Generated for a rejected TCP connection. This event is raised when an originator
+## attempted to setup a TCP connection but the responder replied with a RST packet
 ## denying it.
 event connection_rejected(c: connection)
 	{
@@ -172,9 +173,8 @@ event connection_rejected(c: connection)
 	add_sumstats(c$id, is_reverse_scan);
 	}
 
-## Generated when an endpoint aborted a TCP connection. 
-## The event is raised when one endpoint of an *established* 
-## TCP connection aborted by sending a RST packet.
+## Generated when an endpoint aborted a TCP connection. The event is raised when
+## one endpoint of an *established* TCP connection aborted by sending a RST packet.
 event connection_reset(c: connection)
 	{
 	if ( is_failed_conn(c) )

scripts/policy/protocols/ftp/detect-bruteforcing.bro

@@ -1,3 +1,5 @@
+##! FTP brute-forcing detector, triggering when too  many rejected usernames or
+##! failed passwords have occured from a single address.
 
 @load base/protocols/ftp
 @load base/frameworks/sumstats

testing/btest/Baseline/scripts.policy.frameworks.software.vulnerable/notice.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	notice
+#open	2013-04-28-22-36-26
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double
+1367188586.649122	-	-	-	-	-	-	Software::Vulnerable_Version	1.2.3.4 is running Java 1.7.0.15 which is vulnerable.	Java 1.7.0.15	1.2.3.4	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+1367188586.649122	-	-	-	-	-	-	Software::Vulnerable_Version	1.2.3.5 is running Java 1.6.0.43 which is vulnerable.	Java 1.6.0.43	1.2.3.5	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2013-04-28-22-36-26

testing/btest/scripts/policy/frameworks/software/vulnerable.bro

@@ -0,0 +1,23 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff notice.log
+
+@load frameworks/software/vulnerable
+
+redef Software::asset_tracking = ALL_HOSTS;
+
+global java_1_6_vuln: Software::VulnerableVersionRange = [$max=[$major=1,$minor=6,$minor2=0,$minor3=43]];
+global java_1_7_vuln: Software::VulnerableVersionRange = [$min=[$major=1,$minor=7], $max=[$major=1,$minor=7,$minor2=0,$minor3=20]];
+redef Software::vulnerable_versions += {
+        ["Java"] = set(java_1_6_vuln, java_1_7_vuln)
+};
+
+event bro_init()
+	{
+	Software::found([$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=4.3.2.1, $resp_p=80/tcp], 
+	                [$name="Java", $host=1.2.3.4, $version=[$major=1, $minor=7, $minor2=0, $minor3=15]]);
+	Software::found([$orig_h=1.2.3.5, $orig_p=1234/tcp, $resp_h=4.3.2.1, $resp_p=80/tcp], 
+	                [$name="Java", $host=1.2.3.5, $version=[$major=1, $minor=6, $minor2=0, $minor3=43]]);
+	Software::found([$orig_h=1.2.3.6, $orig_p=1234/tcp, $resp_h=4.3.2.1, $resp_p=80/tcp], 
+	                [$name="Java", $host=1.2.3.6, $version=[$major=1, $minor=6, $minor2=0, $minor3=50]]);
+
+	}