Merge remote-tracking branch 'origin/master' into topic/bernhard/hyperloglog

CHANGES

@@ -1,4 +1,38 @@
 
+2.1-498 | 2013-05-03 17:44:08 -0700
+
+  * Table lookups return copy of non-const &default vals. This
+    prevents unintentional modifications to the &default value itself.
+    Addresses #981.  (Jon Siwek)
+
+2.1-496 | 2013-05-03 15:54:47 -0700
+
+  * Fix memory leak and unnecessary allocations in OpaqueVal.
+    Addresses #986. (Matthias Vallentin)
+
+2.1-492 | 2013-05-02 12:46:26 -0700
+
+  * Work-around for sumstats framework not propagating updates after
+    intermediate check in cluster environments. (Bernhard Amann)
+
+  * Always apply tcp_connection_attempt. Before this change it was
+    only applied when a connection_attempt() event handler was
+    defined. (Robin Sommer)
+
+  * Fixing coverage.bare-mode-errors test. (Robin Sommer)
+
+2.1-487 | 2013-05-01 18:03:22 -0700
+
+  * Always apply tcp_connection_attempt timer, even if no
+    connection_attempt() event handler is defined. (Robin Sommer)
+
+2.1-486 | 2013-05-01 15:28:45 -0700
+
+  * New framework for computing summary statistics in
+    base/framework/sumstats. This replaces the metrics frameworks, and
+    comes with a number of applications build on top, see NEWS. More
+    documentation to follow. (Seth Hall)
+
 2.1-397 | 2013-04-29 21:19:00 -0700
 
   * Fixing memory leaks in CompHash implementation. Addresses #987.

NEWS

@@ -126,6 +126,9 @@ Changed Functionality
 - Removed the byte_len() and length() bif functions. Use the "|...|"
   operator instead.
 
+- The SSH::Login notice has been superseded by an corresponding
+  intelligence framework observation (SSH::SUCCESSFUL_LOGIN).
+
 Bro 2.1
 -------
 
@@ -209,6 +212,27 @@ New Functionality
   outputs. We do not yet recommend them for production (but welcome
   feedback!)
 
+- Summary statistics framework. [Extend]
+
+- A number of new applications build on top of the summary statistics
+  framework:
+
+    * Scan detection: Detectors for port and address scans return. See
+      policy/misc/scan.bro.
+
+    * Tracerouter detector: policy/misc/detect-traceroute
+
+    * Web application detection/measurement: policy/misc/app-metrics.bro
+
+    * FTP brute-forcing detector: policy/protocols/ftp/detect-bruteforcing.bro
+
+    * HTTP-based SQL injection detector: policy/protocols/http/detect-sqli.bro
+      (existed before, but now ported to the new framework)
+
+    * SSH brute-forcing detector feeding the intelligence framework:
+      policy/protocols/ssh/detect-bruteforcing.bro
+
+
 
 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~

VERSION

@@ -1 +1 @@
-2.1-397
+2.1-498

scripts/base/frameworks/sumstats/cluster.bro

@@ -10,49 +10,48 @@
 module SumStats;
 
 export {
-	## Allows a user to decide how large of result groups the 
+	## Allows a user to decide how large of result groups the workers should transmit
-	## workers should transmit values for cluster stats aggregation.
+	## values for cluster stats aggregation.
 	const cluster_send_in_groups_of = 50 &redef;
 
-	## The percent of the full threshold value that needs to be met 
+	## The percent of the full threshold value that needs to be met on a single worker
-	## on a single worker for that worker to send the value to its manager in
+	## for that worker to send the value to its manager in order for it to request a
-	## order for it to request a global view for that value.  There is no
+	## global view for that value.  There is no requirement that the manager requests
-	## requirement that the manager requests a global view for the key
+	## a global view for the key since it may opt not to if it requested a global view
-	## since it may opt not to if it requested a global view for the key
+	## for the key recently.
-	## recently.
 	const cluster_request_global_view_percent = 0.2 &redef;
 
 	## This is to deal with intermediate update overload.  A manager will only allow
-	## this many intermediate update requests to the workers to be inflight at 
+	## this many intermediate update requests to the workers to be inflight at any
-	## any given time.  Requested intermediate updates are currently thrown out
+	## given time.  Requested intermediate updates are currently thrown out and not
-	## and not performed.  In practice this should hopefully have a minimal effect.
+	## performed.  In practice this should hopefully have a minimal effect.
 	const max_outstanding_global_views = 10 &redef;
 
-	## Intermediate updates can cause overload situations on very large clusters.
+	## Intermediate updates can cause overload situations on very large clusters. This
-	## This option may help reduce load and correct intermittent problems.
+	## option may help reduce load and correct intermittent problems. The goal for this
-	## The goal for this option is also meant to be temporary.
+	## option is also meant to be temporary.
 	const enable_intermediate_updates = T &redef;
 
-	## Event sent by the manager in a cluster to initiate the 
+	## Event sent by the manager in a cluster to initiate the collection of values for
-	## collection of values for a sumstat.
+	## a sumstat.
 	global cluster_ss_request: event(uid: string, ssid: string);
 
-	## Event sent by nodes that are collecting sumstats after receiving
+	## Event sent by nodes that are collecting sumstats after receiving a request for
-	## a request for the sumstat from the manager.
+	## the sumstat from the manager.
 	global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
 
-	## This event is sent by the manager in a cluster to initiate the
+	## This event is sent by the manager in a cluster to initiate the collection of
-	## collection of a single key value from a sumstat.  It's typically
+	## a single key value from a sumstat.  It's typically used to get intermediate
-	## used to get intermediate updates before the break interval triggers
+	## updates before the break interval triggers to speed detection of a value
-	## to speed detection of a value crossing a threshold.
+	## crossing a threshold.
 	global cluster_key_request: event(uid: string, ssid: string, key: Key);
 
 	## This event is sent by nodes in response to a
 	## :bro:id:`SumStats::cluster_key_request` event.
 	global cluster_key_response: event(uid: string, ssid: string, key: Key, result: Result);
 
-	## This is sent by workers to indicate that they crossed the percent of the 
+	## This is sent by workers to indicate that they crossed the percent
-	## current threshold by the percentage defined globally in 
+	## of the current threshold by the percentage defined globally in
 	## :bro:id:`SumStats::cluster_request_global_view_percent`
 	global cluster_key_intermediate_response: event(ssid: string, key: SumStats::Key);
 
@@ -125,7 +124,9 @@ event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
 	if ( |data| == 0 )
 		done = T;
 
-	event SumStats::cluster_ss_response(uid, ssid, local_data, done);
+	# Note: copy is needed to compensate serialization caching issue. This should be
+	# changed to something else later. 
+	event SumStats::cluster_ss_response(uid, ssid, copy(local_data), done);
 	if ( ! done )
 		schedule 0.01 sec { SumStats::send_data(uid, ssid, data) };
 	}
@@ -151,7 +152,10 @@ event SumStats::cluster_key_request(uid: string, ssid: string, key: Key)
 	if ( ssid in result_store && key in result_store[ssid] )
 		{
 		#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
-		event SumStats::cluster_key_response(uid, ssid, key, result_store[ssid][key]);
+
+		# Note: copy is needed to compensate serialization caching issue. This should be
+		# changed to something else later. 
+		event SumStats::cluster_key_response(uid, ssid, key, copy(result_store[ssid][key]));
 		}
 	else
 		{

scripts/base/frameworks/sumstats/main.bro

@@ -81,6 +81,7 @@ export {
 	## SumStats represent an aggregation of reducers along with
 	## mechanisms to handle various situations like the epoch ending
 	## or thresholds being crossed.
+	##
 	## It's best to not access any global state outside
 	## of the variables given to the callbacks because there
 	## is no assurance provided as to where the callbacks
@@ -181,16 +182,17 @@ global result_store: table[string] of ResultTable = table();
 # Store of threshold information.
 global thresholds_store: table[string, Key] of bool = table();
 
-# This is called whenever
+# This is called whenever key values are updated and the new val is given as the
-# key values are updated and the new val is given as the `val` argument.
+# `val` argument. It's only prototyped here because cluster and non-cluster have
-# It's only prototyped here because cluster and non-cluster have separate 
+# separate  implementations.
-# implementations.
 global data_added: function(ss: SumStat, key: Key, result: Result);
 
 # Prototype the hook point for plugins to do calculations.
 global observe_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
+
 # Prototype the hook point for plugins to initialize any result values.
 global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
+
 # Prototype the hook point for plugins to merge Results.
 global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
 

scripts/base/frameworks/sumstats/plugins/average.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/max.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/min.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/sample.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 @load base/utils/queue
 
 module SumStats;
@@ -10,10 +10,8 @@ export {
 	};
 
 	redef record ResultVal += {
-		## This is the queue where samples
+		## This is the queue where samples are maintained.  Use the
-		## are maintained.  Use the 
+		## :bro:see:`SumStats::get_samples` function to get a vector of the samples.
-		## :bro:see:`SumStats::get_samples` function
-		## to get a vector of the samples.
 		samples: Queue::Queue &optional;
 	};
 

scripts/base/frameworks/sumstats/plugins/std-dev.bro

@@ -1,5 +1,5 @@
+@load base/frameworks/sumstats/main
 @load ./variance
-@load base/frameworks/sumstats
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/sum.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/unique.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/variance.bro

@@ -1,5 +1,5 @@
+@load base/frameworks/sumstats/main
 @load ./average
-@load base/frameworks/sumstats
 
 module SumStats;
 

scripts/policy/misc/detect-traceroute/main.bro

@@ -1,7 +1,7 @@
-##! This script detects large number of ICMP Time Exceeded messages heading
+##! This script detects a large number of ICMP Time Exceeded messages heading toward
-##! toward hosts that have sent low TTL packets.
+##! hosts that have sent low TTL packets. It generates a notice when the number of
-##! It generates a notice when the number of ICMP Time Exceeded 
+##! ICMP Time Exceeded messages for a source-destination pair exceeds a
-##! messages for a source-destination pair exceeds threshold
+##! threshold.
 @load base/frameworks/sumstats
 @load base/frameworks/signatures
 @load-sigs ./detect-low-ttls.sig
@@ -44,6 +44,8 @@ export {
 		src:   addr &log;
 		## Destination address of the traceroute.
 		dst:   addr &log;
+		## Protocol used for the traceroute.
+		proto: string &log;
 	};
 
 	global log_traceroute: event(rec: Traceroute::Info);
@@ -69,14 +71,15 @@ event bro_init() &priority=5
 	                  $threshold=icmp_time_exceeded_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	local parts = split1(key$str, /-/);
+	                  	local parts = split_n(key$str, /-/, F, 2);
 	                  	local src = to_addr(parts[1]);
 	                  	local dst = to_addr(parts[2]);
-	                  	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
+	                  	local proto = parts[3];
+	                  	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst, $proto=proto]);
 	                  	NOTICE([$note=Traceroute::Detected,
-	                  	        $msg=fmt("%s seems to be running traceroute", src),
+	                  	        $msg=fmt("%s seems to be running traceroute using %s", src, proto),
-	                  	        $src=src, $dst=dst,
+	                  	        $src=src,
-	                  	        $identifier=cat(src)]);
+	                  	        $identifier=cat(src,proto)]);
 	                  	}]);
 	}
 
@@ -84,10 +87,12 @@ event bro_init() &priority=5
 event signature_match(state: signature_state, msg: string, data: string)
 	{
 	if ( state$sig_id == /traceroute-detector.*/ )
-		SumStats::observe("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h)], [$num=1]);
+		{
+		SumStats::observe("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h,"-",get_port_transport_proto(state$conn$id$resp_p))], [$num=1]);
+		}
 	}
 
 event icmp_time_exceeded(c: connection, icmp: icmp_conn, code: count, context: icmp_context)
 	{
-	SumStats::observe("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
+	SumStats::observe("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h,"-",get_port_transport_proto(context$id$resp_p))], [$str=cat(c$id$orig_h)]);
 	}

scripts/policy/misc/scan.bro

@@ -13,11 +13,12 @@ module Scan;
 
 export {
 	redef enum Notice::Type += {
-		## Address scans detect that a host appears to be scanning some number
+		## Address scans detect that a host appears to be scanning some number of
-		## of hosts on a single port. This notice is generated when more than 
+		## destinations on a single port. This notice is generated when more than
-		## :bro:id:`addr_scan_threshold` unique hosts are seen over the 
+		## :bro:id:`addr_scan_threshold` unique hosts are seen over the previous
-		## previous :bro:id:`addr_scan_interval` time range.
+		## :bro:id:`addr_scan_interval` time range.
 		Address_Scan,
+
 		## Port scans detect that an attacking host appears to be scanning a
 		## single victim host on several ports.  This notice is generated when
 		## an attacking host attempts to connect to :bro:id:`port_scan_threshold`
@@ -27,17 +28,19 @@ export {
 	};
 
 	## Failed connection attempts are tracked over this time interval for the address
-	## scan detection.  A higher interval will detect slower scanners, but may 
+	## scan detection.  A higher interval will detect slower scanners, but may also
-	## also yield more false positives.
+	## yield more false positives.
 	const addr_scan_interval = 5min &redef;
-	## Failed connection attempts are tracked over this time interval for the port 
+
-	## scan detection.  A higher interval will detect slower scanners, but may 
+	## Failed connection attempts are tracked over this time interval for the port scan
-	## also yield more false positives.
+	## detection.  A higher interval will detect slower scanners, but may also yield
+	## more false positives.
 	const port_scan_interval = 5min &redef;
 
 	## The threshold of a unique number of hosts a scanning host has to have failed
 	## connections with on a single port.
 	const addr_scan_threshold = 25 &redef;
+
 	## The threshold of a number of unique ports a scanning host has to have failed
 	## connections with on a single victim host.
 	const port_scan_threshold = 15 &redef;
@@ -147,9 +150,8 @@ function is_reverse_failed_conn(c: connection): bool
 ## Generated for an unsuccessful connection attempt. This
 ## event is raised when an originator unsuccessfully attempted
 ## to establish a connection. “Unsuccessful” is defined as at least
-## tcp_attempt_delay seconds having elapsed since the originator 
+## tcp_attempt_delay seconds having elapsed since the originator first sent a
-## first sent a connection establishment packet to the destination 
+## connection establishment packet to the destination without seeing a reply.
-## without seeing a reply.
 event connection_attempt(c: connection)
 	{
 	local is_reverse_scan = F;
@@ -159,9 +161,8 @@ event connection_attempt(c: connection)
 	add_sumstats(c$id, is_reverse_scan);
 	}
 
-## Generated for a rejected TCP connection. This event 
+## Generated for a rejected TCP connection. This event is raised when an originator
-## is raised when an originator attempted to setup a TCP 
+## attempted to setup a TCP connection but the responder replied with a RST packet
-## connection but the responder replied with a RST packet 
 ## denying it.
 event connection_rejected(c: connection)
 	{
@@ -172,9 +173,8 @@ event connection_rejected(c: connection)
 	add_sumstats(c$id, is_reverse_scan);
 	}
 
-## Generated when an endpoint aborted a TCP connection. 
+## Generated when an endpoint aborted a TCP connection. The event is raised when
-## The event is raised when one endpoint of an *established* 
+## one endpoint of an *established* TCP connection aborted by sending a RST packet.
-## TCP connection aborted by sending a RST packet.
 event connection_reset(c: connection)
 	{
 	if ( is_failed_conn(c) )

scripts/policy/protocols/ftp/detect-bruteforcing.bro

@@ -1,3 +1,5 @@
+##! FTP brute-forcing detector, triggering when too  many rejected usernames or
+##! failed passwords have occured from a single address.
 
 @load base/protocols/ftp
 @load base/frameworks/sumstats

src/NetVar.cc

@@ -239,6 +239,11 @@ TableType* record_field_table;
 
 StringVal* cmd_line_bpf_filter;
 
+OpaqueType* md5_type;
+OpaqueType* sha1_type;
+OpaqueType* sha256_type;
+OpaqueType* entropy_type;
+
 #include "const.bif.netvar_def"
 #include "types.bif.netvar_def"
 #include "event.bif.netvar_def"
@@ -298,6 +303,11 @@ void init_general_global_var()
 
 	cmd_line_bpf_filter =
 		internal_val("cmd_line_bpf_filter")->AsStringVal();
+
+	md5_type = new OpaqueType("md5");
+	sha1_type = new OpaqueType("sha1");
+	sha256_type = new OpaqueType("sha256");
+	entropy_type = new OpaqueType("entropy");
 	}
 
 void init_net_var()

src/NetVar.h

@@ -243,6 +243,12 @@ extern TableType* record_field_table;
 
 extern StringVal* cmd_line_bpf_filter;
 
+class OpaqueType;
+extern OpaqueType* md5_type;
+extern OpaqueType* sha1_type;
+extern OpaqueType* sha256_type;
+extern OpaqueType* entropy_type;
+
 // Initializes globals that don't pertain to network/event analysis.
 extern void init_general_global_var();
 

src/OpaqueVal.cc

@@ -1,4 +1,5 @@
 #include "OpaqueVal.h"
+#include "NetVar.h"
 #include "Reporter.h"
 #include "Serializer.h"
 #include "HyperLogLog.h"
@@ -144,6 +145,10 @@ bool HashVal::DoUnserialize(UnserialInfo* info)
 	return UNSERIALIZE(&valid);
 	}
 
+MD5Val::MD5Val() : HashVal(md5_type)
+	{
+	}
+
 void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH])
 	{
 	MD5_CTX h;
@@ -261,6 +266,10 @@ bool MD5Val::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+SHA1Val::SHA1Val() : HashVal(sha1_type)
+	{
+	}
+
 void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH])
 	{
 	SHA_CTX h;
@@ -369,6 +378,10 @@ bool SHA1Val::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+SHA256Val::SHA256Val() : HashVal(sha256_type)
+	{
+	}
+
 void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH])
 	{
 	SHA256_CTX h;
@@ -482,6 +495,9 @@ bool SHA256Val::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+EntropyVal::EntropyVal() : OpaqueVal(entropy_type)
+	{
+	}
 
 bool EntropyVal::Feed(const void* data, size_t size)
 	{

src/OpaqueVal.h

@@ -54,7 +54,7 @@ public:
 			 u_char key[MD5_DIGEST_LENGTH],
 			 u_char result[MD5_DIGEST_LENGTH]);
 
-	MD5Val() : HashVal(new OpaqueType("md5"))	{ }
+	MD5Val();
 
 protected:
 	friend class Val;
@@ -73,7 +73,7 @@ class SHA1Val : public HashVal {
 public:
 	static void digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]);
 
-	SHA1Val() : HashVal(new OpaqueType("sha1")) { }
+	SHA1Val();
 
 protected:
 	friend class Val;
@@ -92,7 +92,7 @@ class SHA256Val : public HashVal {
 public:
 	static void digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]);
 
-	SHA256Val() : HashVal(new OpaqueType("sha256"))	{ }
+	SHA256Val();
 
 protected:
 	friend class Val;
@@ -109,7 +109,7 @@ private:
 
 class EntropyVal : public OpaqueVal {
 public:
-	EntropyVal() : OpaqueVal(new OpaqueType("entropy"))	{ }
+	EntropyVal();
 
 	bool Feed(const void* data, size_t size);
 	bool Get(double *r_ent, double *r_chisq, double *r_mean,

src/TCP.cc

@@ -566,7 +566,7 @@ void TCP_Analyzer::UpdateInactiveState(double t,
 			else
 				endpoint->SetState(TCP_ENDPOINT_SYN_SENT);
 
-			if ( connection_attempt )
+			if ( tcp_attempt_delay )
 				ADD_ANALYZER_TIMER(&TCP_Analyzer::AttemptTimer,
 					t + tcp_attempt_delay, 1,
 					TIMER_TCP_ATTEMPT);
@@ -1497,24 +1497,7 @@ void TCP_Analyzer::ExpireTimer(double t)
 
 		if ( resp->state == TCP_ENDPOINT_INACTIVE )
 			{
-			if ( (orig->state == TCP_ENDPOINT_SYN_SENT ||
+			if ( orig->state == TCP_ENDPOINT_INACTIVE )
-			      orig->state == TCP_ENDPOINT_SYN_ACK_SENT) )
-				{
-				if ( ! connection_attempt )
-					{
-					// Time out the connection attempt,
-					// since the AttemptTimer isn't going
-					// to do it for us, and we don't want
-					// to clog the data structures with
-					// old, failed attempts.
-					Event(connection_timeout);
-					is_active = 0;
-					sessions->Remove(Conn());
-					return;
-					}
-				}
-
-			else if ( orig->state == TCP_ENDPOINT_INACTIVE )
 				{
 				// Nothing ever happened on this connection.
 				// This can occur when we see a trashed

src/Val.cc

@@ -1749,7 +1749,7 @@ Val* TableVal::Default(Val* index)
 
 	if ( def_val->Type()->Tag() != TYPE_FUNC ||
 	     same_type(def_val->Type(), Type()->YieldType()) )
-		return def_val->Ref();
+		return def_attr->AttrExpr()->IsConst() ? def_val->Ref() : def_val->Clone();
 
 	const Func* f = def_val->AsFunc();
 	val_list* vl = new val_list();

testing/btest/Baseline/language.table-default-record/out

@@ -0,0 +1,7 @@
+0
+0
+0
+0
+{
+
+}

testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout

@@ -1 +1,3 @@
-A test metric threshold was crossed with a value of: 100.0
+A test metric threshold was crossed with a value of: 101.0
+End of epoch handler was called
+101.0

testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	socks
-#open	2012-06-20-17-23-38
+#open	2013-05-02-01-02-50
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	user	status	request.host	request.name	request_p	bound.host	bound.name	bound_p
 #types	time	string	addr	port	addr	port	count	string	string	addr	string	port	addr	string	port
-1340213015.276495	UWkUyAuUGXf	10.0.0.55	53994	60.190.189.214	8124	5	-	succeeded	-	www.osnews.com	80	192.168.0.31	-	2688
+1340213015.276495	arKYeMETxOg	10.0.0.55	53994	60.190.189.214	8124	5	-	succeeded	-	www.osnews.com	80	192.168.0.31	-	2688
-#close	2012-06-20-17-28-10
+#close	2013-05-02-01-02-50

testing/btest/Baseline/scripts.policy.frameworks.software.vulnerable/notice.log

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2013-04-25-18-55-26
+#open	2013-04-28-22-36-26
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double	addr	string	subnet
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double
-1366916126.685057	-	-	-	-	-	-	Software::Vulnerable_Version	1.2.3.4 is running Java 1.7.0.15 which is vulnerable.	Java 1.7.0.15	1.2.3.4	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-	-	-	-
+1367188586.649122	-	-	-	-	-	-	Software::Vulnerable_Version	1.2.3.4 is running Java 1.7.0.15 which is vulnerable.	Java 1.7.0.15	1.2.3.4	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-1366916126.685057	-	-	-	-	-	-	Software::Vulnerable_Version	1.2.3.5 is running Java 1.6.0.43 which is vulnerable.	Java 1.6.0.43	1.2.3.5	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-	-	-	-
+1367188586.649122	-	-	-	-	-	-	Software::Vulnerable_Version	1.2.3.5 is running Java 1.6.0.43 which is vulnerable.	Java 1.6.0.43	1.2.3.5	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-#close	2013-04-25-18-55-26
+#close	2013-04-28-22-36-26

testing/btest/core/leaks/basic-cluster.bro

@@ -6,33 +6,38 @@
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
 # @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT
-# @TEST-EXEC: btest-bg-run proxy-1   HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1   bro -m %INPUT
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m %INPUT
-# @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m %INPUT
-# @TEST-EXEC: btest-bg-wait 60
+# @TEST-EXEC: btest-bg-wait 15
-# @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
 	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"],
-	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
+	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"],
-	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
 };
 @TEST-END-FILE
 
 redef Log::default_rotation_interval = 0secs;
 
-redef enum Metrics::ID += {
+global n = 0;
-	TEST_METRIC,
-};
 
 event bro_init() &priority=5
 	{
-	Metrics::add_filter(TEST_METRIC, 
+	local r1: SumStats::Reducer = [$stream="test", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
-		[$name="foo-bar",
+	SumStats::create([$epoch=5secs,
-		 $break_interval=3secs]);
+	                  $reducers=set(r1),
+	                  $epoch_finished(rt: SumStats::ResultTable) =
+	                  	{
+	                  	for ( key in rt )
+	                  		{
+	                  		local r = rt[key]["test"];
+	                  		print fmt("Host: %s - num:%d - sum:%.1f - avg:%.1f - max:%.1f - min:%.1f - var:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$average, r$max, r$min, r$variance, r$std_dev, r$unique);
+	                  		}
+
+	                  	terminate();
+	                  	}]);
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -41,43 +46,40 @@ event remote_connection_closed(p: event_peer)
 	}
 
 global ready_for_data: event();
-
+redef Cluster::manager2worker_events += /^ready_for_data$/;
-redef Cluster::manager2worker_events += /ready_for_data/;
-
-@if ( Cluster::local_node_type() == Cluster::WORKER )
 
 event ready_for_data()
 	{
-	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+	if ( Cluster::node == "worker-1" )
-	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+		{
-	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=34]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=30]);
+		SumStats::observe("test", [$host=6.5.4.3], [$num=1]);
+		SumStats::observe("test", [$host=7.2.1.5], [$num=54]);
+		}
+	if ( Cluster::node == "worker-2" )
+		{
+		SumStats::observe("test", [$host=1.2.3.4], [$num=75]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=30]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=3]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=57]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=52]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=61]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=95]);
+		SumStats::observe("test", [$host=6.5.4.3], [$num=5]);
+		SumStats::observe("test", [$host=7.2.1.5], [$num=91]);
+		SumStats::observe("test", [$host=10.10.10.10], [$num=5]);
+		}
 	}
-
-@endif
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
-global n = 0;
 global peer_count = 0;
-
+event remote_connection_handshake_done(p: event_peer) &priority=-5
-event Metrics::log_metrics(rec: Metrics::Info)
-	{
-	n = n + 1;
-	if ( n == 3 )
-		{
-		terminate_communication();
-		terminate();
-		}
-	}
-
-event remote_connection_handshake_done(p: event_peer)
-	{
-	print p;
-	peer_count = peer_count + 1;
-	if ( peer_count == 3 )
 	{
+	++peer_count;
+	if ( peer_count == 2 )
 		event ready_for_data();
 	}
-	}
 
 @endif

testing/btest/coverage/bare-mode-errors.test

@@ -3,12 +3,13 @@
 # scripts that block after loading, e.g. start listening on a socket.
 #
 # Commonly, this test may fail if one forgets to @load some base/ scripts
-# when writing a new bro scripts.
+# when writing a new bro scripts. Look into "allerrors" to find out
+# which script had trouble.
 #
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: test -d $DIST/scripts
-# @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
+# @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo "=== $script" >>allerrors; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
-# @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors
+# @TEST-EXEC: cat allerrors | grep -v "received termination signal" | grep -v '===' | sort | uniq > unique_errors
 # @TEST-EXEC: if [ $(grep -c LibCURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi
 # @TEST-EXEC: if [ $(grep -c LibCURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi

testing/btest/language/table-default-record.bro

@@ -0,0 +1,24 @@
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+type Foo: record {
+	x: count &default=0;
+};
+
+global foo: table[count] of Foo = {} &default=[];
+
+# returns the &default value as usual
+print(foo[0]$x);
+print(foo[1]$x);
+
+# these are essentially no-ops since a copy of the &default value is returned
+# by the lookup
+foo[0]$x = 0;
+foo[1]$x = 1;
+
+# the &default value isn't modified
+print(foo[0]$x);
+print(foo[1]$x);
+
+# table membership isn't modified
+print(foo);

testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro

@@ -4,7 +4,7 @@
 # @TEST-EXEC: sleep 3
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT 
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/.stdout
 
 @TEST-START-FILE cluster-layout.bro
@@ -20,8 +20,15 @@ redef Log::default_rotation_interval = 0secs;
 event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=1hr,
+	SumStats::create([$epoch=10secs,
 	                  $reducers=set(r1),
+	                  $epoch_finished(data: SumStats::ResultTable) = 
+				  {
+				  print "End of epoch handler was called";
+				  for ( res in data ) 
+				  	print data[res]["test.metric"]$sum;
+	                  	  terminate();
+				  },
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	return double_to_count(result["test.metric"]$sum);
@@ -30,7 +37,6 @@ event bro_init() &priority=5
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	print fmt("A test metric threshold was crossed with a value of: %.1f", result["test.metric"]$sum);
-	                  	terminate();
 	                  	}]);
 	}
 
@@ -52,8 +58,13 @@ event remote_connection_handshake_done(p: event_peer)
 	if ( p$descr == "manager-1" )
 		{
 		if ( Cluster::node == "worker-1" )
+			{
 			schedule 0.1sec { do_stats(1) };
+			schedule 5secs { do_stats(60) };
+			}
 		if ( Cluster::node == "worker-2" )
-			schedule 0.5sec { do_stats(99) };
+			schedule 0.5sec { do_stats(40) };
 		}
 	}
+
+