Merge remote-tracking branch 'origin/master' into topic/bernhard/topk

CHANGES

@@ -1,4 +1,38 @@
 
+2.1-498 | 2013-05-03 17:44:08 -0700
+
+  * Table lookups return copy of non-const &default vals. This
+    prevents unintentional modifications to the &default value itself.
+    Addresses #981.  (Jon Siwek)
+
+2.1-496 | 2013-05-03 15:54:47 -0700
+
+  * Fix memory leak and unnecessary allocations in OpaqueVal.
+    Addresses #986. (Matthias Vallentin)
+
+2.1-492 | 2013-05-02 12:46:26 -0700
+
+  * Work-around for sumstats framework not propagating updates after
+    intermediate check in cluster environments. (Bernhard Amann)
+
+  * Always apply tcp_connection_attempt. Before this change it was
+    only applied when a connection_attempt() event handler was
+    defined. (Robin Sommer)
+
+  * Fixing coverage.bare-mode-errors test. (Robin Sommer)
+
+2.1-487 | 2013-05-01 18:03:22 -0700
+
+  * Always apply tcp_connection_attempt timer, even if no
+    connection_attempt() event handler is defined. (Robin Sommer)
+
+2.1-486 | 2013-05-01 15:28:45 -0700
+
+  * New framework for computing summary statistics in
+    base/framework/sumstats. This replaces the metrics frameworks, and
+    comes with a number of applications build on top, see NEWS. More
+    documentation to follow. (Seth Hall)
+
 2.1-397 | 2013-04-29 21:19:00 -0700
 
   * Fixing memory leaks in CompHash implementation. Addresses #987.

VERSION

@@ -1 +1 @@
-2.1-397
+2.1-498

scripts/base/frameworks/sumstats/cluster.bro

@@ -124,7 +124,9 @@ event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
 	if ( |data| == 0 )
 		done = T;
 
-	event SumStats::cluster_ss_response(uid, ssid, local_data, done);
+	# Note: copy is needed to compensate serialization caching issue. This should be
+	# changed to something else later. 
+	event SumStats::cluster_ss_response(uid, ssid, copy(local_data), done);
 	if ( ! done )
 		schedule 0.01 sec { SumStats::send_data(uid, ssid, data) };
 	}
@@ -150,7 +152,10 @@ event SumStats::cluster_key_request(uid: string, ssid: string, key: Key)
 	if ( ssid in result_store && key in result_store[ssid] )
 		{
 		#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
-		event SumStats::cluster_key_response(uid, ssid, key, result_store[ssid][key]);
+
+		# Note: copy is needed to compensate serialization caching issue. This should be
+		# changed to something else later. 
+		event SumStats::cluster_key_response(uid, ssid, key, copy(result_store[ssid][key]));
 		}
 	else
 		{

scripts/base/frameworks/sumstats/plugins/average.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/max.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/min.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/sample.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 @load base/utils/queue
 
 module SumStats;

scripts/base/frameworks/sumstats/plugins/std-dev.bro

@@ -1,5 +1,5 @@
+@load base/frameworks/sumstats/main
 @load ./variance
-@load base/frameworks/sumstats
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/sum.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/unique.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load base/frameworks/sumstats/main
 
 module SumStats;
 

scripts/base/frameworks/sumstats/plugins/variance.bro

@@ -1,5 +1,5 @@
+@load base/frameworks/sumstats/main
 @load ./average
-@load base/frameworks/sumstats
 
 module SumStats;
 

src/NetVar.cc

@@ -239,6 +239,11 @@ TableType* record_field_table;
 
 StringVal* cmd_line_bpf_filter;
 
+OpaqueType* md5_type;
+OpaqueType* sha1_type;
+OpaqueType* sha256_type;
+OpaqueType* entropy_type;
+
 #include "const.bif.netvar_def"
 #include "types.bif.netvar_def"
 #include "event.bif.netvar_def"
@@ -298,6 +303,11 @@ void init_general_global_var()
 
 	cmd_line_bpf_filter =
 		internal_val("cmd_line_bpf_filter")->AsStringVal();
+
+	md5_type = new OpaqueType("md5");
+	sha1_type = new OpaqueType("sha1");
+	sha256_type = new OpaqueType("sha256");
+	entropy_type = new OpaqueType("entropy");
 	}
 
 void init_net_var()
@@ -346,7 +356,7 @@ void init_net_var()
 		opt_internal_int("tcp_excessive_data_without_further_acks");
 
 	x509_type = internal_type("X509")->AsRecordType();
-	
+
 	socks_address = internal_type("SOCKS::Address")->AsRecordType();
 
 	non_analyzed_lifetime = opt_internal_double("non_analyzed_lifetime");

src/NetVar.h

@@ -243,6 +243,12 @@ extern TableType* record_field_table;
 
 extern StringVal* cmd_line_bpf_filter;
 
+class OpaqueType;
+extern OpaqueType* md5_type;
+extern OpaqueType* sha1_type;
+extern OpaqueType* sha256_type;
+extern OpaqueType* entropy_type;
+
 // Initializes globals that don't pertain to network/event analysis.
 extern void init_general_global_var();
 

src/OpaqueVal.cc

@@ -1,4 +1,5 @@
 #include "OpaqueVal.h"
+#include "NetVar.h"
 #include "Reporter.h"
 #include "Serializer.h"
 
@@ -72,6 +73,10 @@ bool HashVal::DoUnserialize(UnserialInfo* info)
 	return UNSERIALIZE(&valid);
 	}
 
+MD5Val::MD5Val() : HashVal(md5_type)
+	{
+	}
+
 void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH])
 	{
 	MD5_CTX h;
@@ -189,6 +194,10 @@ bool MD5Val::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+SHA1Val::SHA1Val() : HashVal(sha1_type)
+	{
+	}
+
 void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH])
 	{
 	SHA_CTX h;
@@ -297,6 +306,10 @@ bool SHA1Val::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+SHA256Val::SHA256Val() : HashVal(sha256_type)
+	{
+	}
+
 void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH])
 	{
 	SHA256_CTX h;
@@ -410,6 +423,9 @@ bool SHA256Val::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+EntropyVal::EntropyVal() : OpaqueVal(entropy_type)
+	{
+	}
 
 bool EntropyVal::Feed(const void* data, size_t size)
 	{

src/OpaqueVal.h

@@ -36,7 +36,7 @@ public:
 			 u_char key[MD5_DIGEST_LENGTH],
 			 u_char result[MD5_DIGEST_LENGTH]);
 
-	MD5Val() : HashVal(new OpaqueType("md5"))	{ }
+	MD5Val();
 
 protected:
 	friend class Val;
@@ -55,7 +55,7 @@ class SHA1Val : public HashVal {
 public:
 	static void digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]);
 
-	SHA1Val() : HashVal(new OpaqueType("sha1")) { }
+	SHA1Val();
 
 protected:
 	friend class Val;
@@ -74,7 +74,7 @@ class SHA256Val : public HashVal {
 public:
 	static void digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]);
 
-	SHA256Val() : HashVal(new OpaqueType("sha256"))	{ }
+	SHA256Val();
 
 protected:
 	friend class Val;
@@ -91,7 +91,7 @@ private:
 
 class EntropyVal : public OpaqueVal {
 public:
-	EntropyVal() : OpaqueVal(new OpaqueType("entropy"))	{ }
+	EntropyVal();
 
 	bool Feed(const void* data, size_t size);
 	bool Get(double *r_ent, double *r_chisq, double *r_mean,

src/TCP.cc

@@ -566,7 +566,7 @@ void TCP_Analyzer::UpdateInactiveState(double t,
 			else
 				endpoint->SetState(TCP_ENDPOINT_SYN_SENT);
 
-			if ( connection_attempt )
+			if ( tcp_attempt_delay )
 				ADD_ANALYZER_TIMER(&TCP_Analyzer::AttemptTimer,
 					t + tcp_attempt_delay, 1,
 					TIMER_TCP_ATTEMPT);
@@ -1497,24 +1497,7 @@ void TCP_Analyzer::ExpireTimer(double t)
 
 		if ( resp->state == TCP_ENDPOINT_INACTIVE )
 			{
-			if ( (orig->state == TCP_ENDPOINT_SYN_SENT ||
+			if ( orig->state == TCP_ENDPOINT_INACTIVE )
-			      orig->state == TCP_ENDPOINT_SYN_ACK_SENT) )
-				{
-				if ( ! connection_attempt )
-					{
-					// Time out the connection attempt,
-					// since the AttemptTimer isn't going
-					// to do it for us, and we don't want
-					// to clog the data structures with
-					// old, failed attempts.
-					Event(connection_timeout);
-					is_active = 0;
-					sessions->Remove(Conn());
-					return;
-					}
-				}
-
-			else if ( orig->state == TCP_ENDPOINT_INACTIVE )
 				{
 				// Nothing ever happened on this connection.
 				// This can occur when we see a trashed

src/Val.cc

@@ -1749,7 +1749,7 @@ Val* TableVal::Default(Val* index)
 
 	if ( def_val->Type()->Tag() != TYPE_FUNC ||
 	     same_type(def_val->Type(), Type()->YieldType()) )
-		return def_val->Ref();
+		return def_attr->AttrExpr()->IsConst() ? def_val->Ref() : def_val->Clone();
 
 	const Func* f = def_val->AsFunc();
 	val_list* vl = new val_list();

testing/btest/Baseline/language.table-default-record/out

@@ -0,0 +1,7 @@
+0
+0
+0
+0
+{
+
+}

testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout

@@ -1 +1,3 @@
-A test metric threshold was crossed with a value of: 100.0
+A test metric threshold was crossed with a value of: 101.0
+End of epoch handler was called
+101.0

testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	socks
-#open	2012-06-20-17-23-38
+#open	2013-05-02-01-02-50
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	user	status	request.host	request.name	request_p	bound.host	bound.name	bound_p
 #types	time	string	addr	port	addr	port	count	string	string	addr	string	port	addr	string	port
-1340213015.276495	UWkUyAuUGXf	10.0.0.55	53994	60.190.189.214	8124	5	-	succeeded	-	www.osnews.com	80	192.168.0.31	-	2688
+1340213015.276495	arKYeMETxOg	10.0.0.55	53994	60.190.189.214	8124	5	-	succeeded	-	www.osnews.com	80	192.168.0.31	-	2688
-#close	2012-06-20-17-28-10
+#close	2013-05-02-01-02-50

testing/btest/coverage/bare-mode-errors.test

@@ -3,12 +3,13 @@
 # scripts that block after loading, e.g. start listening on a socket.
 #
 # Commonly, this test may fail if one forgets to @load some base/ scripts
-# when writing a new bro scripts.
+# when writing a new bro scripts. Look into "allerrors" to find out
+# which script had trouble.
 #
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: test -d $DIST/scripts
-# @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
+# @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo "=== $script" >>allerrors; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
-# @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors
+# @TEST-EXEC: cat allerrors | grep -v "received termination signal" | grep -v '===' | sort | uniq > unique_errors
 # @TEST-EXEC: if [ $(grep -c LibCURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi
 # @TEST-EXEC: if [ $(grep -c LibCURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi

testing/btest/language/table-default-record.bro

@@ -0,0 +1,24 @@
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+type Foo: record {
+	x: count &default=0;
+};
+
+global foo: table[count] of Foo = {} &default=[];
+
+# returns the &default value as usual
+print(foo[0]$x);
+print(foo[1]$x);
+
+# these are essentially no-ops since a copy of the &default value is returned
+# by the lookup
+foo[0]$x = 0;
+foo[1]$x = 1;
+
+# the &default value isn't modified
+print(foo[0]$x);
+print(foo[1]$x);
+
+# table membership isn't modified
+print(foo);

testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro

@@ -4,7 +4,7 @@
 # @TEST-EXEC: sleep 3
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT 
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/.stdout
 
 @TEST-START-FILE cluster-layout.bro
@@ -20,8 +20,15 @@ redef Log::default_rotation_interval = 0secs;
 event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=1hr,
+	SumStats::create([$epoch=10secs,
 	                  $reducers=set(r1),
+	                  $epoch_finished(data: SumStats::ResultTable) = 
+				  {
+				  print "End of epoch handler was called";
+				  for ( res in data ) 
+				  	print data[res]["test.metric"]$sum;
+	                  	  terminate();
+				  },
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	return double_to_count(result["test.metric"]$sum);
@@ -30,7 +37,6 @@ event bro_init() &priority=5
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	print fmt("A test metric threshold was crossed with a value of: %.1f", result["test.metric"]$sum);
-	                  	terminate();
 	                  	}]);
 	}
 
@@ -52,8 +58,13 @@ event remote_connection_handshake_done(p: event_peer)
 	if ( p$descr == "manager-1" )
 		{
 		if ( Cluster::node == "worker-1" )
+			{
 			schedule 0.1sec { do_stats(1) };
+			schedule 5secs { do_stats(60) };
+			}
 		if ( Cluster::node == "worker-2" )
-			schedule 0.5sec { do_stats(99) };
+			schedule 0.5sec { do_stats(40) };
 		}
 	}
+
+