SumStats test checkpoint.

DocSourcesList.cmake

@@ -46,17 +46,6 @@ rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
 rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
 rest_target(${psd} base/frameworks/logging/writers/none.bro)
-rest_target(${psd} base/frameworks/measurement/cluster.bro)
-rest_target(${psd} base/frameworks/measurement/main.bro)
-rest_target(${psd} base/frameworks/measurement/non-cluster.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/average.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/max.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/min.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/sample.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/std-dev.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/sum.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/unique.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/variance.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@@ -72,6 +61,17 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/sumstats/cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/main.bro)
+rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
 rest_target(${psd} base/frameworks/tunnels/main.bro)
 rest_target(${psd} base/misc/find-checksum-offloading.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
@@ -147,7 +147,6 @@ rest_target(${psd} policy/misc/stats.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/metrics.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
@@ -166,7 +165,6 @@ rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
 rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
 rest_target(${psd} policy/protocols/smtp/blocklists.bro)
 rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
-rest_target(${psd} policy/protocols/smtp/metrics.bro)
 rest_target(${psd} policy/protocols/smtp/software.bro)
 rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ssh/geo-data.bro)

scripts/base/frameworks/sumstats/cluster.bro

@@ -34,15 +34,15 @@ export {
 	const enable_intermediate_updates = T &redef;
 
 	## Event sent by the manager in a cluster to initiate the 
-	## collection of metrics values for a measurement.
+	## collection of metrics values for a sumstat.
 	global cluster_ss_request: event(uid: string, ssid: string);
 
 	## Event sent by nodes that are collecting metrics after receiving
-	## a request for the metric measurement from the manager.
+	## a request for the metric sumstat from the manager.
 	global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
 
 	## This event is sent by the manager in a cluster to initiate the
-	## collection of a single key value from a measurement.  It's typically
+	## collection of a single key value from a sumstat.  It's typically
 	## used to get intermediate updates before the break interval triggers
 	## to speed detection of a value crossing a threshold.
 	global cluster_key_request: event(uid: string, ssid: string, key: Key);
@@ -130,13 +130,13 @@ event SumStats::cluster_ss_request(uid: string, ssid: string)
 	{
 	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
 	
-	# Initiate sending all of the data for the requested measurement.
+	# Initiate sending all of the data for the requested stats.
 	if ( ssid in result_store )
 		event SumStats::send_data(uid, ssid, result_store[ssid]);
 	else
 		event SumStats::send_data(uid, ssid, table());
 
-	# Lookup the actual measurement and reset it, the reference to the data
+	# Lookup the actual sumstats and reset it, the reference to the data
 	# currently stored will be maintained internally by the send_data event.
 	if ( ssid in stats_store )
 		reset(stats_store[ssid]);
@@ -181,9 +181,9 @@ global done_with: table[string] of count &read_expire=1min &default=0;
 global key_requests: table[string] of Result &read_expire=1min;
 
 # This variable is maintained by managers to prevent overwhelming communication due
-# to too many intermediate updates.  Each measurement is tracked separately so that 
+# to too many intermediate updates.  Each sumstat is tracked separately so that 
-# one won't overwhelm and degrade other quieter measurements. 
+# one won't overwhelm and degrade other quieter sumstats. 
-# Indexed on a measurement id.
+# Indexed on a sumstat id.
 global outstanding_global_views: table[string] of count &default=0;
 
 const zero_time = double_to_time(0.0);
@@ -192,7 +192,7 @@ event SumStats::finish_epoch(ss: SumStat)
 	{
 	if ( network_time() > zero_time )
 		{
-		#print fmt("%.6f MANAGER: breaking %s measurement for %s metric", network_time(), measurement$name, measurement$id);
+		#print fmt("%.6f MANAGER: breaking %s sumstat for %s metric", network_time(), ss$name, ss$id);
 		local uid = unique_id("");
 		
 		if ( uid in stats_results )
@@ -207,8 +207,8 @@ event SumStats::finish_epoch(ss: SumStat)
 	schedule ss$epoch { SumStats::finish_epoch(ss) };
 	}
 
-# This is unlikely to be called often, but it's here in case there are measurements
+# This is unlikely to be called often, but it's here in 
-# being collected by managers.
+# case there are sumstats being collected by managers.
 function data_added(ss: SumStat, key: Key, result: Result)
 	{
 	if ( check_thresholds(ss, key, result, 1.0) )
@@ -305,7 +305,7 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable
 		# Clean up
 		delete stats_results[uid];
 		delete done_with[uid];
-		# Not sure I need to reset the measurement on the manager.
+		# Not sure I need to reset the sumstat on the manager.
 		reset(ss);
 		}
 	}

scripts/base/frameworks/sumstats/plugins/average.bro

@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( AVERAGE in r$apply )
 		{

scripts/base/frameworks/sumstats/plugins/max.bro

@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( MAX in r$apply )
 		{

scripts/base/frameworks/sumstats/plugins/min.bro

@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( MIN in r$apply )
 		{

scripts/base/frameworks/sumstats/plugins/sample.bro

@@ -29,13 +29,13 @@ function get_samples(rv: ResultVal): vector of Observation
 	return s;
 	}
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( r$samples > 0 )
 		{
 		if ( ! rv?$samples )
 			rv$samples = Queue::init([$max_len=r$samples]);
-		Queue::put(rv$samples, data);
+		Queue::put(rv$samples, obs);
 		}
 	}
 

scripts/base/frameworks/sumstats/plugins/std-dev.bro

@@ -22,13 +22,10 @@ function calc_std_dev(rv: ResultVal)
 	}
 
 # This depends on the variance plugin which uses priority -5
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) &priority=-10
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10
 	{
 	if ( STD_DEV in r$apply )
-		{
+		calc_std_dev(rv);
-		if ( rv?$variance )
-			calc_std_dev(rv);
-		}
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10

scripts/base/frameworks/sumstats/plugins/sum.bro

@@ -34,7 +34,7 @@ hook init_resultval_hook(r: Reducer, rv: ResultVal)
 		rv$sum = 0;
 	}
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( SUM in r$apply )
 		rv$sum += val;

scripts/base/frameworks/sumstats/plugins/unique.bro

@@ -23,13 +23,13 @@ redef record ResultVal += {
 	unique_vals: set[Observation] &optional;
 };
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( UNIQUE in r$apply )
 		{
 		if ( ! rv?$unique_vals ) 
 			rv$unique_vals=set();
-		add rv$unique_vals[data];
+		add rv$unique_vals[obs];
 		rv$unique = |rv$unique_vals|;
 		}
 	}

scripts/base/frameworks/sumstats/plugins/variance.bro

@@ -29,7 +29,7 @@ function calc_variance(rv: ResultVal)
 	}
 
 # Reduced priority since this depends on the average
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) &priority=-5
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5
 	{
 	if ( VARIANCE in r$apply )
 		{

scripts/policy/misc/app-metrics.bro

@@ -32,7 +32,7 @@ redef record connection += {
 
 event bro_init() &priority=3
 	{
-	Log::create_stream(AppSumStats::LOG, [$columns=Info]);
+	Log::create_stream(AppStats::LOG, [$columns=Info]);
 
 	local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
 	local r2: SumStats::Reducer = [$stream="apps.hits",  $apply=set(SumStats::UNIQUE)];

scripts/policy/protocols/conn/metrics.bro

@@ -1,24 +0,0 @@
-@load base/frameworks/measurement
-@load base/utils/site
-
-event bro_init() &priority=3
-	{
-	Metrics::add_filter("conns.country", [$every=1hr, $measure=set(Metrics::SUM),
-	                                      $period_finished=Metrics::write_log]);
-	Metrics::add_filter("hosts.active", [$every=1hr, $measure=set(Metrics::SUM),
-	                                     $period_finished=Metrics::write_log]);
-	}
-
-event connection_established(c: connection) &priority=3
-	{
-	if ( Site::is_local_addr(c$id$orig_h) )
-		{
-		local loc = lookup_location(c$id$resp_h);
-		if ( loc?$country_code )
-			Metrics::add_data("conns.country", [$str=loc$country_code], [$num=1]);
-		}
-		
-	local the_host = Site::is_local_addr(c$id$orig_h) ? c$id$orig_h : c$id$resp_h;
-	# There is no index for this.
-	Metrics::add_data("hosts.active", [], [$str=cat(the_host)]);
-	}

scripts/policy/protocols/smtp/metrics.bro

@@ -1,37 +0,0 @@
-##! This script is meant to answer the following questions...
-##!   "How many unique 'MAIL FROM' addresses are being used by local mail servers per hour?"
-##!   "How much mail is being sent from each local mail server per hour?"
-
-@load base/protocols/smtp
-@load base/frameworks/measurement
-@load base/utils/site
-@load base/utils/directions-and-hosts
-
-module SMTPMetrics;
-
-export {
-	## Define the break intervals for all of the metrics collected and logged by this script.
-	const breaks=1hr &redef;
-}
-
-event bro_init() &priority=5
-	{
-	Metrics::add_filter("smtp.mailfrom", [$every=breaks,
-	                                      $measure=set(Metrics::SUM),
-	                                      $pred(index: Metrics::Index, data: Metrics::Observation) = { 
-	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); 
-	                                      }]);
-	Metrics::add_filter("smtp.messages", [$every=breaks,
-	                                      $measure=set(Metrics::SUM),
-	                                      $pred(index: Metrics::Index, data: Metrics::Observation) = { 
-	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); 
-	                                      }]);
-	}
-
-event SMTP::log_smtp(rec: SMTP::Info)
-	{
-	Metrics::add_data("smtp.messages", [$host=rec$id$orig_h], [$num=1]);
-	
-	if ( rec?$mailfrom )
-		Metrics::add_data("smtp.mailfrom", [$host=rec$id$orig_h], [$str=rec$mailfrom]);
-	}

scripts/test-all-policy.bro

@@ -32,9 +32,13 @@
 @load integration/collective-intel/__load__.bro
 @load integration/collective-intel/main.bro
 @load misc/analysis-groups.bro
+@load misc/app-metrics.bro
 @load misc/capture-loss.bro
+@load misc/detect-traceroute/__load__.bro
+@load misc/detect-traceroute/main.bro
 @load misc/loaded-scripts.bro
 @load misc/profiling.bro
+@load misc/scan.bro
 @load misc/stats.bro
 @load misc/trim-trace-file.bro
 @load protocols/conn/known-hosts.bro
@@ -42,6 +46,7 @@
 @load protocols/conn/weirds.bro
 @load protocols/dns/auth-addl.bro
 @load protocols/dns/detect-external-names.bro
+@load protocols/ftp/detect-bruteforcing.bro
 @load protocols/ftp/detect.bro
 @load protocols/ftp/software.bro
 @load protocols/http/detect-MHR.bro

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-04-02-04-22-32
+#open	2013-04-16-03-43-22
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -68,23 +68,23 @@ scripts/base/init-default.bro
     scripts/base/frameworks/software/main.bro
   scripts/base/frameworks/communication/__load__.bro
     scripts/base/frameworks/communication/main.bro
-  scripts/base/frameworks/measurement/__load__.bro
-    scripts/base/frameworks/measurement/main.bro
-    scripts/base/frameworks/measurement/plugins/__load__.bro
-      scripts/base/frameworks/measurement/plugins/average.bro
-      scripts/base/frameworks/measurement/plugins/max.bro
-      scripts/base/frameworks/measurement/plugins/min.bro
-      scripts/base/frameworks/measurement/plugins/sample.bro
-      scripts/base/frameworks/measurement/plugins/std-dev.bro
-        scripts/base/frameworks/measurement/plugins/variance.bro
-      scripts/base/frameworks/measurement/plugins/sum.bro
-      scripts/base/frameworks/measurement/plugins/unique.bro
-    scripts/base/frameworks/measurement/non-cluster.bro
   scripts/base/frameworks/intel/__load__.bro
     scripts/base/frameworks/intel/main.bro
     scripts/base/frameworks/intel/input.bro
   scripts/base/frameworks/reporter/__load__.bro
     scripts/base/frameworks/reporter/main.bro
+  scripts/base/frameworks/sumstats/__load__.bro
+    scripts/base/frameworks/sumstats/main.bro
+    scripts/base/frameworks/sumstats/plugins/__load__.bro
+      scripts/base/frameworks/sumstats/plugins/average.bro
+      scripts/base/frameworks/sumstats/plugins/max.bro
+      scripts/base/frameworks/sumstats/plugins/min.bro
+      scripts/base/frameworks/sumstats/plugins/sample.bro
+      scripts/base/frameworks/sumstats/plugins/variance.bro
+      scripts/base/frameworks/sumstats/plugins/std-dev.bro
+      scripts/base/frameworks/sumstats/plugins/sum.bro
+      scripts/base/frameworks/sumstats/plugins/unique.bro
+    scripts/base/frameworks/sumstats/non-cluster.bro
   scripts/base/frameworks/tunnels/__load__.bro
     scripts/base/frameworks/tunnels/main.bro
   scripts/base/protocols/conn/__load__.bro
@@ -130,4 +130,4 @@ scripts/base/init-default.bro
     scripts/base/protocols/syslog/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-04-02-04-22-32
+#close	2013-04-16-03-43-22

testing/btest/Baseline/coverage.init-default/missing_loads

@@ -3,5 +3,5 @@
 -./frameworks/cluster/nodes/worker.bro
 -./frameworks/cluster/setup-connections.bro
 -./frameworks/intel/cluster.bro
--./frameworks/measurement/cluster.bro
 -./frameworks/notice/cluster.bro
+-./frameworks/sumstats/cluster.bro