Large update for the SumStats framework.

- On-demand access to sumstats results through "return from"
   functions named SumStats::request and Sumstats::request_key.
   Both functions are tested in standalone and clustered modes.

 - $name field has returned to SumStats which simplifies cluster
   code and makes the on-demand access stuff possible.

 - Clustered results can only be collected for 1 minute from their
   time of creation now instead of time of last read.

 - Thresholds use doubles instead of counts everywhere now.

 - Calculation dependency resolution occurs at start up time now
   instead of doing it at observation time which provide a minor
   cpu performance improvement.  A new plugin registration mechanism
   was created to support this change.

 - AppStats now has a minimal doc string and is broken into hook-based
   plugins.

 - AppStats and traceroute detection added to local.bro

scripts/policy/misc/app-stats/__load__.bro

@@ -0,0 +1,2 @@
+@load ./main
+@load ./plugins

scripts/policy/misc/app-stats/main.bro

@@ -1,3 +1,6 @@
+#! AppStats collects information about web applications in use
+#! on the network.  
+
 @load base/protocols/http
 @load base/protocols/ssl
 @load base/frameworks/sumstats
@@ -30,13 +33,17 @@ redef record connection += {
 	resp_hostname: string &optional;
 };
 
+global add_sumstats: hook(id: conn_id, hostname: string, size: count);
+
+
 event bro_init() &priority=3
 	{
 	Log::create_stream(AppStats::LOG, [$columns=Info]);
 
 	local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
 	local r2: SumStats::Reducer = [$stream="apps.hits",  $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=break_interval,
+	SumStats::create([$name="app-metrics",
+	                  $epoch=break_interval,
 	                  $reducers=set(r1, r2),
 	                  $epoch_finished(data: SumStats::ResultTable) =
 	                  	{
@@ -55,41 +62,6 @@ event bro_init() &priority=3
 	                  	}]);
 	}
 
-function add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="youtube"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /(\.facebook\.com|\.fbcdn\.net)$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="facebook"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.google\.com$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="google"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.nflximg\.com$/ in hostname && size > 200*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="netflix"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="pandora"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.gmail\.com$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="gmail"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
-		}
-}
-
-
 event ssl_established(c: connection)
 	{
 	if ( c?$ssl && c$ssl?$server_name )
@@ -99,11 +71,11 @@ event ssl_established(c: connection)
 event connection_finished(c: connection)
 	{
 	if ( c?$resp_hostname )
-		add_sumstats(c$id, c$resp_hostname, c$resp$size);
+		hook add_sumstats(c$id, c$resp_hostname, c$resp$size);
 	}
 
 event HTTP::log_http(rec: HTTP::Info)
 	{
 	if( rec?$host )
-		add_sumstats(rec$id, rec$host, rec$response_body_len);
+		hook add_sumstats(rec$id, rec$host, rec$response_body_len);
 	}

scripts/policy/misc/app-stats/plugins/__load__.bro

@@ -0,0 +1,6 @@
+@load ./facebook
+@load ./gmail
+@load ./google
+@load ./netflix
+@load ./pandora
+@load ./youtube

scripts/policy/misc/app-stats/plugins/facebook.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.(facebook\.com|fbcdn\.net)$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="facebook"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/app-stats/plugins/gmail.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.gmail\.com$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="gmail"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/app-stats/plugins/google.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.google\.com$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="google"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/app-stats/plugins/netflix.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.nflximg\.com$/ in hostname && size > 200*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="netflix"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/app-stats/plugins/pandora.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="pandora"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/app-stats/plugins/youtube.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="youtube"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/detect-traceroute/main.bro

@@ -29,7 +29,7 @@ export {
 	## Defines the threshold for ICMP Time Exceeded messages for a src-dst pair.
 	## This threshold only comes into play after a host is found to be
 	## sending low ttl packets.
-	const icmp_time_exceeded_threshold = 3 &redef;
+	const icmp_time_exceeded_threshold: double = 3 &redef;
 
 	## Interval at which to watch for the
 	## :bro:id:`ICMPTimeExceeded::icmp_time_exceeded_threshold` variable to be crossed.
@@ -57,16 +57,17 @@ event bro_init() &priority=5
 
 	local r1: SumStats::Reducer = [$stream="traceroute.time_exceeded", $apply=set(SumStats::UNIQUE)];
 	local r2: SumStats::Reducer = [$stream="traceroute.low_ttl_packet", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=icmp_time_exceeded_interval,
+	SumStats::create([$name="traceroute-detection",
+	                  $epoch=icmp_time_exceeded_interval,
 	                  $reducers=set(r1, r2),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	# Give a threshold value of zero depending on if the host
 	                  	# sends a low ttl packet.
 	                  	if ( require_low_ttl_packets && result["traceroute.low_ttl_packet"]$sum == 0 )
-	                  		return 0;
+	                  		return 0.0;
 	                  	else
-	                  		return result["traceroute.time_exceeded"]$unique;
+	                  		return result["traceroute.time_exceeded"]$unique+0;
 	                  	},
 	                  $threshold=icmp_time_exceeded_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =

scripts/policy/misc/scan.bro

@@ -39,15 +39,11 @@ export {
 
 	## The threshold of a unique number of hosts a scanning host has to have failed
 	## connections with on a single port.
-	const addr_scan_threshold = 25 &redef;
+	const addr_scan_threshold = 25.0 &redef;
 
 	## The threshold of a number of unique ports a scanning host has to have failed
 	## connections with on a single victim host.
-	const port_scan_threshold = 15 &redef;
-
-	## Custom thresholds based on service for address scan.  This is primarily
-	## useful for setting reduced thresholds for specific ports.
-	const addr_scan_custom_thresholds: table[port] of count &redef;
+	const port_scan_threshold = 15.0 &redef;
 
 	global Scan::addr_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
 	global Scan::port_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
@@ -56,11 +52,12 @@ export {
 event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="scan.addr.fail", $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=addr_scan_interval,
+	SumStats::create([$name="addr-scan",
+	                  $epoch=addr_scan_interval,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["scan.addr.fail"]$unique);
+	                  	return result["scan.addr.fail"]$unique+0.0;
 	                  	},
 	                  #$threshold_func=check_addr_scan_threshold,
 	                  $threshold=addr_scan_threshold,
@@ -80,11 +77,12 @@ event bro_init() &priority=5
 
 	# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
 	local r2: SumStats::Reducer = [$stream="scan.port.fail", $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=port_scan_interval,
+	SumStats::create([$name="port-scan",
+	                  $epoch=port_scan_interval,
 	                  $reducers=set(r2),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["scan.port.fail"]$unique);
+	                  	return result["scan.port.fail"]$unique+0.0;
 	                  	},
 	                  $threshold=port_scan_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =

scripts/policy/protocols/ftp/detect-bruteforcing.bro

@@ -17,7 +17,7 @@ export {
 
 	## How many rejected usernames or passwords are required before being
 	## considered to be bruteforcing.
-	const bruteforce_threshold = 20 &redef;
+	const bruteforce_threshold: double = 20 &redef;
 
 	## The time period in which the threshold needs to be crossed before
 	## being reset.
@@ -28,11 +28,12 @@ export {
 event bro_init()
 	{
 	local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=bruteforce_measurement_interval,
+	SumStats::create([$name="ftp-detect-bruteforcing",
+	                  $epoch=bruteforce_measurement_interval,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return result["ftp.failed_auth"]$num;
+	                  	return result["ftp.failed_auth"]$num+0.0;
 	                  	},
 	                  $threshold=bruteforce_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =

scripts/policy/protocols/http/detect-sqli.bro

@@ -28,7 +28,7 @@ export {
 	## Defines the threshold that determines if an SQL injection attack
 	## is ongoing based on the number of requests that appear to be SQL
 	## injection attacks.
-	const sqli_requests_threshold = 50 &redef;
+	const sqli_requests_threshold: double = 50.0 &redef;
 
 	## Interval at which to watch for the
 	## :bro:id:`HTTP::sqli_requests_threshold` variable to be crossed.
@@ -64,11 +64,12 @@ event bro_init() &priority=3
 	# determine when it looks like an actual attack and how to respond when
 	# thresholds are crossed.
 	local r1: SumStats::Reducer = [$stream="http.sqli.attacker", $apply=set(SumStats::SUM), $samples=collect_SQLi_samples];
-	SumStats::create([$epoch=sqli_requests_interval,
+	SumStats::create([$name="detect-sqli-attackers",
+	                  $epoch=sqli_requests_interval,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["http.sqli.attacker"]$sum);
+	                  	return result["http.sqli.attacker"]$sum;
 	                  	},
 	                  $threshold=sqli_requests_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
@@ -82,11 +83,12 @@ event bro_init() &priority=3
 	                  	}]);
 
 	local r2: SumStats::Reducer = [$stream="http.sqli.victim", $apply=set(SumStats::SUM), $samples=collect_SQLi_samples];
-	SumStats::create([$epoch=sqli_requests_interval,
+	SumStats::create([$name="detect-sqli-victims",
+	                  $epoch=sqli_requests_interval,
 	                  $reducers=set(r2),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["http.sqli.victim"]$sum);
+	                  	return result["http.sqli.victim"]$sum;
 	                  	},
 	                  $threshold=sqli_requests_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -27,7 +27,7 @@ export {
 
 	## The number of failed SSH connections before a host is designated as
 	## guessing passwords.
-	const password_guesses_limit = 30 &redef;
+	const password_guesses_limit: double = 30 &redef;
 
 	## The amount of time to remember presumed non-successful logins to build
 	## model of a password guesser.
@@ -43,11 +43,12 @@ export {
 event bro_init()
 	{
 	local r1: SumStats::Reducer = [$stream="ssh.login.failure", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=guessing_timeout,
+	SumStats::create([$name="detect-ssh-bruteforcing",
+	                  $epoch=guessing_timeout,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["ssh.login.failure"]$sum);
+	                  	return result["ssh.login.failure"]$sum;
 	                  	},
 	                  $threshold=password_guesses_limit,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =