Removed app-stats scripts.

Addresses BIT-1171.

NEWS

@@ -145,6 +145,11 @@ Deprecated Functionality
       decode_base64() and encode_base64(), which take an optional
       parameter to change the Base64 alphabet.
 
+    - The app-stats scripts have been removed because they weren't
+      being maintained and they were becoming inaccurate. They
+      were also prone to needing more regular updates as the internet
+      changed and will likely be more relevant if maintained externally.
+
 Bro 2.4
 =======
 

scripts/policy/misc/app-stats/README

@@ -1 +0,0 @@
-AppStats collects information about web applications in use on the network.

scripts/policy/misc/app-stats/__load__.bro

@@ -1,2 +0,0 @@
-@load ./main
-@load ./plugins

scripts/policy/misc/app-stats/main.bro

@@ -1,77 +0,0 @@
-##! AppStats collects information about web applications in use
-##! on the network.  
-
-@load base/protocols/http
-@load base/protocols/ssl
-@load base/frameworks/sumstats
-
-module AppStats;
-
-export {
-	redef enum Log::ID += { LOG };
-
-	type Info: record {
-		## Timestamp when the log line was finished and written.
-		ts:         time   &log;
-		## Time interval that the log line covers.
-		ts_delta:   interval &log;
-		## The name of the "app", like "facebook" or "netflix".
-		app:        string &log;
-		## The number of unique local hosts using the app.
-		uniq_hosts: count  &log;
-		## The number of hits to the app in total.
-		hits:       count  &log;
-		## The total number of bytes received by users of the app.
-		bytes:      count  &log;
-	};
-
-	## The frequency of logging the stats collected by this script.
-	const break_interval = 15mins &redef;
-}
-
-redef record connection += {
-	resp_hostname: string &optional;
-};
-
-global add_sumstats: hook(id: conn_id, hostname: string, size: count);
-
-
-event bro_init() &priority=3
-	{
-	Log::create_stream(AppStats::LOG, [$columns=Info, $path="app_stats"]);
-
-	local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
-	local r2: SumStats::Reducer = [$stream="apps.hits",  $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$name="app-metrics",
-	                  $epoch=break_interval,
-	                  $reducers=set(r1, r2),
-	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
-	                  	{
-	                  	local l: Info;
-	                  	l$ts         = network_time();
-	                  	l$ts_delta   = break_interval;
-	                  	l$app        = key$str;
-	                  	l$bytes      = double_to_count(floor(result["apps.bytes"]$sum));
-	                  	l$hits       = result["apps.hits"]$num;
-	                  	l$uniq_hosts = result["apps.hits"]$unique;
-	                  	Log::write(LOG, l);
-	                  	}]);
-	}
-
-event ssl_established(c: connection)
-	{
-	if ( c?$ssl && c$ssl?$server_name )
-		c$resp_hostname = c$ssl$server_name;
-	}
-
-event connection_finished(c: connection)
-	{
-	if ( c?$resp_hostname )
-		hook add_sumstats(c$id, c$resp_hostname, c$resp$size);
-	}
-
-event HTTP::log_http(rec: HTTP::Info)
-	{
-	if( rec?$host )
-		hook add_sumstats(rec$id, rec$host, rec$response_body_len);
-	}

scripts/policy/misc/app-stats/plugins/README

@@ -1 +0,0 @@
-Plugins for AppStats.

scripts/policy/misc/app-stats/plugins/__load__.bro

@@ -1,6 +0,0 @@
-@load ./facebook
-#@load ./gmail
-#@load ./google
-#@load ./netflix
-#@load ./pandora
-#@load ./youtube

scripts/policy/misc/app-stats/plugins/facebook.bro

@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.(facebook\.com|fbcdn\.net)$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="facebook"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
-		}
-	}

scripts/policy/misc/app-stats/plugins/gmail.bro

@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.gmail\.com$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="gmail"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
-		}
-	}

scripts/policy/misc/app-stats/plugins/google.bro

@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.google\.com$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="google"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
-		}
-	}

scripts/policy/misc/app-stats/plugins/netflix.bro

@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.nflximg\.com$/ in hostname && size > 200*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="netflix"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
-		}
-	}

scripts/policy/misc/app-stats/plugins/pandora.bro

@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="pandora"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
-		}
-	}

scripts/policy/misc/app-stats/plugins/youtube.bro

@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="youtube"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
-		}
-	}

scripts/site/local.bro

@@ -11,10 +11,6 @@
 # Load the scan detection script.
 @load misc/scan
 
-# Log some information about web applications being used by users
-# on your network.
-@load misc/app-stats
-
 # Detect traceroute being run on the network.
 @load misc/detect-traceroute
 

scripts/test-all-policy.bro

@@ -41,15 +41,6 @@
 @load integration/barnyard2/types.bro
 @load integration/collective-intel/__load__.bro
 @load integration/collective-intel/main.bro
-@load misc/app-stats/__load__.bro
-@load misc/app-stats/main.bro
-@load misc/app-stats/plugins/__load__.bro
-@load misc/app-stats/plugins/facebook.bro
-@load misc/app-stats/plugins/gmail.bro
-@load misc/app-stats/plugins/google.bro
-@load misc/app-stats/plugins/netflix.bro
-@load misc/app-stats/plugins/pandora.bro
-@load misc/app-stats/plugins/youtube.bro
 @load misc/capture-loss.bro
 @load misc/detect-traceroute/__load__.bro
 @load misc/detect-traceroute/main.bro

testing/btest/Baseline/coverage.find-bro-logs/out

@@ -1,4 +1,3 @@
-app_stats
 barnyard2
 capture_loss
 cluster

testing/btest/scripts/site/local-compat.test

@@ -24,10 +24,6 @@
 # Load the scan detection script.
 @load misc/scan
 
-# Log some information about web applications being used by users 
-# on your network.
-@load misc/app-stats
-
 # Detect traceroute being run on the network.  
 @load misc/detect-traceroute