From 6971a70903d8cbb0245a7727446c41029c4fada7 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 17 Jun 2016 11:55:26 -0400
Subject: [PATCH] Removed app-stats scripts.

Addresses BIT-1171.
---
 NEWS                                          |  5 ++
 scripts/policy/misc/app-stats/README          |  1 -
 scripts/policy/misc/app-stats/__load__.bro    |  2 -
 scripts/policy/misc/app-stats/main.bro        | 77 -------------------
 scripts/policy/misc/app-stats/plugins/README  |  1 -
 .../misc/app-stats/plugins/__load__.bro       |  6 --
 .../misc/app-stats/plugins/facebook.bro       | 12 ---
 .../policy/misc/app-stats/plugins/gmail.bro   | 12 ---
 .../policy/misc/app-stats/plugins/google.bro  | 12 ---
 .../policy/misc/app-stats/plugins/netflix.bro | 12 ---
 .../policy/misc/app-stats/plugins/pandora.bro | 12 ---
 .../policy/misc/app-stats/plugins/youtube.bro | 12 ---
 scripts/site/local.bro                        |  4 -
 scripts/test-all-policy.bro                   |  9 ---
 .../btest/Baseline/coverage.find-bro-logs/out |  1 -
 testing/btest/scripts/site/local-compat.test  |  4 -
 16 files changed, 5 insertions(+), 177 deletions(-)
 delete mode 100644 scripts/policy/misc/app-stats/README
 delete mode 100644 scripts/policy/misc/app-stats/__load__.bro
 delete mode 100644 scripts/policy/misc/app-stats/main.bro
 delete mode 100644 scripts/policy/misc/app-stats/plugins/README
 delete mode 100644 scripts/policy/misc/app-stats/plugins/__load__.bro
 delete mode 100644 scripts/policy/misc/app-stats/plugins/facebook.bro
 delete mode 100644 scripts/policy/misc/app-stats/plugins/gmail.bro
 delete mode 100644 scripts/policy/misc/app-stats/plugins/google.bro
 delete mode 100644 scripts/policy/misc/app-stats/plugins/netflix.bro
 delete mode 100644 scripts/policy/misc/app-stats/plugins/pandora.bro
 delete mode 100644 scripts/policy/misc/app-stats/plugins/youtube.bro

diff --git a/NEWS b/NEWS
index 5059da1139..87a1a0ca0c 100644
--- a/NEWS
+++ b/NEWS
@@ -145,6 +145,11 @@ Deprecated Functionality
       decode_base64() and encode_base64(), which take an optional
       parameter to change the Base64 alphabet.
 
+    - The app-stats scripts have been removed because they weren't
+      being maintained and they were becoming inaccurate. They
+      were also prone to needing more regular updates as the internet
+      changed and will likely be more relevant if maintained externally.
+
 Bro 2.4
 =======
 
diff --git a/scripts/policy/misc/app-stats/README b/scripts/policy/misc/app-stats/README
deleted file mode 100644
index a0fe433cc8..0000000000
--- a/scripts/policy/misc/app-stats/README
+++ /dev/null
@@ -1 +0,0 @@
-AppStats collects information about web applications in use on the network.
diff --git a/scripts/policy/misc/app-stats/__load__.bro b/scripts/policy/misc/app-stats/__load__.bro
deleted file mode 100644
index c468d055ee..0000000000
--- a/scripts/policy/misc/app-stats/__load__.bro
+++ /dev/null
@@ -1,2 +0,0 @@
-@load ./main
-@load ./plugins
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/main.bro b/scripts/policy/misc/app-stats/main.bro
deleted file mode 100644
index d80763c699..0000000000
--- a/scripts/policy/misc/app-stats/main.bro
+++ /dev/null
@@ -1,77 +0,0 @@
-##! AppStats collects information about web applications in use
-##! on the network.  
-
-@load base/protocols/http
-@load base/protocols/ssl
-@load base/frameworks/sumstats
-
-module AppStats;
-
-export {
-	redef enum Log::ID += { LOG };
-
-	type Info: record {
-		## Timestamp when the log line was finished and written.
-		ts:         time   &log;
-		## Time interval that the log line covers.
-		ts_delta:   interval &log;
-		## The name of the "app", like "facebook" or "netflix".
-		app:        string &log;
-		## The number of unique local hosts using the app.
-		uniq_hosts: count  &log;
-		## The number of hits to the app in total.
-		hits:       count  &log;
-		## The total number of bytes received by users of the app.
-		bytes:      count  &log;
-	};
-
-	## The frequency of logging the stats collected by this script.
-	const break_interval = 15mins &redef;
-}
-
-redef record connection += {
-	resp_hostname: string &optional;
-};
-
-global add_sumstats: hook(id: conn_id, hostname: string, size: count);
-
-
-event bro_init() &priority=3
-	{
-	Log::create_stream(AppStats::LOG, [$columns=Info, $path="app_stats"]);
-
-	local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
-	local r2: SumStats::Reducer = [$stream="apps.hits",  $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$name="app-metrics",
-	                  $epoch=break_interval,
-	                  $reducers=set(r1, r2),
-	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
-	                  	{
-	                  	local l: Info;
-	                  	l$ts         = network_time();
-	                  	l$ts_delta   = break_interval;
-	                  	l$app        = key$str;
-	                  	l$bytes      = double_to_count(floor(result["apps.bytes"]$sum));
-	                  	l$hits       = result["apps.hits"]$num;
-	                  	l$uniq_hosts = result["apps.hits"]$unique;
-	                  	Log::write(LOG, l);
-	                  	}]);
-	}
-
-event ssl_established(c: connection)
-	{
-	if ( c?$ssl && c$ssl?$server_name )
-		c$resp_hostname = c$ssl$server_name;
-	}
-
-event connection_finished(c: connection)
-	{
-	if ( c?$resp_hostname )
-		hook add_sumstats(c$id, c$resp_hostname, c$resp$size);
-	}
-
-event HTTP::log_http(rec: HTTP::Info)
-	{
-	if( rec?$host )
-		hook add_sumstats(rec$id, rec$host, rec$response_body_len);
-	}
diff --git a/scripts/policy/misc/app-stats/plugins/README b/scripts/policy/misc/app-stats/plugins/README
deleted file mode 100644
index cb2e04d8ba..0000000000
--- a/scripts/policy/misc/app-stats/plugins/README
+++ /dev/null
@@ -1 +0,0 @@
-Plugins for AppStats.
diff --git a/scripts/policy/misc/app-stats/plugins/__load__.bro b/scripts/policy/misc/app-stats/plugins/__load__.bro
deleted file mode 100644
index 64126764fb..0000000000
--- a/scripts/policy/misc/app-stats/plugins/__load__.bro
+++ /dev/null
@@ -1,6 +0,0 @@
-@load ./facebook
-#@load ./gmail
-#@load ./google
-#@load ./netflix
-#@load ./pandora
-#@load ./youtube
diff --git a/scripts/policy/misc/app-stats/plugins/facebook.bro b/scripts/policy/misc/app-stats/plugins/facebook.bro
deleted file mode 100644
index edcb02b72a..0000000000
--- a/scripts/policy/misc/app-stats/plugins/facebook.bro
+++ /dev/null
@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.(facebook\.com|fbcdn\.net)$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="facebook"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
-		}
-	}
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/gmail.bro b/scripts/policy/misc/app-stats/plugins/gmail.bro
deleted file mode 100644
index 1642fb7651..0000000000
--- a/scripts/policy/misc/app-stats/plugins/gmail.bro
+++ /dev/null
@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.gmail\.com$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="gmail"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
-		}
-	}
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/google.bro b/scripts/policy/misc/app-stats/plugins/google.bro
deleted file mode 100644
index e1da3a9068..0000000000
--- a/scripts/policy/misc/app-stats/plugins/google.bro
+++ /dev/null
@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.google\.com$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="google"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
-		}
-	}
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/netflix.bro b/scripts/policy/misc/app-stats/plugins/netflix.bro
deleted file mode 100644
index 5d429f0caf..0000000000
--- a/scripts/policy/misc/app-stats/plugins/netflix.bro
+++ /dev/null
@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.nflximg\.com$/ in hostname && size > 200*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="netflix"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
-		}
-	}
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/pandora.bro b/scripts/policy/misc/app-stats/plugins/pandora.bro
deleted file mode 100644
index 6cfbfab72d..0000000000
--- a/scripts/policy/misc/app-stats/plugins/pandora.bro
+++ /dev/null
@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="pandora"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
-		}
-	}
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/youtube.bro b/scripts/policy/misc/app-stats/plugins/youtube.bro
deleted file mode 100644
index af872cfdac..0000000000
--- a/scripts/policy/misc/app-stats/plugins/youtube.bro
+++ /dev/null
@@ -1,12 +0,0 @@
-@load ../main
-
-module AppStats;
-
-hook add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="youtube"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
-		}
-	}
\ No newline at end of file
diff --git a/scripts/site/local.bro b/scripts/site/local.bro
index 704ca596dc..084a5d33a0 100644
--- a/scripts/site/local.bro
+++ b/scripts/site/local.bro
@@ -11,10 +11,6 @@
 # Load the scan detection script.
 @load misc/scan
 
-# Log some information about web applications being used by users
-# on your network.
-@load misc/app-stats
-
 # Detect traceroute being run on the network.
 @load misc/detect-traceroute
 
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index 2299fd3043..5f63c00db8 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -41,15 +41,6 @@
 @load integration/barnyard2/types.bro
 @load integration/collective-intel/__load__.bro
 @load integration/collective-intel/main.bro
-@load misc/app-stats/__load__.bro
-@load misc/app-stats/main.bro
-@load misc/app-stats/plugins/__load__.bro
-@load misc/app-stats/plugins/facebook.bro
-@load misc/app-stats/plugins/gmail.bro
-@load misc/app-stats/plugins/google.bro
-@load misc/app-stats/plugins/netflix.bro
-@load misc/app-stats/plugins/pandora.bro
-@load misc/app-stats/plugins/youtube.bro
 @load misc/capture-loss.bro
 @load misc/detect-traceroute/__load__.bro
 @load misc/detect-traceroute/main.bro
diff --git a/testing/btest/Baseline/coverage.find-bro-logs/out b/testing/btest/Baseline/coverage.find-bro-logs/out
index 9619ebb4b9..f7b7398bc8 100644
--- a/testing/btest/Baseline/coverage.find-bro-logs/out
+++ b/testing/btest/Baseline/coverage.find-bro-logs/out
@@ -1,4 +1,3 @@
-app_stats
 barnyard2
 capture_loss
 cluster
diff --git a/testing/btest/scripts/site/local-compat.test b/testing/btest/scripts/site/local-compat.test
index b7ed0ad932..a1013a855c 100644
--- a/testing/btest/scripts/site/local-compat.test
+++ b/testing/btest/scripts/site/local-compat.test
@@ -24,10 +24,6 @@
 # Load the scan detection script.
 @load misc/scan
 
-# Log some information about web applications being used by users 
-# on your network.
-@load misc/app-stats
-
 # Detect traceroute being run on the network.  
 @load misc/detect-traceroute