Updates and fixes for HTTP analysis scripts.

- File hashing is enabled by default and fixed.
- Other small fixes.

policy/http.bro

@@ -2,14 +2,12 @@
 
 ## Author: Seth Hall <seth@icir.org> - Inspired by the work of many others.
 
-
-@load http/utils
 @load http/base
 @load http/base-extended
 @load http/detect-sqli
 @load http/detect-intel
 @load http/file-ident
+@load http/file-hash
 @load http/software
 @load http/headers
-
 @load http/detect-webapps

policy/http/base.bro

@@ -89,8 +89,6 @@ function new_http_session(c: connection): Info
 	tmp$ts=network_time();
 	tmp$uid=c$uid;
 	tmp$id=c$id;
-	# TODO: remove this when &default on this set isn't segfaulting Bro anymore.
-	#tmp$tags = set();
 	return tmp;
 	}
 	

policy/http/file-hash.bro

@@ -43,7 +43,7 @@ export {
 
 # Once a file that we're interested has begun downloading, initialize
 # an MD5 hash.
-event file_transferred(c: connection, prefix: string, descr: string, mime_type: string) &priority=-5
+event file_transferred(c: connection, prefix: string, descr: string, mime_type: string) &priority=5
 	{
 	if ( ! c?$http ) return;
 	
@@ -66,7 +66,7 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string
 	
 # When the file finishes downloading, finish the hash, check for the hash
 # in the MHR, and raise a notice if the hash is there.
-event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority=-5
+event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority=-4
 	{
 	if ( is_orig || ! c?$http ) return;
 	
@@ -98,7 +98,7 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &
 		}
 	}
 
-event connect_state_remove(c: connection) &priority=-5
+event connection_state_remove(c: connection) &priority=-5
 	{
 	if ( c?$http && c$http$calculating_md5 )
 		md5_hash_finish(c$id);

policy/http/file-ident.bro

@@ -59,9 +59,8 @@ redef Signatures::ignored_ids += /^matchfile-/;
 
 event signature_match(state: signature_state, msg: string, data: string) &priority=5
 	{
-	#print "signature match";
 	# Only signatures matching file types are dealt with here.
-	if ( /^matchfile/ !in state$sig_id ) return;
+	if ( /^matchfile-/ !in state$sig_id ) return;
 	
 	local c = state$conn;
 	
@@ -73,6 +72,14 @@ event signature_match(state: signature_state, msg: string, data: string) &priori
 	# Set the mime type that was detected.
 	c$http$mime_type = msg;
 	
+	# Fire the file_transferred event so that it can be picked up by other
+	# scripts, like the http/file-hash script since that uses file type to
+	# conditionally calculate an MD5 sum.
+	# TODO: We are leaving the descr field blank for now, but it shouldn't 
+	#       matter too much and hopefully the more generic file analysis code
+	#       will make this completely irrelevant.
+	event file_transferred(c, data, "", msg);
+	
 	if ( msg in mime_types_extensions && 
 	     c$http?$uri && mime_types_extensions[msg] !in c$http$uri )
 		{

policy/http/software.bro

@@ -8,13 +8,18 @@ module HTTP;
 redef enum Software::Type += {
 	WEB_SERVER,
 	WEB_BROWSER,
-	WEB_BROWSER_PLUGIN,
+	WEB_BROWSER_PLUGIN
 };
 
 
 export {
 	## The pattern of HTTP User-Agents which you would like to ignore.
 	const ignored_user_agents = /NO_DEFAULT/ &redef;
+	
+	## These are patterns to identify browser plugins (including toolbars)
+	## based on the User-Agent header.
+	const plugin_user_agents = /BingBar [0-9\.]*/         # Bing toolbar
+	                          | /GoogleToolbar [0-9\.]*;/ &redef; # Google toolbar
 }
 
 event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=2
@@ -24,7 +29,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 		if ( name == "USER-AGENT" && ignored_user_agents !in value )
 			{
 			local ua_type = WEB_BROWSER;
-			if ( /^Java/ in value )
+			if ( plugin_user_agents in value )
 				ua_type = WEB_BROWSER_PLUGIN;
 				
 			Software::found(c$id, Software::parse(value, c$id$orig_h, ua_type));