Script cleanup.

- Defaults for all built-in asset tracking changed to LOCAL_HOSTS
- Added a tuning script for changing asset tracking
  to ALL_HOSTS in all of the core scripts that do
  asset tracking.
- Default Notice::policy files notices instead of alarming on them.
- Moved KnownHosts::Info back to export section because
  the log_known_hosts event can't be defined in the
  export section without it.
- Moved the Malware Hash Registry detection out of
  the core HTTP protocol scripts.

policy/bro.init

@@ -318,7 +318,6 @@ function add_interface(iold: string, inew: string): string
 	else
 		return fmt("%s %s", iold, inew);
 	}
-
 global interfaces = "" &add_func = add_interface;
 
 function add_signature_file(sold: string, snew: string): string
@@ -328,7 +327,6 @@ function add_signature_file(sold: string, snew: string): string
 	else
 		return cat(sold, " ", snew);
 	}
-
 global signature_files = "" &add_func = add_signature_file;
 
 const passive_fingerprint_file = "signatures/p0f.fp" &redef;
@@ -1266,56 +1264,6 @@ type bt_tracker_headers: table[string] of string;
 
 @load event.bif.bro
 
-function subst(s: string, from: pattern, to: string): string
-	{
-	local p = split_all(s, from);
-
-	for ( p_i in p )
-		if ( p_i % 2 == 0 )
-			p[p_i] = to;
-
-	return cat_string_array(p);
-	}
-
-
-type pattern_match_result: record {
-	matched: bool;
-	str: string;
-	off: count;
-};
-
-function match_pattern(s: string, p:pattern): pattern_match_result
-	{
-	local a = split_n(s, p, T, 1);
-
-	if ( length(a) == 1 )
-		# no match
-		return [$matched = F, $str = "", $off = 0];
-	else
-		return [$matched = T, $str = a[2], $off = byte_len(a[1]) + 1];
-	}
-
-function cut_tail(s: string, tail_len: count): string
-	{
-	local len = byte_len(s);
-
-	if ( tail_len > len )
-		tail_len = len;
-
-	return sub_bytes(s, 1, int_to_count(len - tail_len));
-	}
-
-# Given a string, returns an escaped version.  This means that
-# (1) any occurrences of any character in "chars" are escaped using '\', and
-# (2) any '\'s are likewise escaped.
-function string_escape(s: string, chars: string): string
-	{
-	s = subst_string(s, "\\", "\\\\");
-	for ( c in chars )
-		s = subst_string(s, c, cat("\\", c));
-	return s;
-	}
-
 # The filter the user has set via the -f command line options, or
 # empty if none.
 const cmd_line_bpf_filter = "" &redef;
@@ -1539,13 +1487,6 @@ const skip_http_data = F &redef;
 # UDP tunnels. See also: udp_tunnel_port, policy/udp-tunnel.bro.
 const parse_udp_tunnels = F &redef;
 
-
-@load utils/directions-and-hosts
-## Scripts that do some sort of asset tracking like the software framework
-## or the conn/known-hosts script should use this as their default setting
-## for which hosts should be detected and tracked.
-const default_asset_tracking = ALL_HOSTS &redef;
-
-#@load utils
 #@load site
 #@load dpd
+@load weird

policy/detectors/http-MHR.bro

@@ -0,0 +1,35 @@
+##! This script takes MD5 sums of files transferred over HTTP and checks them with
+##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
+##! By default, not all file transfers will have MD5 sums calculated.  Read the
+##! documentation for the protocols/http/file-hash.bro script to see how to 
+##! configure which transfers will have hashes calculated.
+
+@load notice
+@load http/file-hash
+
+export {
+	redef enum Notice::Type += { 
+		## If the MD5 sum of a file transferred over HTTP 
+		Malware_Hash_Registry_Match
+	};
+}
+
+event log_http(rec: HTTP::Info)
+	{
+	if ( rec?$md5 )
+		{
+		local url = HTTP::build_url(rec);
+		local hash_domain = fmt("%s.malware.hash.cymru.com", rec$md5);
+		
+		when ( local addrs = lookup_hostname(hash_domain) )
+			{
+			# 127.0.0.2 indicates that the md5 sum was found in the MHR.
+			if ( 127.0.0.2 in addrs )
+				{
+				local message = fmt("%s %s %s", rec$id$orig_h, rec$md5, url);
+				NOTICE([$note=Malware_Hash_Registry_Match, 
+				        $msg=message, $id=rec$id, $URL=url]);
+				}
+			}
+		}
+	}

policy/frameworks/notice/base.bro

@@ -91,7 +91,7 @@ export {
 	# policy is set.
 	const policy: set[Notice::PolicyItem] = {
 		[$pred(n: Notice::Info) = { return T; },
-		 $result = ACTION_ALARM_ALWAYS,
+		 $result = ACTION_FILE,
 		 $priority = 0],
 	} &redef;
 	

policy/frameworks/signatures/base.bro

@@ -116,35 +116,13 @@ event bro_init()
 	{
 	Log::create_stream(SIGNATURES, [$columns=Info, $ev=log_signature]);
 	}
-	
-function add_signature_file(sold: string, snew: string): string
-	{
-	if ( sold == "" )
-		return snew;
-	else
-		return cat(sold, " ", snew);
-	}
-
-global signature_files = "" &add_func = add_signature_file;
-
-
-# Given a string, returns an escaped version suitable for being
-# printed in the colon-separated notice format.  This means that
-# (1) any colons are escaped using '\', and (2) any '\'s are
-# likewise escaped.
-function signature_escape(s: string): string
-	{
-	s = subst_string(s, "\\", "\\\\");
-	return subst_string(s, ":", "\\:");
-	}
-	
+		
 # Returns true if the given signature has already been triggered for the given
 # [orig, resp] pair.
 function has_signature_matched(id: string, orig: addr, resp: addr): bool
 	{
 	return [orig, resp] in vert_table ? id in vert_table[orig, resp] : F;
 	}
-	
 
 event sig_summary(orig: addr, id: string, msg: string)
 	{

policy/frameworks/software/base.bro

@@ -65,7 +65,7 @@ export {
 	
 	## The hosts whose software should be detected and tracked.
 	## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS
-	const asset_tracking = default_asset_tracking &redef;
+	const asset_tracking = LOCAL_HOSTS &redef;
 	
 	## Some software is more interesting when the version changes and this
 	## a set of all software that should raise a notice when a different 

policy/protocols/conn/known-hosts.bro

@@ -3,14 +3,24 @@
 ##! output provides an easy way to determine a count of the IP addresses in
 ##! use on a network per day.
 
+@load utils/directions-and-hosts
+
 module KnownHosts;
 
 export {
 	redef enum Log::ID += { KNOWN_HOSTS };
 	
+	type Info: record {
+		## The timestamp at which the host was detected.
+		ts:      time &log;
+		## The address that was detected originating or responding to a TCP 
+		## connection.
+		host:    addr &log;
+	};
+	
 	## The hosts whose existence should be logged and tracked.
 	## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS
-	const asset_tracking = default_asset_tracking &redef;
+	const asset_tracking = LOCAL_HOSTS &redef;
 	
 	## The set of all known addresses to store for preventing duplicate 
 	## logging of addresses.  It can also be used from other scripts to 
@@ -22,16 +32,6 @@ export {
 	global log_known_hosts: event(rec: Info);
 }
 
-## This type is left out of the export section because users have no way
-## to add extra data to it before it is logged.
-type Info: record {
-	## The timestamp at which the host was detected.
-	ts:      time &log;
-	## The address that was detected originating or responding to a TCP 
-	## connection.
-	host:    addr &log;
-};
-
 event bro_init()
 	{
 	Log::create_stream(KNOWN_HOSTS, [$columns=Info, $ev=log_known_hosts]);

policy/protocols/conn/known-services.bro

@@ -1,4 +1,9 @@
-@load functions
+##! This script logs and tracks services.  In the case of this script, a service
+##! is defined as an IP address and port which has responded to and fully 
+##! completed a TCP handshake with another host.  If a protocol is detected
+##! during the session, the protocol will also be logged.
+
+@load utils/directions-and-hosts
 
 module KnownServices;
 
@@ -16,7 +21,7 @@ export {
 	};
 	
 	## The hosts whose services should be tracked and logged.
-	const asset_tracking = default_asset_tracking &redef;
+	const asset_tracking = LOCAL_HOSTS &redef;
 	
 	global known_services: set[addr, port] &create_expire=1day &synchronized;
 	

policy/protocols/http/file-extract.bro

@@ -1,12 +1,5 @@
-# Extracts the items from HTTP traffic, one per file.
-# Files are named:
-#
-#    <prefix>.<n>.<orig-addr>_<orig-port>.<resp-addr>_<resp-port>.<is-orig>
-#
-# where <prefix> is a redef'able prefix (default: "http-item"), <n> is
-# a number uniquely identifying the item, the next four are describe
-# the connection tuple, and <is-orig> is "orig" if the item was transferred
-# from the originator to the responder, "resp" otherwise.
+##! Extracts the items from HTTP traffic, one per file.  At this time only 
+##! the message body from the server can be extracted with this script.
 
 @load http/file-ident
 @load utils/files
@@ -24,32 +17,32 @@ export {
 		## This field can be set per-connection to determine if the entity body
 		## will be extracted.  It must be set to T on or before the first 
 		## entity_body_data event.
-		extract_file:        bool &default=F;
+		extract_file:     bool &default=F;
 	
 		## This is the holder for the file handle as the file is being written
 		## to disk.
-		extraction_file:      file &log &optional;
+		extraction_file:  file &log &optional;
 	};
 
 	redef record State += {
-		entity_bodies:       count &optional;
+		entity_bodies:    count &optional;
 	};
 }
 
-## Mark files to be extracted if they were identified as a mime type matched 
-## by the extract_file_types variable and they aren't being extracted yet.
-event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=6
-	{
-	if ( ! c$http$extract_file &&
-	     c$http?$mime_type &&
-		 extract_file_types in c$http$mime_type )
-		c$http$extract_file = T;
-	}
-
 event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=5
 	{
+	# Client body extraction is not currently supported in this script.
+	if ( is_orig ) return;
+		
 	if ( ! c$http$extract_file )
-		return;
+		{
+		# Mark files to be extracted if they were identified as a mime type matched 
+		# by the extract_file_types variable and they aren't being extracted yet.
+		if ( c$http?$mime_type && extract_file_types in c$http$mime_type )
+			c$http$extract_file = T;
+		else
+			return;
+		}
 		
 	# Open a file handle if this file hasn't seen any data yet.
 	if ( ! c$http?$extraction_file )

policy/protocols/http/file-hash.bro

@@ -9,10 +9,6 @@ export {
 	redef enum Notice::Type += {
 		## Indicates that an MD5 sum was calculated for an HTTP response body.
 		MD5,
-		
-		## Indicates an MD5 sum was found in Team Cymru's Malware Hash Registry.
-		## http://www.team-cymru.org/Services/MHR/
-		MHR_Malware,
 	};
 
 	redef record Info += {
@@ -35,24 +31,16 @@ export {
 	                   &redef;
 }
 
-# Once a file that we're interested has begun downloading, initialize
-# an MD5 hash.
-event file_transferred(c: connection, prefix: string, descr: string, mime_type: string) &priority=5
+# Initialize and calculate the hash.
+event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=-5
 	{
-	if ( ! c?$http ) return;
+	if ( is_orig || ! c?$http || ! c$http$calc_md5 ) return;
 	
-	if ( (generate_md5 in mime_type || c$http$calc_md5 ) && 
-		 ! c$http$calculating_md5 )
+	if ( ! c$http$calculating_md5 )
 		{
 		c$http$calculating_md5 = T;
 		md5_hash_init(c$id);
 		}
-	}
-
-# As the file downloads, continue building the hash.
-event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=-5
-	{
-	if ( is_orig ) return;
 	
 	if ( c$http$calculating_md5 )
 		md5_hash_update(c$id, data);
@@ -72,17 +60,6 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &
 		
 		NOTICE([$note=MD5, $msg=fmt("%s %s %s", c$id$orig_h, c$http$md5, url),
 		        $sub=c$http$md5, $conn=c, $URL=url]);
-		
-		local hash_domain = fmt("%s.malware.hash.cymru.com", c$http$md5);
-		when ( local addrs = lookup_hostname(hash_domain) )
-			{
-			# 127.0.0.2 indicates that the md5 sum was found in the MHR.
-			if ( 127.0.0.2 in addrs )
-				{
-				local message = fmt("%s %s %s", c$id$orig_h, c$http$md5, url);
-				NOTICE([$note=MHR_Malware, $msg=message, $conn=c, $URL=url]);
-				}
-			}
 		}
 	}
 

policy/protocols/http/file-ident.bro

@@ -7,6 +7,10 @@
 @load notice
 @load signatures
 
+redef signature_files += "http/file-ident.sig";
+# Ignore the signatures used to match files
+redef Signatures::ignored_ids += /^matchfile-/;
+
 module HTTP;
 
 export {
@@ -32,17 +36,12 @@ export {
 	} &redef;
 }
 
-redef signature_files += "http/file-ident.sig";
-# Ignore the signatures used to match files
-redef Signatures::ignored_ids += /^matchfile-/;
-
 event signature_match(state: signature_state, msg: string, data: string) &priority=5
 	{
 	# Only signatures matching file types are dealt with here.
 	if ( /^matchfile-/ !in state$sig_id ) return;
-	
+
 	local c = state$conn;
-	
 	set_state(c, F, F);
 	
 	# Not much point in any of this if we don't know about the HTTP session.

policy/tuning/track-all-assets.bro

@@ -0,0 +1,8 @@
+
+@load software
+@load conn/known-hosts
+@load conn/known-services
+
+redef Software::asset_tracking      = ALL_HOSTS;
+redef KnownHosts::asset_tracking    = ALL_HOSTS;
+redef KnownServices::asset_tracking = ALL_HOSTS;

policy/utils/pattern.bro

@@ -1,4 +1,4 @@
-##! Functions for creating patterns.
+##! Functions for creating and working with patterns.
 
 ## This function only works at or before init time.  Given a pattern as a string
 ## with two tildes (~~) contained in it, it will return a pattern with the
@@ -19,3 +19,34 @@ function set_to_regex(ss: set[string], pat: string): pattern
 		}
 	return string_to_pattern(sub(pat, /~~/, return_pat), F);
 	}
+
+type PatternMatchResult: record {
+	## T if a match was found, F otherwise.
+	matched: bool;
+	## Portion of string that first matched.
+	str: string;
+	## 1-based offset where match starts.
+	off: count;
+};
+
+## Matches the given pattern against the given string, returning
+## a :bro:type:`PatternMatchResult` record.
+## For example:
+##     match_pattern("foobar", /o*[a-k]/)
+## returns:
+##     [matched=T, str=f, off=1]
+## because the *first* match is for zero o's followed by an [a-k],
+## while:
+##     match_pattern("foobar", /o+[a-k]/)
+## returns:
+##     [matched=T, str=oob, off=2]
+function match_pattern(s: string, p: pattern): PatternMatchResult
+	{
+	local a = split_n(s, p, T, 1);
+
+	if ( |a| == 1 )
+		# no match
+		return [$matched = F, $str = "", $off = 0];
+	else
+		return [$matched = T, $str = a[2], $off = |a[1]| + 1];
+	}

policy/utils/strings.bro

@@ -25,3 +25,22 @@ function join_string_set(ss: set[string], j: string): string
 		}
 	return output;
 	}
+
+## Given a string, returns an escaped version.  This means that
+## (1) any occurrences of any character in "chars" are escaped using '\', and
+## (2) any '\'s are likewise escaped.
+function string_escape(s: string, chars: string): string
+	{
+	s = subst_string(s, "\\", "\\\\");
+	for ( c in chars )
+		s = subst_string(s, c, cat("\\", c));
+	return s;
+	}
+
+## Cut a number of character from the end of the given string.
+function cut_tail(s: string, tail_len: count): string
+	{
+	if ( tail_len > |s| )
+		tail_len = |s|;
+	return sub_bytes(s, 1, int_to_count(|s| - tail_len));
+	}

policy/weird.bro

@@ -1,6 +1,4 @@
-@load functions
 @load notice
-
 @load utils/conn_ids
 
 module Weird;