Merge remote-tracking branch 'origin/topic/seth/unified2-analyzer' into topic/robin/unified2-analyzer-merge

* origin/topic/seth/unified2-analyzer:
  Fixed a problem where the Unified2 analyzer was attached to every file.
  Fixing intel framework tests.
  Updating submodule(s).
  Add file name support to intel framework.
  Add file support to intel framework and slightly restructure intel http handling.

Conflicts:
	CHANGES
	VERSION
	scripts/base/files/unified2/main.bro
	testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
	testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

scripts/base/files/unified2/main.bro

@@ -136,7 +136,7 @@ event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Inp
 		}
 	}
 
-event bro_init()
+event bro_init() &priority=5
 	{
 	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]);
 
@@ -200,8 +200,8 @@ event file_new(f: fa_file)
 	if ( |parts| == 3 )
 		file_dir = parts[1];
 
-	if ( f$source in watch_file ||
-		compress_path(watch_dir) == file_dir )
+	if ( (watch_file != "" && f$source == watch_file) || 
+	     (watch_dir != "" && compress_path(watch_dir) == file_dir) )
 		{
 		Files::add_analyzer(f, Files::ANALYZER_UNIFIED2);
 		f$u2_events = table();

scripts/base/frameworks/intel/main.bro

@@ -27,6 +27,9 @@ export {
 		## File hash which is non-hash type specific.  It's up to the user to query
 		## for any relevant hash types.
 		FILE_HASH,
+		## File names.  Typically with protocols with definite indications
+		## of a file name.
+		FILE_NAME,
 		## Certificate SHA-1 hash.
 		CERT_HASH,
 	};
@@ -80,6 +83,10 @@ export {
 		## If the data was discovered within a connection, the 
 		## connection record should go into get to give context to the data.
 		conn:            connection    &optional;
+
+		## If the data was discovered within a file, the file record
+		## should go here to provide context to the data.
+		f:               fa_file       &optional;
 	};
 
 	## Record used for the logging framework representing a positive
@@ -95,6 +102,16 @@ export {
 		## this is the conn_id for the connection.
 		id:       conn_id        &log &optional;
 
+		## If a file was associated with this intelligence hit,
+		## this is the uid for the file.
+		fuid:           string   &log &optional;
+		## A mime type if the intelligence hit is related to a file.  
+		## If the $f field is provided this will be automatically filled out.
+		file_mime_type: string   &log &optional;
+		## Frequently files can be "described" to give a bit more context.
+		## If the $f field is provided this field will be automatically filled out.
+		file_desc:      string   &log &optional;
+
 		## Where the data was seen.
 		seen:     Seen           &log;
 		## Sources which supplied data that resulted in this match.
@@ -248,7 +265,25 @@ function has_meta(check: MetaData, metas: set[MetaData]): bool
 
 event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
-	local info: Info = [$ts=network_time(), $seen=s];
+	local info = Info($ts=network_time(), $seen=s);
+
+	if ( s?$f )
+		{
+		if ( s$f?$conns && |s$f$conns| == 1 )
+			{
+			for ( cid in s$f$conns )
+				s$conn = s$f$conns[cid];
+			}
+
+		if ( ! info?$fuid )
+			info$fuid = s$f$id;
+
+		if ( ! info?$file_mime_type && s$f?$mime_type )
+			info$file_mime_type = s$f$mime_type;
+
+		if ( ! info?$file_desc )
+			info$file_desc = Files::describe(s$f);
+		}
 
 	if ( s?$conn )
 		{

scripts/policy/frameworks/intel/seen/__load__.bro

@@ -1,8 +1,9 @@
 @load ./conn-established
 @load ./dns
-@load ./http-host-header
+@load ./file-hashes
+@load ./file-names
+@load ./http-headers
 @load ./http-url
-@load ./http-user-agents
 @load ./ssl
 @load ./smtp
 @load ./smtp-url-extraction

scripts/policy/frameworks/intel/seen/file-hashes.bro

@@ -0,0 +1,12 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event file_hash(f: fa_file, kind: string, hash: string)
+	{
+	local seen = Intel::Seen($indicator=hash,
+	                         $indicator_type=Intel::FILE_HASH,
+	                         $f=f,
+	                         $where=Files::IN_HASH);
+	
+	Intel::seen(seen);
+	}

scripts/policy/frameworks/intel/seen/file-names.bro

@@ -0,0 +1,11 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event file_new(f: fa_file)
+	{
+	if ( f?$info && f$info?$filename )
+		Intel::seen([$indicator=f$info$filename,
+		             $indicator_type=Intel::FILE_NAME,
+		             $f=f,
+		             $where=Files::IN_NAME]);
+	}

scripts/policy/frameworks/intel/seen/http-headers.bro

@@ -0,0 +1,46 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event http_header(c: connection, is_orig: bool, name: string, value: string)
+	{
+	if ( is_orig )
+		{
+		switch ( name ) 
+			{
+			case "HOST": 
+			Intel::seen([$indicator=value,
+			             $indicator_type=Intel::DOMAIN,
+			             $conn=c,
+			             $where=HTTP::IN_HOST_HEADER]);
+			break;
+
+			case "REFERER":
+			Intel::seen([$indicator=sub(value, /^.*:\/\//, ""),
+			             $indicator_type=Intel::URL,
+			             $conn=c,
+			             $where=HTTP::IN_REFERRER_HEADER]);
+			break;
+
+			case "X-FORWARDED-FOR":
+			if ( is_valid_ip(value) )
+				{
+				local addrs = find_ip_addresses(value);
+				for ( i in addrs )
+					{
+					Intel::seen([$host=to_addr(addrs[i]),
+					             $indicator_type=Intel::ADDR,
+					             $conn=c,
+					             $where=HTTP::IN_X_FORWARDED_FOR_HEADER]);
+					}
+				}
+			break;
+
+			case "USER-AGENT":
+			Intel::seen([$indicator=value,
+			             $indicator_type=Intel::SOFTWARE,
+			             $conn=c,
+			             $where=HTTP::IN_USER_AGENT_HEADER]);
+			break;
+			}
+		}
+	}

scripts/policy/frameworks/intel/seen/http-host-header.bro

@@ -1,11 +0,0 @@
-@load base/frameworks/intel
-@load ./where-locations
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( is_orig && name == "HOST" )
-		Intel::seen([$indicator=value,
-		             $indicator_type=Intel::DOMAIN,
-		             $conn=c,
-		             $where=HTTP::IN_HOST_HEADER]);
-	}

scripts/policy/frameworks/intel/seen/http-user-agents.bro

@@ -1,12 +0,0 @@
-@load base/frameworks/intel
-@load ./where-locations
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( is_orig && name == "USER-AGENT" )
-		Intel::seen([$indicator=value,
-		             $indicator_type=Intel::SOFTWARE,
-		             $conn=c,
-		             $where=HTTP::IN_USER_AGENT_HEADER]);
-	}
-

scripts/policy/frameworks/intel/seen/where-locations.bro

@@ -4,10 +4,14 @@ export {
 	redef enum Intel::Where += {
 		Conn::IN_ORIG,
 		Conn::IN_RESP,
+		Files::IN_HASH,
+		Files::IN_NAME,
 		DNS::IN_REQUEST,
 		DNS::IN_RESPONSE,
 		HTTP::IN_HOST_HEADER,
+		HTTP::IN_REFERRER_HEADER,
 		HTTP::IN_USER_AGENT_HEADER,
+		HTTP::IN_X_FORWARDED_FOR_HEADER,
 		HTTP::IN_URL,
 		SMTP::IN_MAIL_FROM,
 		SMTP::IN_RCPT_TO,

scripts/test-all-policy.bro

@@ -18,9 +18,10 @@
 @load frameworks/intel/seen/__load__.bro
 @load frameworks/intel/seen/conn-established.bro
 @load frameworks/intel/seen/dns.bro
-@load frameworks/intel/seen/http-host-header.bro
+@load frameworks/intel/seen/file-hashes.bro
+@load frameworks/intel/seen/file-names.bro
+@load frameworks/intel/seen/http-headers.bro
 @load frameworks/intel/seen/http-url.bro
-@load frameworks/intel/seen/http-user-agents.bro
 @load frameworks/intel/seen/smtp-url-extraction.bro
 @load frameworks/intel/seen/smtp.bro
 @load frameworks/intel/seen/ssl.bro