Add file support to intel framework and slightly restructure intel http handling.

scripts/base/frameworks/intel/main.bro

@@ -80,6 +80,10 @@ export {
 		## If the data was discovered within a connection, the 
 		## connection record should go into get to give context to the data.
 		conn:            connection    &optional;
+
+		## If the data was discovered within a file, the file record
+		## should go here to provide context to the data.
+		f:               fa_file       &optional;
 	};
 
 	## Record used for the logging framework representing a positive
@@ -95,6 +99,16 @@ export {
 		## this is the conn_id for the connection.
 		id:       conn_id        &log &optional;
 
+		## If a file was associated with this intelligence hit,
+		## this is the uid for the file.
+		fuid:           string          &log &optional;
+		## A mime type if the intelligence hit is related to a file.  
+		## If the $f field is provided this will be automatically filled out.
+		file_mime_type: string          &log &optional;
+		## Frequently files can be "described" to give a bit more context.
+		## If the $f field is provided this field will be automatically filled out.
+		file_desc:      string          &log &optional;
+
 		## Where the data was seen.
 		seen:     Seen           &log;
 		## Sources which supplied data that resulted in this match.
@@ -248,7 +262,22 @@ function has_meta(check: MetaData, metas: set[MetaData]): bool
 
 event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
-	local info: Info = [$ts=network_time(), $seen=s];
+	if ( s$f?$conns && |s$f$conns| == 1 )
+		{
+		for ( cid in s$f$conns )
+			s$conn = s$f$conns[cid];
+		}
+
+	local info = Info($ts=network_time(), $seen=s);
+
+	if ( ! info?$fuid )
+		info$fuid = s$f$id;
+
+	if ( ! info?$file_mime_type && s$f?$mime_type )
+		info$file_mime_type = s$f$mime_type;
+
+	if ( ! info?$file_desc )
+		info$file_desc = Files::describe(s$f);
 
 	if ( s?$conn )
 		{

scripts/policy/frameworks/intel/seen/__load__.bro

@@ -1,8 +1,8 @@
 @load ./conn-established
 @load ./dns
-@load ./http-host-header
+@load ./file-hashes
+@load ./http-headers
 @load ./http-url
-@load ./http-user-agents
 @load ./ssl
 @load ./smtp
 @load ./smtp-url-extraction

scripts/policy/frameworks/intel/seen/file-hashes.bro

@@ -0,0 +1,12 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event file_hash(f: fa_file , kind: string , hash: string)
+	{
+	local seen = Intel::Seen($indicator=hash,
+	                         $indicator_type=Intel::FILE_HASH,
+	                         $f=f,
+	                         $where=Files::IN_HASH);
+	
+	Intel::seen(seen);
+	}

scripts/policy/frameworks/intel/seen/http-headers.bro

@@ -0,0 +1,46 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event http_header(c: connection, is_orig: bool, name: string, value: string)
+	{
+	if ( is_orig )
+		{
+		switch ( name ) 
+			{
+			case "HOST": 
+			Intel::seen([$indicator=value,
+			             $indicator_type=Intel::DOMAIN,
+			             $conn=c,
+			             $where=HTTP::IN_HOST_HEADER]);
+			break;
+
+			case "REFERER":
+			Intel::seen([$indicator=sub(value, /^.*:\/\//, ""),
+			             $indicator_type=Intel::URL,
+			             $conn=c,
+			             $where=HTTP::IN_REFERRER_HEADER]);
+			break;
+
+			case "X-FORWARDED-FOR":
+			if ( is_valid_ip(value) )
+				{
+				local addrs = find_ip_addresses(value);
+				for ( i in addrs )
+					{
+					Intel::seen([$host=to_addr(addrs[i]),
+					             $indicator_type=Intel::ADDR,
+					             $conn=c,
+					             $where=HTTP::IN_X_FORWARDED_FOR_HEADER]);
+					}
+				}
+			break;
+
+			case "USER-AGENT":
+			Intel::seen([$indicator=value,
+			             $indicator_type=Intel::SOFTWARE,
+			             $conn=c,
+			             $where=HTTP::IN_USER_AGENT_HEADER]);
+			break;
+			}
+		}
+	}

scripts/policy/frameworks/intel/seen/http-host-header.bro

@@ -1,11 +0,0 @@
-@load base/frameworks/intel
-@load ./where-locations
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( is_orig && name == "HOST" )
-		Intel::seen([$indicator=value,
-		             $indicator_type=Intel::DOMAIN,
-		             $conn=c,
-		             $where=HTTP::IN_HOST_HEADER]);
-	}

scripts/policy/frameworks/intel/seen/http-user-agents.bro

@@ -1,12 +0,0 @@
-@load base/frameworks/intel
-@load ./where-locations
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( is_orig && name == "USER-AGENT" )
-		Intel::seen([$indicator=value,
-		             $indicator_type=Intel::SOFTWARE,
-		             $conn=c,
-		             $where=HTTP::IN_USER_AGENT_HEADER]);
-	}
-

scripts/policy/frameworks/intel/seen/where-locations.bro

@@ -4,10 +4,13 @@ export {
 	redef enum Intel::Where += {
 		Conn::IN_ORIG,
 		Conn::IN_RESP,
+		Files::IN_HASH,
 		DNS::IN_REQUEST,
 		DNS::IN_RESPONSE,
 		HTTP::IN_HOST_HEADER,
+		HTTP::IN_REFERRER_HEADER,
 		HTTP::IN_USER_AGENT_HEADER,
+		HTTP::IN_X_FORWARDED_FOR_HEADER,
 		HTTP::IN_URL,
 		SMTP::IN_MAIL_FROM,
 		SMTP::IN_RCPT_TO,