Merge remote-tracking branch 'origin/topic/seth/intel-updates'

* origin/topic/seth/intel-updates:
  Fixing intel framework tests.
  Add file name support to intel framework.
  Add file support to intel framework and slightly restructure intel http handling.

CHANGES

@@ -1,4 +1,11 @@
 
+2.1-1059 | 2013-08-13 23:52:41 -0400
+
+  * Add file name support to intel framework. (Seth Hall)
+
+  * Add file support to intel framework and slightly restructure 
+    intel http handling. (Seth Hall)
+
 2.1-1052 | 2013-08-12 14:38:14 -0700
 
   * Fixing bug in DNP3 analyzer flagged by compiler warning. (Robin

VERSION

@@ -1 +1 @@
-2.1-1052
+2.1-1059

doc/scripts/DocSourcesList.cmake

@@ -199,9 +199,10 @@ rest_target(${psd} policy/frameworks/files/hash-all-files.bro)
 rest_target(${psd} policy/frameworks/intel/do_notice.bro)
 rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro)
 rest_target(${psd} policy/frameworks/intel/seen/dns.bro)
-rest_target(${psd} policy/frameworks/intel/seen/http-host-header.bro)
+rest_target(${psd} policy/frameworks/intel/seen/file-hashes.bro)
+rest_target(${psd} policy/frameworks/intel/seen/file-names.bro)
+rest_target(${psd} policy/frameworks/intel/seen/http-headers.bro)
 rest_target(${psd} policy/frameworks/intel/seen/http-url.bro)
-rest_target(${psd} policy/frameworks/intel/seen/http-user-agents.bro)
 rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro)
 rest_target(${psd} policy/frameworks/intel/seen/smtp.bro)
 rest_target(${psd} policy/frameworks/intel/seen/ssl.bro)

scripts/base/frameworks/intel/main.bro

@@ -27,6 +27,9 @@ export {
 		## File hash which is non-hash type specific.  It's up to the user to query
 		## for any relevant hash types.
 		FILE_HASH,
+		## File names.  Typically with protocols with definite indications
+		## of a file name.
+		FILE_NAME,
 		## Certificate SHA-1 hash.
 		CERT_HASH,
 	};
@@ -80,6 +83,10 @@ export {
 		## If the data was discovered within a connection, the 
 		## connection record should go into get to give context to the data.
 		conn:            connection    &optional;
+
+		## If the data was discovered within a file, the file record
+		## should go here to provide context to the data.
+		f:               fa_file       &optional;
 	};
 
 	## Record used for the logging framework representing a positive
@@ -95,6 +102,16 @@ export {
 		## this is the conn_id for the connection.
 		id:       conn_id        &log &optional;
 
+		## If a file was associated with this intelligence hit,
+		## this is the uid for the file.
+		fuid:           string   &log &optional;
+		## A mime type if the intelligence hit is related to a file.  
+		## If the $f field is provided this will be automatically filled out.
+		file_mime_type: string   &log &optional;
+		## Frequently files can be "described" to give a bit more context.
+		## If the $f field is provided this field will be automatically filled out.
+		file_desc:      string   &log &optional;
+
 		## Where the data was seen.
 		seen:     Seen           &log;
 		## Sources which supplied data that resulted in this match.
@@ -248,7 +265,25 @@ function has_meta(check: MetaData, metas: set[MetaData]): bool
 
 event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
-	local info: Info = [$ts=network_time(), $seen=s];
+	local info = Info($ts=network_time(), $seen=s);
+
+	if ( s?$f )
+		{
+		if ( s$f?$conns && |s$f$conns| == 1 )
+			{
+			for ( cid in s$f$conns )
+				s$conn = s$f$conns[cid];
+			}
+
+		if ( ! info?$fuid )
+			info$fuid = s$f$id;
+
+		if ( ! info?$file_mime_type && s$f?$mime_type )
+			info$file_mime_type = s$f$mime_type;
+
+		if ( ! info?$file_desc )
+			info$file_desc = Files::describe(s$f);
+		}
 
 	if ( s?$conn )
 		{

scripts/policy/frameworks/intel/seen/__load__.bro

@@ -1,8 +1,9 @@
 @load ./conn-established
 @load ./dns
-@load ./http-host-header
+@load ./file-hashes
+@load ./file-names
+@load ./http-headers
 @load ./http-url
-@load ./http-user-agents
 @load ./ssl
 @load ./smtp
 @load ./smtp-url-extraction

scripts/policy/frameworks/intel/seen/file-hashes.bro

@@ -0,0 +1,12 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event file_hash(f: fa_file, kind: string, hash: string)
+	{
+	local seen = Intel::Seen($indicator=hash,
+	                         $indicator_type=Intel::FILE_HASH,
+	                         $f=f,
+	                         $where=Files::IN_HASH);
+	
+	Intel::seen(seen);
+	}

scripts/policy/frameworks/intel/seen/file-names.bro

@@ -0,0 +1,11 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event file_new(f: fa_file)
+	{
+	if ( f?$info && f$info?$filename )
+		Intel::seen([$indicator=f$info$filename,
+		             $indicator_type=Intel::FILE_NAME,
+		             $f=f,
+		             $where=Files::IN_NAME]);
+	}

scripts/policy/frameworks/intel/seen/http-headers.bro

@@ -0,0 +1,46 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event http_header(c: connection, is_orig: bool, name: string, value: string)
+	{
+	if ( is_orig )
+		{
+		switch ( name ) 
+			{
+			case "HOST": 
+			Intel::seen([$indicator=value,
+			             $indicator_type=Intel::DOMAIN,
+			             $conn=c,
+			             $where=HTTP::IN_HOST_HEADER]);
+			break;
+
+			case "REFERER":
+			Intel::seen([$indicator=sub(value, /^.*:\/\//, ""),
+			             $indicator_type=Intel::URL,
+			             $conn=c,
+			             $where=HTTP::IN_REFERRER_HEADER]);
+			break;
+
+			case "X-FORWARDED-FOR":
+			if ( is_valid_ip(value) )
+				{
+				local addrs = find_ip_addresses(value);
+				for ( i in addrs )
+					{
+					Intel::seen([$host=to_addr(addrs[i]),
+					             $indicator_type=Intel::ADDR,
+					             $conn=c,
+					             $where=HTTP::IN_X_FORWARDED_FOR_HEADER]);
+					}
+				}
+			break;
+
+			case "USER-AGENT":
+			Intel::seen([$indicator=value,
+			             $indicator_type=Intel::SOFTWARE,
+			             $conn=c,
+			             $where=HTTP::IN_USER_AGENT_HEADER]);
+			break;
+			}
+		}
+	}

scripts/policy/frameworks/intel/seen/http-host-header.bro

@@ -1,11 +0,0 @@
-@load base/frameworks/intel
-@load ./where-locations
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( is_orig && name == "HOST" )
-		Intel::seen([$indicator=value,
-		             $indicator_type=Intel::DOMAIN,
-		             $conn=c,
-		             $where=HTTP::IN_HOST_HEADER]);
-	}

scripts/policy/frameworks/intel/seen/http-user-agents.bro

@@ -1,12 +0,0 @@
-@load base/frameworks/intel
-@load ./where-locations
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( is_orig && name == "USER-AGENT" )
-		Intel::seen([$indicator=value,
-		             $indicator_type=Intel::SOFTWARE,
-		             $conn=c,
-		             $where=HTTP::IN_USER_AGENT_HEADER]);
-	}
-

scripts/policy/frameworks/intel/seen/where-locations.bro

@@ -4,10 +4,14 @@ export {
 	redef enum Intel::Where += {
 		Conn::IN_ORIG,
 		Conn::IN_RESP,
+		Files::IN_HASH,
+		Files::IN_NAME,
 		DNS::IN_REQUEST,
 		DNS::IN_RESPONSE,
 		HTTP::IN_HOST_HEADER,
+		HTTP::IN_REFERRER_HEADER,
 		HTTP::IN_USER_AGENT_HEADER,
+		HTTP::IN_X_FORWARDED_FOR_HEADER,
 		HTTP::IN_URL,
 		SMTP::IN_MAIL_FROM,
 		SMTP::IN_RCPT_TO,

scripts/test-all-policy.bro

@@ -18,9 +18,10 @@
 @load frameworks/intel/seen/__load__.bro
 @load frameworks/intel/seen/conn-established.bro
 @load frameworks/intel/seen/dns.bro
-@load frameworks/intel/seen/http-host-header.bro
+@load frameworks/intel/seen/file-hashes.bro
+@load frameworks/intel/seen/file-names.bro
+@load frameworks/intel/seen/http-headers.bro
 @load frameworks/intel/seen/http-url.bro
-@load frameworks/intel/seen/http-user-agents.bro
 @load frameworks/intel/seen/smtp-url-extraction.bro
 @load frameworks/intel/seen/smtp.bro
 @load frameworks/intel/seen/ssl.bro

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2013-07-19-17-05-48
+#open	2013-08-14-03-46-32
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	sources
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	sources
-#types	time	string	addr	port	addr	port	string	enum	enum	table[string]
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	table[string]
-1374253548.038580	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-1
+1376451992.872806	-	-	-	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-1
-#close	2013-07-19-17-05-57
+#close	2013-08-14-03-46-42

testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2013-07-19-17-04-26
+#open	2013-08-14-03-47-03
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	sources
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	sources
-#types	time	string	addr	port	addr	port	string	enum	enum	table[string]
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	table[string]
-1374253466.857185	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	source1
+1376452023.137179	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	source1
-1374253466.857185	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	source1
+1376452023.137179	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	source1
-#close	2013-07-19-17-04-26
+#close	2013-08-14-03-47-03

testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log

@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2013-07-19-17-06-57
+#open	2013-08-14-03-47-23
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	sources
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	sources
-#types	time	string	addr	port	addr	port	string	enum	enum	table[string]
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	table[string]
-1374253617.312158	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	source1
+1376452043.835810	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	source1
-1374253617.312158	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	source1
+1376452043.835810	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	source1
-1374253618.332565	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	source1
+1376452044.855238	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	source1
-1374253618.332565	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	source1
+1376452044.855238	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	source1
-#close	2013-07-19-17-07-06
+#close	2013-08-14-03-47-32