Merge branch 'master' into topic/jsiwek/faf-updates

Conflicts:
	testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

scripts/policy/frameworks/intel/conn-established.bro

@@ -1,8 +0,0 @@
-@load base/frameworks/intel
-@load ./where-locations
-
-event connection_established(c: connection)
-	{
-	Intel::seen([$host=c$id$orig_h, $conn=c, $where=Conn::IN_ORIG]);
-	Intel::seen([$host=c$id$resp_h, $conn=c, $where=Conn::IN_RESP]);
-	}

scripts/policy/frameworks/intel/do_notice.bro

@@ -0,0 +1,44 @@
+
+@load base/frameworks/intel
+@load base/frameworks/notice
+
+module Intel;
+
+export {
+	redef enum Notice::Type += {
+		## Intel::Notice is a notice that happens when an intelligence 
+		## indicator is denoted to be notice-worthy.
+		Intel::Notice
+	};
+
+	redef record Intel::MetaData += {
+		## A boolean value to allow the data itself to represent
+		## if the indicator that this metadata is attached to 
+		## is notice worthy.
+		do_notice: bool &default=F;
+
+		## Restrictions on when notices are created to only create
+		## them if the do_notice field is T and the notice was
+		## seen in the indicated location.
+		if_in: Intel::Where &optional;
+	};
+}
+
+event Intel::match(s: Seen, items: set[Item])
+	{
+	for ( item in items )
+		{
+		if ( item$meta$do_notice &&
+		     (! item$meta?$if_in || s$where == item$meta$if_in) )
+			{
+			local n = Notice::Info($note=Intel::Notice,
+			                       $msg=fmt("Intel hit on %s at %s", s$indicator, s$where),
+			                       $sub=s$indicator);
+
+			if ( s?$conn )
+				n$conn = s$conn;
+
+			NOTICE(n);
+			}
+		}
+	}

scripts/policy/frameworks/intel/seen/conn-established.bro

@@ -0,0 +1,12 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event connection_established(c: connection)
+	{
+	if ( c$orig$state == TCP_ESTABLISHED &&
+	     c$resp$state == TCP_ESTABLISHED )
+		{
+		Intel::seen([$host=c$id$orig_h, $conn=c, $where=Conn::IN_ORIG]);
+		Intel::seen([$host=c$id$resp_h, $conn=c, $where=Conn::IN_RESP]);
+		}
+	}

scripts/policy/frameworks/intel/seen/dns.bro

@@ -3,8 +3,8 @@
 
 event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count)
 	{
-	Intel::seen([$str=query,
-	             $str_type=Intel::DOMAIN,
+	Intel::seen([$indicator=query,
+	             $indicator_type=Intel::DOMAIN,
 	             $conn=c,
 	             $where=DNS::IN_REQUEST]);
 	}

scripts/policy/frameworks/intel/seen/http-host-header.bro

@@ -4,8 +4,8 @@
 event http_header(c: connection, is_orig: bool, name: string, value: string)
 	{
 	if ( is_orig && name == "HOST" )
-		Intel::seen([$str=value,
-		             $str_type=Intel::DOMAIN,
+		Intel::seen([$indicator=value,
+		             $indicator_type=Intel::DOMAIN,
 		             $conn=c,
 		             $where=HTTP::IN_HOST_HEADER]);
 	}

scripts/policy/frameworks/intel/seen/http-url.bro

@@ -5,8 +5,8 @@
 event http_message_done(c: connection, is_orig: bool, stat: http_message_stat)
 	{
 	if ( is_orig && c?$http )
-		Intel::seen([$str=HTTP::build_url(c$http),
-		             $str_type=Intel::URL,
+		Intel::seen([$indicator=HTTP::build_url(c$http),
+		             $indicator_type=Intel::URL,
 		             $conn=c,
 		             $where=HTTP::IN_URL]);
 	}

scripts/policy/frameworks/intel/seen/http-user-agents.bro

@@ -4,8 +4,8 @@
 event http_header(c: connection, is_orig: bool, name: string, value: string)
 	{
 	if ( is_orig && name == "USER-AGENT" )
-		Intel::seen([$str=value,
-		             $str_type=Intel::USER_AGENT,
+		Intel::seen([$indicator=value,
+		             $indicator_type=Intel::SOFTWARE,
 		             $conn=c,
 		             $where=HTTP::IN_USER_AGENT_HEADER]);
 	}

scripts/policy/frameworks/intel/seen/smtp-url-extraction.bro

@@ -5,7 +5,7 @@
 
 event intel_mime_data(f: fa_file, data: string)
 	{
-	if ( ! f?$conns ) 
+	if ( ! f?$conns )
 		return;
 
 	for ( cid in f$conns )
@@ -14,8 +14,8 @@ event intel_mime_data(f: fa_file, data: string)
 		local urls = find_all_urls_without_scheme(data);
 		for ( url in urls )
 			{
-			Intel::seen([$str=url,
-			             $str_type=Intel::URL,
+			Intel::seen([$indicator=url,
+			             $indicator_type=Intel::URL,
 			             $conn=c,
 			             $where=SMTP::IN_MESSAGE]);
 			}

scripts/policy/frameworks/intel/seen/smtp.bro

@@ -0,0 +1,97 @@
+@load base/frameworks/intel
+@load base/protocols/smtp
+@load ./where-locations
+
+event mime_end_entity(c: connection)
+	{
+	if ( c?$smtp )
+		{
+		if ( c$smtp?$path )
+			{
+			local path = c$smtp$path;
+			for ( i in path )
+				{
+				Intel::seen([$host=path[i],
+				             $conn=c,
+				             $where=SMTP::IN_RECEIVED_HEADER]);
+				}
+			}
+
+		if ( c$smtp?$user_agent )
+			Intel::seen([$indicator=c$smtp$user_agent,
+			             $indicator_type=Intel::SOFTWARE,
+			             $conn=c,
+			             $where=SMTP::IN_HEADER]);
+
+		if ( c$smtp?$x_originating_ip )
+			Intel::seen([$host=c$smtp$x_originating_ip,
+			             $conn=c,
+			             $where=SMTP::IN_X_ORIGINATING_IP_HEADER]);
+
+		if ( c$smtp?$mailfrom )
+			{
+			local mailfromparts = split_n(c$smtp$mailfrom, /<.+>/, T, 1);
+			if ( |mailfromparts| > 2 )
+				{
+				Intel::seen([$indicator=mailfromparts[2][1:-2],
+				             $indicator_type=Intel::EMAIL,
+				             $conn=c,
+				             $where=SMTP::IN_MAIL_FROM]);
+				}
+			}
+
+		if ( c$smtp?$rcptto )
+			{
+			for ( rcptto in c$smtp$rcptto )
+				{
+				local rcpttoparts = split_n(rcptto, /<.+>/, T, 1);
+				if ( |rcpttoparts| > 2 )
+					{
+					Intel::seen([$indicator=rcpttoparts[2][1:-2],
+					             $indicator_type=Intel::EMAIL,
+					             $conn=c,
+					             $where=SMTP::IN_RCPT_TO]);
+					}
+				}
+			}
+
+		if ( c$smtp?$from )
+			{
+			local fromparts = split_n(c$smtp$from, /<.+>/, T, 1);
+			if ( |fromparts| > 2 )
+				{
+				Intel::seen([$indicator=fromparts[2][1:-2],
+				             $indicator_type=Intel::EMAIL,
+				             $conn=c,
+				             $where=SMTP::IN_FROM]);
+				}
+			}
+
+		if ( c$smtp?$to )
+			{
+			for ( email_to in c$smtp$to )
+				{
+				local toparts = split_n(email_to, /<.+>/, T, 1);
+				if ( |toparts| > 2 )
+					{
+					Intel::seen([$indicator=toparts[2][1:-2],
+					             $indicator_type=Intel::EMAIL,
+					             $conn=c,
+					             $where=SMTP::IN_TO]);
+					}
+				}
+			}
+
+		if ( c$smtp?$reply_to )
+			{
+			local replytoparts = split_n(c$smtp$reply_to, /<.+>/, T, 1);
+			if ( |replytoparts| > 2 )
+				{
+				Intel::seen([$indicator=replytoparts[2][1:-2],
+				             $indicator_type=Intel::EMAIL,
+				             $conn=c,
+				             $where=SMTP::IN_REPLY_TO]);
+				}
+			}
+		}
+	}

scripts/policy/frameworks/intel/seen/ssl.bro

@@ -10,14 +10,14 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
 			{
 			local email = sub(cert$subject, /^.*emailAddress=/, "");
 			email = sub(email, /,.*$/, "");
-			Intel::seen([$str=email,
-			             $str_type=Intel::EMAIL,
+			Intel::seen([$indicator=email,
+			             $indicator_type=Intel::EMAIL,
 			             $conn=c,
 			             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
 			}
 
-		Intel::seen([$str=sha1_hash(der_cert),
-		             $str_type=Intel::CERT_HASH,
+		Intel::seen([$indicator=sha1_hash(der_cert),
+		             $indicator_type=Intel::CERT_HASH,
 		             $conn=c,
 		             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
 		}
@@ -27,8 +27,8 @@ event ssl_extension(c: connection, is_orig: bool, code: count, val: string)
 	{
 	if ( is_orig && SSL::extensions[code] == "server_name" && 
 	     c?$ssl && c$ssl?$server_name )
-		Intel::seen([$str=c$ssl$server_name,
-		             $str_type=Intel::DOMAIN,
+		Intel::seen([$indicator=c$ssl$server_name,
+		             $indicator_type=Intel::DOMAIN,
 		             $conn=c,
 		             $where=SSL::IN_SERVER_NAME]);
 	}

scripts/policy/frameworks/intel/smtp.bro

@@ -1,71 +0,0 @@
-@load base/frameworks/intel
-@load base/protocols/smtp
-@load ./where-locations
-
-event mime_end_entity(c: connection)
-	{
-	if ( c?$smtp )
-		{
-		if ( c$smtp?$path )
-			{
-			local path = c$smtp$path;
-			for ( i in path )
-				{
-				Intel::seen([$host=path[i],
-				             $conn=c,
-				             $where=SMTP::IN_RECEIVED_HEADER]);
-				}
-			}
-
-		if ( c$smtp?$user_agent )
-			Intel::seen([$str=c$smtp$user_agent,
-			             $str_type=Intel::USER_AGENT,
-			             $conn=c,
-			             $where=SMTP::IN_HEADER]);
-
-		if ( c$smtp?$x_originating_ip )
-			Intel::seen([$host=c$smtp$x_originating_ip,
-			             $conn=c,
-			             $where=SMTP::IN_X_ORIGINATING_IP_HEADER]);
-
-		if ( c$smtp?$mailfrom )
-			Intel::seen([$str=c$smtp$mailfrom,
-			             $str_type=Intel::EMAIL,
-			             $conn=c,
-			             $where=SMTP::IN_MAIL_FROM]);
-
-		if ( c$smtp?$rcptto )
-			{
-			for ( rcptto in c$smtp$rcptto )
-				{
-				Intel::seen([$str=rcptto,
-				             $str_type=Intel::EMAIL,
-				             $conn=c,
-				             $where=SMTP::IN_RCPT_TO]);
-				}
-			}
-
-		if ( c$smtp?$from )
-			Intel::seen([$str=c$smtp$from,
-			             $str_type=Intel::EMAIL,
-			             $conn=c,
-			             $where=SMTP::IN_FROM]);
-
-		if ( c$smtp?$to )
-			{
-			for ( email_to in c$smtp$to )
-				{
-				Intel::seen([$str=email_to,
-				             $str_type=Intel::EMAIL,
-				             $conn=c,
-				             $where=SMTP::IN_TO]);
-				}
-			}
-
-		if ( c$smtp?$reply_to )
-			Intel::seen([$str=c$smtp$reply_to,
-			             $str_type=Intel::EMAIL,
-			             $conn=c,
-			             $where=SMTP::IN_REPLY_TO]);
-		}
-	}