Intelligence framework checkpoint

- Basic API seems to works, but tests aren't updated yet.

 - Several scripts are available in policy/frameworks/intel that
   call the "seen" function to provide data into the intel
   framework to be tested.

 - Intel::policy is not done yet and needs to be discussed to
   figure out what it needs to have.

 - Running the intel framework and having it do something finally
   is really cool!

scripts/base/frameworks/intel/cluster.bro

@@ -21,14 +21,14 @@ redef record Item += {
 };
 
 # Primary intelligence distribution comes from manager.
-redef Cluster::manager2worker_events += /Intel::cluster_(new|updated)_item/;
+redef Cluster::manager2worker_events += /^Intel::cluster_.*/;
 # If a worker finds intelligence and adds it, it should share it back to the manager.
-redef Cluster::worker2manager_events += /Intel::(match_in_.*_no_items|cluster_(new|updated)_item)/;
+redef Cluster::worker2manager_events += /^Intel::(cluster_.*|match_no_items)/;
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
-event Intel::match_in_conn_no_items(c: connection, seen: Seen) &priority=5
+event Intel::match_no_items(s: Seen) &priority=5
 	{
-	event Intel::match_in_conn(c, seen, Intel::get_items(seen));
+	event Intel::match(c, s, Intel::get_items(s));
 	}
 @endif
 

scripts/base/frameworks/intel/http-user-agents.bro

@@ -1,67 +0,0 @@
-
-@load base/protocols/http
-@load base/frameworks/intel
-
-module HTTP;
-
-export {
-	redef enum Intel::Where += { 
-		HTTP::IN_HEADER,
-		HTTP::IN_REQUEST,
-		HTTP::IN_HOST_HEADER,
-		HTTP::IN_CONN_EST,
-		HTTP::IN_DNS_REQUEST,
-	};
-}
-
-event connection_established(c: connection)
-	{
-	Intel::found_in_conn(c, [$host=c$id$orig_h, $where=IN_CONN_EST]);
-	Intel::found_in_conn(c, [$host=c$id$resp_h, $where=IN_CONN_EST]);
-	}
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( is_orig && name == "USER-AGENT" )
-		Intel::found_in_conn(c, [$str=value,
-		                         $str_type=Intel::USER_AGENT,
-		                         $where=IN_HEADER]);
-
-	if ( is_orig && name == "HOST" )
-		Intel::found_in_conn(c, [$str=value,
-		                         $str_type=Intel::DOMAIN,
-		                         $where=IN_HOST_HEADER]);
-	}
-
-event http_message_done(c: connection, is_orig: bool, stat: http_message_stat)
-	{
-	if ( c?$http )
-		{
-		if ( c$http?$user_agent )
-			Intel::found_in_conn(c, [$str=c$http$user_agent, 
-			                         $str_type=Intel::USER_AGENT, 
-			                         $where=IN_HEADER]);
-
-		Intel::found_in_conn(c, [$str=HTTP::build_url(c$http),
-		                         $str_type=Intel::URL,
-		                         $where=IN_REQUEST]);
-		}
-	}
-
-
-event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count)
-	{
-	Intel::found_in_conn(c, [$str=query,
-	                         $str_type=Intel::DOMAIN,
-	                         $where=IN_DNS_REQUEST]);
-
-	}
-
-event Intel::match_in_conn(c: connection, found: Intel::Found, items: set[Intel::Item])
-	{
-	print "matched one!";
-	for ( i in items )
-		{
-		print "    " + i$meta$desc;
-		}
-	}

scripts/base/frameworks/intel/main.bro

@@ -11,8 +11,7 @@ export {
 	redef enum Log::ID += { LOG };
 	
 	redef enum Notice::Type += {
-		## This notice should be used in all detector scripts to indicate 
+		## Notice type to indicate an intelligence hit.
-		## an intelligence based detection.
 		Detection,
 	};
 	
@@ -64,29 +63,42 @@ export {
 	};
 
 	type Seen: record {
-		host:      addr          &optional;
+		host:      addr          &optional &log;
-		str:       string        &optional;
+		str:       string        &optional &log;
-		str_type:  StrType       &optional;
+		str_type:  StrType       &optional &log;
 
-		where:     Where;
+		where:     Where         &log;
+		
+		conn:      connection    &optional;
+	};
+
+	type Info: record {
+		ts:   time    &log;
+
+		uid:  string  &log &optional;
+		id:   conn_id &log &optional;
+
+		seen: Seen    &log;
 	};
 
 	type PolicyItem: record {
-		pred:   function(seen: Seen, item: Item): bool &optional;
+		pred:   function(s: Seen, item: Item): bool &optional;
 
 		log_it: bool &default=T;
 	};
 
 	## Intelligence data manipulation functions.
 	global insert:      function(item: Item);
-	global delete_item: function(item: Item): bool;
 
 	## Function to declare discovery of a piece of data in order to check
 	## it against known intelligence for matches.
-	global seen_in_conn:  function(c: connection, seen: Seen);
+	global seen: function(s: Seen);
 
 	## Intelligence policy variable for handling matches.
-	const policy: set[PolicyItem] = {} &redef;
+	const policy: set[PolicyItem] = {
+	#	[$pred(s: Seen) = { return T; },
+	#	 $action=Intel::ACTION_LOG]
+	} &redef;
 
 	## API Events that indicate when various things happen internally within the 
 	## intelligence framework.
@@ -94,34 +106,40 @@ export {
 	global updated_item: event(item: Item);
 }
 
-## Event to represent a match happening in a connection.  On clusters there
+# Event to represent a match happening in a connection.  On clusters there
-## is no assurance as to where this event will be generated so don't 
+# is no assurance as to where this event will be generated so don't 
-## assume that arbitrary global state beyond the given data
+# assume that arbitrary global state beyond the given data
-## will be available.
+# will be available.
-global match_in_conn: event(c: connection, seen: Seen, items: set[Item]);
+global match: event(s: Seen, items: set[Item]);
 
 # Internal handler for conn oriented matches with no metadata based on the have_full_data setting.
-global match_in_conn_no_items: event(c: connection, seen: Seen);
+global match_no_items: event(s: Seen);
 
-## Optionally store metadata.  This is used internally depending on
+# Optionally store metadata.  This is used internally depending on
-## if this is a cluster deployment or not.
+# if this is a cluster deployment or not.
 const have_full_data = T &redef;
 
+# The in memory data structure for holding intelligence.
 type DataStore: record {
 	net_data:    table[subnet] of set[MetaData];
 	string_data: table[string, StrType] of set[MetaData];
 };
 global data_store: DataStore;
 
-function find(seen: Seen): bool
+event bro_init() &priority=5
 	{
-	if ( seen?$host && 
+	Log::create_stream(LOG, [$columns=Info]);
-	     seen$host in data_store$net_data )
+	}
+
+function find(s: Seen): bool
+	{
+	if ( s?$host && 
+	     s$host in data_store$net_data )
 		{
 		return T;
 		}
-	else if ( seen?$str && seen?$str_type &&
+	else if ( s?$str && s?$str_type &&
-	          [seen$str, seen$str_type] in data_store$string_data )
+	          [s$str, s$str_type] in data_store$string_data )
 		{
 		return T;
 		}
@@ -131,7 +149,7 @@ function find(seen: Seen): bool
 		}
 	}
 
-function get_items(seen: Seen): set[Item]
+function get_items(s: Seen): set[Item]
 	{
 	local item: Item;
 	local return_data: set[Item] = set();
@@ -144,28 +162,28 @@ function get_items(seen: Seen): set[Item]
 		return return_data;
 		}
 
-	if ( seen?$host )
+	if ( s?$host )
 		{
 		# See if the host is known about and it has meta values
-		if ( seen$host in data_store$net_data )
+		if ( s$host in data_store$net_data )
 			{
-			for ( m in data_store$net_data[seen$host] )
+			for ( m in data_store$net_data[s$host] )
 				{
 				# TODO: the lookup should be finding all and not just most specific
 				#       and $host/$net should have the correct value.
-				item = [$host=seen$host, $meta=m];
+				item = [$host=s$host, $meta=m];
 				add return_data[item];
 				}
 			}
 		}
-	else if ( seen?$str && seen?$str_type )
+	else if ( s?$str && s?$str_type )
 		{
 		# See if the string is known about and it has meta values
-		if ( [seen$str, seen$str_type] in data_store$string_data )
+		if ( [s$str, s$str_type] in data_store$string_data )
 			{
-			for ( m in data_store$string_data[seen$str, seen$str_type] )
+			for ( m in data_store$string_data[s$str, s$str_type] )
 				{
-				item = [$str=seen$str, $str_type=seen$str_type, $meta=m];
+				item = [$str=s$str, $str_type=s$str_type, $meta=m];
 				add return_data[item];
 				}
 			}
@@ -174,18 +192,31 @@ function get_items(seen: Seen): set[Item]
 	return return_data;
 	}
 
-function Intel::seen_in_conn(c: connection, seen: Seen)
+event Intel::match(s: Seen, items: set[Item])
 	{
-	if ( find(seen) )
+	local info: Info = [$ts=network_time(), $seen=s];
+
+	if ( s?$conn )
+		{
+		info$uid = s$conn$uid;
+		info$id  = s$conn$id;
+		}
+
+	Log::write(Intel::LOG, info);
+	}
+
+function Intel::seen(s: Seen)
+	{
+	if ( find(s) )
 		{
 		if ( have_full_data )
 			{
-			local items = get_items(seen);
+			local items = get_items(s);
-			event Intel::match_in_conn(c, seen, items);
+			event Intel::match(s, items);
 			}
 		else
 			{
-			event Intel::match_in_conn_no_items(c, seen);
+			event Intel::match_no_items(s);
 			}
 		}
 	}

scripts/policy/frameworks/intel/__load__.bro

@@ -0,0 +1,6 @@
+@load ./conn-established
+@load ./dns
+@load ./http-host-header
+@load ./http-url
+@load ./http-user-agent
+@load ./ssl

scripts/policy/frameworks/intel/conn-established.bro

@@ -0,0 +1,14 @@
+@load base/frameworks/intel
+
+export {
+	redef enum Intel::Where += {
+		Conn::IN_ORIG,
+		Conn::IN_RESP,
+	};
+}
+
+event connection_established(c: connection)
+	{
+	Intel::seen([$host=c$id$orig_h, $conn=c, $where=Conn::IN_ORIG]);
+	Intel::seen([$host=c$id$resp_h, $conn=c, $where=Conn::IN_RESP]);
+	}

scripts/policy/frameworks/intel/dns.bro

@@ -0,0 +1,16 @@
+@load base/frameworks/intel
+
+export {
+	redef enum Intel::Where += {
+		DNS::IN_REQUEST,
+		DNS::IN_RESPONSE,
+	};
+}
+
+event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count)
+	{
+	Intel::seen([$str=query,
+	             $str_type=Intel::DOMAIN,
+	             $conn=c,
+	             $where=DNS::IN_REQUEST]);
+	}

scripts/policy/frameworks/intel/http-host-header.bro

@@ -0,0 +1,16 @@
+@load base/frameworks/intel
+
+export {
+	redef enum Intel::Where += {
+		HTTP::IN_HOST_HEADER,
+	};
+}
+
+event http_header(c: connection, is_orig: bool, name: string, value: string)
+	{
+	if ( is_orig && name == "HOST" )
+		Intel::seen([$str=value,
+		             $str_type=Intel::DOMAIN,
+		             $conn=c,
+		             $where=HTTP::IN_HOST_HEADER]);
+	}

scripts/policy/frameworks/intel/http-url.bro

@@ -0,0 +1,16 @@
+@load base/frameworks/intel
+
+export {
+	redef enum Intel::Where += {
+		HTTP::IN_URL,
+	};
+}
+
+event http_message_done(c: connection, is_orig: bool, stat: http_message_stat)
+	{
+	if ( is_orig && c?$http )
+		Intel::seen([$str=HTTP::build_url(c$http),
+		             $str_type=Intel::URL,
+		             $conn=c,
+		             $where=HTTP::IN_URL]);
+	}

scripts/policy/frameworks/intel/http-user-agent.bro

@@ -0,0 +1,16 @@
+@load base/frameworks/intel
+
+export {
+	redef enum Intel::Where += {
+		HTTP::IN_USER_AGENT_HEADER,
+	};
+}
+
+event http_header(c: connection, is_orig: bool, name: string, value: string)
+	{
+	if ( is_orig && name == "USER-AGENT" )
+		Intel::seen([$str=value,
+		             $str_type=Intel::USER_AGENT,
+		             $conn=c,
+		             $where=HTTP::IN_USER_AGENT_HEADER]);
+	}

scripts/policy/frameworks/intel/ssl.bro

@@ -0,0 +1,41 @@
+@load base/frameworks/intel
+
+export {
+	redef enum Intel::Where += {
+		SSL::IN_SERVER_CERT,
+		SSL::IN_CLIENT_CERT,
+		SSL::IN_SERVER_NAME,
+	};
+}
+
+
+event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string)
+	{
+	if ( chain_idx == 0 )
+		{
+		if ( /emailAddress=/ in cert$subject )
+			{
+			local email = sub(cert$subject, /^.*emailAddress=/, "");
+			email = sub(email, /,.*$/, "");
+			Intel::seen([$str=email,
+			             $str_type=Intel::EMAIL,
+			             $conn=c,
+			             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
+			}
+
+		Intel::seen([$str=sha1_hash(der_cert),
+		             $str_type=Intel::CERT_HASH,
+		             $conn=c,
+		             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
+		}
+	}
+
+event ssl_extension(c: connection, is_orig: bool, code: count, val: string)
+	{
+	if ( is_orig && SSL::extensions[code] == "server_name" && 
+	     c?$ssl && c$ssl?$server_name )
+		Intel::seen([$str=c$ssl$server_name,
+		             $str_type=Intel::DOMAIN,
+		             $conn=c,
+		             $where=SSL::IN_SERVER_NAME]);
+	}