Merge branch 'master' into topic/jsiwek/gtp

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -13,8 +13,12 @@
 ## Turn off remote logging since this is the manager and should only log here.
 redef Log::enable_remote_logging = F;
 
+## Log rotation interval.
 redef Log::default_rotation_interval = 1 hrs;
 
+## Alarm summary mail interval.
+redef Log::default_mail_alarms_interval = 24 hrs;
+
 ## Use the cluster's archive logging script.
 redef Log::default_rotation_postprocessor_cmd = "archive-log";
 

scripts/base/frameworks/intel/__load__.bro

@@ -1 +1,11 @@
-@load ./main
+@load ./main
+
+# The cluster framework must be loaded first.
+@load base/frameworks/cluster
+
+@if ( Cluster::is_enabled() )
+@load ./cluster
+@endif
+
+# This needs cluster support to only read on the manager.
+@load ./input

scripts/base/frameworks/intel/cluster.bro

@@ -0,0 +1,61 @@
+##! Cluster transparency support for the intelligence framework.  This is mostly oriented
+##! toward distributing intelligence information across clusters.
+
+@load base/frameworks/cluster
+@load ./input
+
+module Intel;
+
+redef record Item += {
+	## This field is used internally for cluster transparency to avoid 
+	## re-dispatching intelligence items over and over from workers.
+	first_dispatch: bool &default=T;
+};
+
+# If this process is not a manager process, we don't want the full metadata
+@if ( Cluster::local_node_type() != Cluster::MANAGER )
+redef have_full_data = F;
+@endif
+
+global cluster_new_item: event(item: Item);
+
+# Primary intelligence distribution comes from manager.
+redef Cluster::manager2worker_events += /^Intel::(cluster_new_item)$/;
+# If a worker finds intelligence and adds it, it should share it back to the manager.
+redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|match_no_items)$/;
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+event Intel::match_no_items(s: Seen) &priority=5
+	{
+	event Intel::match(s, Intel::get_items(s));
+	}
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	# When a worker connects, send it the complete minimal data store.
+	# It will be kept up to date after this by the cluster_new_item event.
+	if ( Cluster::nodes[p$descr]$node_type == Cluster::WORKER )
+		{
+		send_id(p, "Intel::min_data_store");
+		}
+	}
+@endif
+
+event Intel::cluster_new_item(item: Intel::Item) &priority=5
+	{
+	# Ignore locally generated events to avoid event storms.
+	if ( is_remote_event() )
+		Intel::insert(item);
+	}
+
+event Intel::new_item(item: Intel::Item) &priority=5
+	{
+	# The cluster manager always rebroadcasts intelligence.
+	# Workers redistribute it if it was locally generated.
+	if ( Cluster::local_node_type() == Cluster::MANAGER ||
+	     item$first_dispatch )
+		{
+		item$first_dispatch=F;
+		event Intel::cluster_new_item(item);
+		}
+	}

scripts/base/frameworks/intel/input.bro

@@ -0,0 +1,33 @@
+@load ./main
+
+module Intel;
+
+export {
+	## Intelligence files that will be read off disk.  The files are
+	## reread everytime they are updated so updates much be atomic with
+	## "mv" instead of writing the file in place.
+	const read_files: set[string] = {} &redef;
+}
+
+event Intel::read_entry(desc: Input::EventDescription, tpe: Input::Event, item: Intel::Item)
+	{
+	Intel::insert(item);
+	}
+
+event bro_init() &priority=5
+	{
+	if ( ! Cluster::is_enabled() ||
+	     Cluster::local_node_type() == Cluster::MANAGER )
+		{
+		for ( a_file in read_files )
+			{
+			Input::add_event([$source=a_file,
+			                  $reader=Input::READER_ASCII,
+			                  $mode=Input::REREAD,
+			                  $name=cat("intel-", a_file),
+			                  $fields=Intel::Item,
+			                  $ev=Intel::read_entry]);
+			}
+		}
+	}
+

scripts/base/frameworks/intel/main.bro

@@ -1,323 +1,345 @@
 ##! The intelligence framework provides a way to store and query IP addresses,
-##! strings (with a subtype), and numeric (with a subtype) data.  Metadata 
-##! also be associated with the intelligence like tags which are arbitrary
-##! strings, time values, and longer descriptive strings.
-
-# Example string subtypes:
-#   url
-#   email
-#   domain
-#   software
-#   user_name
-#   file_name
-#   file_md5
-#   x509_md5
-
-# Example tags: 
-#   infrastructure
-#   malicious
-#   sensitive
-#   canary
-#   friend
+##! and strings (with a str_type).  Metadata can
+##! also be associated with the intelligence like for making more informed
+##! decisions about matching and handling of intelligence.
 
 @load base/frameworks/notice
 
 module Intel;
 
 export {
-	## The intel logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
-	redef enum Notice::Type += {
-		## This notice should be used in all detector scripts to indicate 
-		## an intelligence based detection.
-		Detection,
+	## String data needs to be further categoried since it could represent
+	## and number of types of data.
+	type StrType: enum {
+		## A complete URL without the prefix "http://".
+		URL,
+		## User-Agent string, typically HTTP or mail message body.
+		USER_AGENT,
+		## Email address.
+		EMAIL,
+		## DNS domain name.
+		DOMAIN,
+		## A user name.
+		USER_NAME,
+		## File hash which is non-hash type specific.  It's up to the user to query
+		## for any relevant hash types.
+		FILE_HASH,
+		## Certificate SHA-1 hash.
+		CERT_HASH,
 	};
 	
-	## Record type used for logging information from the intelligence framework.
-	## Primarily for problems or oddities with inserting and querying data.  
-	## This is important since the content of the intelligence framework can 
-	## change quite dramatically during runtime and problems may be introduced 
-	## into the data.
-	type Info: record {
-		## The current network time.
-		ts:      time   &log;
-		## Represents the severity of the message. 
-		## This value should be one of: "info", "warn", "error"
-		level:   string &log;
-		## The message.
-		message: string &log;
-	};
-	
-	## Record to represent metadata associated with a single piece of
-	## intelligence.
+	## Data about an :bro:type:`Intel::Item`
 	type MetaData: record {
-		## A description for the data.
+		## An arbitrary string value representing the data source.  Typically,
+		## the convention for this field will be the source name and feed name
+		## separated by a hyphen.  For example: "source1-c&c".
+		source:      string;
+		## A freeform description for the data.
 		desc:        string      &optional;
-		## A URL where more information may be found about the intelligence.
+		## A URL for more information about the data.
 		url:         string      &optional;
-		## The time at which the data was first declared to be intelligence.
-		first_seen:  time        &optional;
-		## When this data was most recent inserted into the framework.
-		latest_seen: time        &optional;
-		## Arbitrary text tags for the data.
-		tags:        set[string];
 	};
 	
-	## Record to represent a singular piece of intelligence.
+	## Represents a piece of intelligence.
 	type Item: record {
-		## If the data is an IP address, this hold the address.
-		ip:          addr        &optional;
-		## If the data is textual, this holds the text.
-		str:         string      &optional;
-		## If the data is numeric, this holds the number.
-		num:         int         &optional;
-		## The subtype of the data for when either the $str or $num fields are
-		## given.  If one of those fields are given, this field must be present.
-		subtype:     string      &optional;
+		## The IP address if the intelligence is about an IP address.
+		host:        addr           &optional;
+		## The network if the intelligence is about a CIDR block.
+		net:         subnet         &optional;
+		## The string if the intelligence is about a string.
+		str:         string         &optional;
+		## The type of data that is in the string if the $str field is set.
+		str_type:    StrType        &optional;
 		
-		## The next five fields are temporary until a better model for 
-		## attaching metadata to an intelligence item is created.
-		desc:        string      &optional;
-		url:         string      &optional;
-		first_seen:  time        &optional;
-		latest_seen: time        &optional;
-		tags:        set[string];
-		
-		## These single string tags are throw away until pybroccoli supports sets.
-		tag1: string &optional;
-		tag2: string &optional;
-		tag3: string &optional;
+		## Metadata for the item.  Typically represents more deeply \
+		## descriptive data for a piece of intelligence.
+		meta:        MetaData;
 	};
 	
-	## Record model used for constructing queries against the intelligence 
-	## framework.
-	type QueryItem: record {
-		## If an IP address is being queried for, this field should be given.
-		ip:        addr        &optional;
-		## If a string is being queried for, this field should be given.
-		str:       string      &optional;
-		## If numeric data is being queried for, this field should be given.
-		num:       int         &optional;
-		## If either a string or number is being queried for, this field should
-		## indicate the subtype of the data.
-		subtype:   string      &optional;
-		
-		## A set of tags where if a single metadata record attached to an item
-		## has any one of the tags defined in this field, it will match.
-		or_tags:   set[string] &optional;
-		## A set of tags where a single metadata record attached to an item 
-		## must have all of the tags defined in this field.
-		and_tags:  set[string] &optional; 
-		
-		## The predicate can be given when searching for a match.  It will
-		## be tested against every :bro:type:`Intel::MetaData` item associated
-		## with the data being matched on.  If it returns T a single time, the 
-		## matcher will consider that the item has matched.  This field can
-		## be used for constructing arbitrarily complex queries that may not
-		## be possible with the $or_tags or $and_tags fields.
-		pred:      function(meta: Intel::MetaData): bool &optional;
+	## Enum to represent where data came from when it was discovered.
+	## The convention is to prefix the name with ``IN_``.
+	type Where: enum {
+		## A catchall value to represent data of unknown provenance.
+		IN_ANYWHERE,
 	};
-	
-	## Function to insert data into the intelligence framework.
-	## 
-	## item: The data item.
+
+	## The $host field and combination of $str and $str_type fields are mutually 
+	## exclusive.  These records *must* represent either an IP address being
+	## seen or a string being seen.
+	type Seen: record {
+		## The IP address if the data seen is an IP address.
+		host:      addr          &log &optional;
+		## The string if the data is about a string.
+		str:       string        &log &optional;
+		## The type of data that is in the string if the $str field is set.
+		str_type:  StrType       &log &optional;
+
+		## Where the data was discovered.
+		where:     Where         &log;
+		
+		## If the data was discovered within a connection, the 
+		## connection record should go into get to give context to the data.
+		conn:      connection    &optional;
+	};
+
+	## Record used for the logging framework representing a positive
+	## hit within the intelligence framework.
+	type Info: record {
+		## Timestamp when the data was discovered.
+		ts:       time           &log;
+
+		## If a connection was associated with this intelligence hit,
+		## this is the uid for the connection
+		uid:      string         &log &optional;
+		## If a connection was associated with this intelligence hit,
+		## this is the conn_id for the connection.
+		id:       conn_id        &log &optional;
+
+		## Where the data was seen.
+		seen:     Seen           &log;
+		## Sources which supplied data that resulted in this match.
+		sources:  set[string]    &log;
+	};
+
+	## Intelligence data manipulation functions.
+	global insert: function(item: Item);
+
+	## Function to declare discovery of a piece of data in order to check
+	## it against known intelligence for matches.
+	global seen: function(s: Seen);
+
+	## Event to represent a match in the intelligence data from data that was seen.  
+	## On clusters there is no assurance as to where this event will be generated 
+	## so do not assume that arbitrary global state beyond the given data
+	## will be available.
 	##
-	## Returns: T if the data was successfully inserted into the framework,
-	##          otherwise it returns F.
-	global insert: function(item: Item): bool;
-	
-	## A wrapper for the :bro:id:`Intel::insert` function.  This is primarily
-	## used as the external API for inserting data into the intelligence 
-	## using Broccoli.
-	global insert_event: event(item: Item);
-	
-	## Function for matching data within the intelligence framework.
-	global matcher: function(item: QueryItem): bool;
+	## This is the primary mechanism where a user will take actions based on data
+	## within the intelligence framework.
+	global match: event(s: Seen, items: set[Item]);
+
+	global log_intel: event(rec: Info);
 }
 
-type MetaDataStore: table[count] of MetaData;
+# Internal handler for matches with no metadata available.
+global match_no_items: event(s: Seen);
+
+# Internal events for cluster data distribution
+global new_item: event(item: Item);
+global updated_item: event(item: Item);
+
+# Optionally store metadata.  This is used internally depending on
+# if this is a cluster deployment or not.
+const have_full_data = T &redef;
+
+# The in memory data structure for holding intelligence.
 type DataStore: record {
-	ip_data:     table[addr] of MetaDataStore;
-	# The first string is the actual value and the second string is the subtype.
-	string_data: table[string, string] of MetaDataStore;
-	int_data:    table[int, string] of MetaDataStore;
+	net_data:    table[subnet] of set[MetaData];
+	string_data: table[string, StrType] of set[MetaData];
 };
-global data_store: DataStore;
+global data_store: DataStore &redef;
 
-event bro_init()
+# The in memory data structure for holding the barest matchable intelligence.
+# This is primarily for workers to do the initial quick matches and store
+# a minimal amount of data for the full match to happen on the manager.
+type MinDataStore: record {
+	net_data:    set[subnet];
+	string_data: set[string, StrType];
+};
+global min_data_store: MinDataStore &redef;
+
+
+event bro_init() &priority=5
 	{
-	Log::create_stream(Intel::LOG, [$columns=Info]);
+	Log::create_stream(LOG, [$columns=Info, $ev=log_intel]);
 	}
 
-
-function insert(item: Item): bool
+function find(s: Seen): bool
 	{
-	local err_msg = "";
-	if ( (item?$str || item?$num) && ! item?$subtype )
-		err_msg = "You must provide a subtype to insert_sync or this item doesn't make sense.";
-	
-	if ( err_msg == "" )
+	if ( s?$host && 
+	     ((have_full_data && s$host in data_store$net_data) || 
+	      (s$host in min_data_store$net_data)))
 		{
-		# Create and fill out the meta data item.
-		local meta: MetaData;
-		if ( item?$first_seen )
-			meta$first_seen = item$first_seen;
-		if ( item?$latest_seen )
-			meta$latest_seen = item$latest_seen;
-		if ( item?$tags )
-			meta$tags = item$tags;
-		if ( item?$desc )
-			meta$desc = item$desc;
-		if ( item?$url )
-			meta$url = item$url;
-		
-		
-		# This is hopefully only temporary until pybroccoli supports sets.
-		if ( item?$tag1 )
-			add item$tags[item$tag1];
-		if ( item?$tag2 )
-			add item$tags[item$tag2];
-		if ( item?$tag3 )
-			add item$tags[item$tag3];
-		
-		if ( item?$ip )
-			{
-			if ( item$ip !in data_store$ip_data )
-				data_store$ip_data[item$ip] = table();
-			data_store$ip_data[item$ip][|data_store$ip_data[item$ip]|] = meta;
-			return T;
-			}
-		else if ( item?$str )
-			{
-			if ( [item$str, item$subtype] !in data_store$string_data )
-				data_store$string_data[item$str, item$subtype] = table();
-			
-			data_store$string_data[item$str, item$subtype][|data_store$string_data[item$str, item$subtype]|] = meta;
-			return T;
-			}
-		else if ( item?$num )
-			{
-			if ( [item$num, item$subtype] !in data_store$int_data )
-				data_store$int_data[item$num, item$subtype] = table();
+		return T;
+		}
+	else if ( s?$str && s?$str_type &&
+	          ((have_full_data && [s$str, s$str_type] in data_store$string_data) ||
+	           ([s$str, s$str_type] in min_data_store$string_data)))
+		{
+		return T;
+		}
+	else
+		{
+		return F;
+		}
+	}
 
-			data_store$int_data[item$num, item$subtype][|data_store$int_data[item$num, item$subtype]|] = meta;
-			return T;
+function get_items(s: Seen): set[Item]
+	{
+	local item: Item;
+	local return_data: set[Item] = set();
+
+	if ( ! have_full_data )
+		{
+		# A reporter warning should be generated here because this function
+		# should never be called from a host that doesn't have the full data.
+		# TODO: do a reporter warning.
+		return return_data;
+		}
+
+	if ( s?$host )
+		{
+		# See if the host is known about and it has meta values
+		if ( s$host in data_store$net_data )
+			{
+			for ( m in data_store$net_data[s$host] )
+				{
+				# TODO: the lookup should be finding all and not just most specific
+				#       and $host/$net should have the correct value.
+				item = [$host=s$host, $meta=m];
+				add return_data[item];
+				}
+			}
+		}
+	else if ( s?$str && s?$str_type )
+		{
+		# See if the string is known about and it has meta values
+		if ( [s$str, s$str_type] in data_store$string_data )
+			{
+			for ( m in data_store$string_data[s$str, s$str_type] )
+				{
+				item = [$str=s$str, $str_type=s$str_type, $meta=m];
+				add return_data[item];
+				}
+			}
+		}
+
+	return return_data;
+	}
+
+function Intel::seen(s: Seen)
+	{
+	if ( find(s) )
+		{
+		if ( have_full_data )
+			{
+			local items = get_items(s);
+			event Intel::match(s, items);
 			}
 		else
-			err_msg = "Failed to insert intelligence item for some unknown reason.";
-		}
-	
-	if ( err_msg != "" )
-		Log::write(Intel::LOG, [$ts=network_time(), $level="warn", $message=fmt(err_msg)]);
-	return F;
-	}
-	
-event insert_event(item: Item)
-	{
-	insert(item);
-	}
-	
-function match_item_with_metadata(item: QueryItem, meta: MetaData): bool
-	{
-	if ( item?$and_tags )
-		{
-		local matched = T;
-		# Every tag given has to match in a single MetaData entry.
-		for ( tag in item$and_tags )
 			{
-			if ( tag !in meta$tags )
-				matched = F;
+			event Intel::match_no_items(s);
 			}
-		if ( matched )
+		}
+	}
+
+
+function has_meta(check: MetaData, metas: set[MetaData]): bool
+	{
+	local check_hash = md5_hash(check);
+	for ( m in metas )
+		{
+		if ( check_hash == md5_hash(m) )
 			return T;
 		}
-	else if ( item?$or_tags )
-		{
-		# For OR tags, only a single tag has to match.
-		for ( tag in item$or_tags )
-			{
-			if ( tag in meta$tags )
-				return T;
-			}
-		}
-	else if ( item?$pred )
-		return item$pred(meta);
 
-	# This indicates some sort of failure in the query
+	# The records must not be equivalent if we made it this far.
 	return F;
 	}
+
+event Intel::match(s: Seen, items: set[Item]) &priority=5
+	{
+	local empty_set: set[string] = set();
+	local info: Info = [$ts=network_time(), $seen=s, $sources=empty_set];
+
+	if ( s?$conn )
+		{
+		info$uid = s$conn$uid;
+		info$id  = s$conn$id;
+		}
+
+	for ( item in items )
+		add info$sources[item$meta$source];
+
+	Log::write(Intel::LOG, info);
+	}
+
+function insert(item: Item)
+	{
+	if ( item?$str && !item?$str_type )
+		{
+		event reporter_warning(network_time(), fmt("You must provide a str_type for strings or this item doesn't make sense.  Item: %s", item), "");
+		return;
+		}
+
+	# Create and fill out the meta data item.
+	local meta = item$meta;
+	local metas: set[MetaData];
+
+	if ( item?$host )
+		{
+		local host = mask_addr(item$host, is_v4_addr(item$host) ? 32 : 128);
+		if ( have_full_data )
+			{
+			if ( host !in data_store$net_data )
+				data_store$net_data[host] = set();
+
+			metas = data_store$net_data[host];
+			}
+
+		add min_data_store$net_data[host];
+		}
+	else if ( item?$net )
+		{
+		if ( have_full_data )
+			{
+			if ( item$net !in data_store$net_data )
+				data_store$net_data[item$net] = set();
+
+			metas = data_store$net_data[item$net];
+			}
+
+		add min_data_store$net_data[item$net];
+		}
+	else if ( item?$str )
+		{
+		if ( have_full_data )
+			{
+			if ( [item$str, item$str_type] !in data_store$string_data )
+				data_store$string_data[item$str, item$str_type] = set();
+
+			metas = data_store$string_data[item$str, item$str_type];
+			}
+
+		add min_data_store$string_data[item$str, item$str_type];
+		}
+
+	local updated = F;
+	if ( have_full_data )
+		{
+		for ( m in metas )
+			{
+			if ( meta$source == m$source )
+				{
+				if ( has_meta(meta, metas) )
+					{
+					# It's the same item being inserted again.
+					return;
+					}
+				else
+					{
+					# Same source, different metadata means updated item.
+					updated = T;
+					}
+				}
+			}
+		add metas[item$meta];
+		}
+	
+	if ( updated )
+		event Intel::updated_item(item);
+	else
+		event Intel::new_item(item);
+	}
 	
-function matcher(item: QueryItem): bool
-	{
-	local err_msg = "";
-	if ( ! (item?$ip || item?$str || item?$num) )
-		err_msg = "You must supply one of the $ip, $str, or $num fields to search on";
-	else if ( (item?$or_tags || item?$and_tags) && item?$pred )
-		err_msg = "You can't match with both tags and a predicate.";
-	else if ( item?$or_tags && item?$and_tags )
-		err_msg = "You can't match with both OR'd together tags and AND'd together tags";
-	else if ( (item?$str || item?$num) && ! item?$subtype )
-		err_msg = "You must provide a subtype to matcher or this item doesn't make sense.";
-	else if ( item?$str && item?$num )
-		err_msg = "You must only provide $str or $num, not both.";
-		
-	local meta: MetaData;
-
-	if ( err_msg == "" )
-		{
-		if ( item?$ip )
-			{
-			if ( item$ip in data_store$ip_data )
-				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
-					return T;
-			
-				for ( i in data_store$ip_data[item$ip] )
-					{
-					meta = data_store$ip_data[item$ip][i];
-					if ( match_item_with_metadata(item, meta) )
-						return T;
-					}
-				}
-			}
-		
-		else if ( item?$str )
-			{
-			if ( [item$str, item$subtype] in data_store$string_data )
-				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
-					return T;
-
-				for ( i in data_store$string_data[item$str, item$subtype] )
-					{
-					meta = data_store$string_data[item$str, item$subtype][i];
-					if ( match_item_with_metadata(item, meta) )
-						return T;
-					}
-				}
-			}
-		
-		else if ( item?$num )
-			{
-			if ( [item$num, item$subtype] in data_store$int_data )
-				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
-					return T;
-
-				for ( i in data_store$int_data[item$num, item$subtype] )
-					{
-					meta = data_store$int_data[item$num, item$subtype][i];
-					if ( match_item_with_metadata(item, meta) )
-						return T;
-					}
-				}
-			}
-		else
-			err_msg = "Failed to query intelligence data for some unknown reason.";
-		}
-		
-	if ( err_msg != "" )
-		Log::write(Intel::LOG, [$ts=network_time(), $level="error", $message=fmt(err_msg)]);
-	return F;
-	}

scripts/base/frameworks/logging/main.bro

@@ -60,6 +60,9 @@ export {
 	## Default rotation interval. Zero disables rotation.
 	const default_rotation_interval = 0secs &redef;
 
+	## Default alarm summary mail interval. Zero disables alarm summary mails.
+	const default_mail_alarms_interval = 0secs &redef;
+
 	## Default naming format for timestamps embedded into filenames.
 	## Uses a ``strftime()`` style.
 	const default_rotation_date_format = "%Y-%m-%d-%H-%M-%S" &redef;

scripts/base/frameworks/logging/writers/ascii.bro

@@ -1,5 +1,13 @@
 ##! Interface for the ASCII log writer.  Redefinable options are available
 ##! to tweak the output format of ASCII logs.
+##!
+##! The ASCII writer supports currently one writer-specific filter option via
+##! ``config``: setting ``only_single_header_row`` to ``T`` turns the output into
+##! into CSV mode where only a single header row with the column names is printed
+##! out as meta information. Example filter using this::
+##! 
+##!    local my_filter: Log::Filter = [$name = "my-filter", $writer = Log::WRITER_ASCII, $config = table(["only_single_header_row"] = "T")]; 
+##!
 
 module LogAscii;
 

scripts/base/frameworks/notice/actions/pp-alarms.bro

@@ -101,7 +101,7 @@ event bro_init()
 	# This replaces the standard non-pretty-printing filter.
 	Log::add_filter(Notice::ALARM_LOG,
 			[$name="alarm-mail", $writer=Log::WRITER_NONE,
-			 $interv=Log::default_rotation_interval,
+			 $interv=Log::default_mail_alarms_interval,
 			 $postprocessor=pp_postprocessor]);
 	}
 

scripts/base/init-bare.bro

@@ -826,7 +826,7 @@ const tcp_storm_interarrival_thresh = 1 sec &redef;
 ## peer's ACKs.  Set to zero to turn off this determination.
 ##
 ## .. bro:see:: tcp_max_above_hole_without_any_acks tcp_excessive_data_without_further_acks
-const tcp_max_initial_window = 4096;
+const tcp_max_initial_window = 4096 &redef;
 
 ## If we're not seeing our peer's ACKs, the maximum volume of data above a sequence
 ## hole that we'll tolerate before assuming that there's been a packet drop and we
@@ -834,7 +834,7 @@ const tcp_max_initial_window = 4096;
 ## up.
 ##
 ## .. bro:see:: tcp_max_initial_window tcp_excessive_data_without_further_acks
-const tcp_max_above_hole_without_any_acks = 4096;
+const tcp_max_above_hole_without_any_acks = 4096 &redef;
 
 ## If we've seen this much data without any of it being acked, we give up
 ## on that connection to avoid memory exhaustion due to buffering all that
@@ -843,7 +843,7 @@ const tcp_max_above_hole_without_any_acks = 4096;
 ## has in fact gone too far, but for now we just make this quite beefy.
 ##
 ## .. bro:see:: tcp_max_initial_window tcp_max_above_hole_without_any_acks
-const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024;
+const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024 &redef;
 
 ## For services without an a handler, these sets define originator-side ports that
 ## still trigger reassembly.
@@ -2495,6 +2495,16 @@ type bittorrent_benc_dir: table[string] of bittorrent_benc_value;
 ##    bt_tracker_response_not_ok
 type bt_tracker_headers: table[string] of string;
 
+type ModbusCoils: vector of bool;
+type ModbusRegisters: vector of count;
+
+type ModbusHeaders: record {
+	tid:           count;
+	pid:           count;
+	uid:           count;
+	function_code: count;
+};
+
 module SOCKS;
 export {
 	## This record is for a SOCKS client or server to provide either a 

scripts/base/init-default.bro

@@ -14,6 +14,7 @@
 @load base/utils/patterns
 @load base/utils/strings
 @load base/utils/thresholds
+@load base/utils/urls
 
 # This has some deep interplay between types and BiFs so it's 
 # loaded in base/init-bare.bro
@@ -36,8 +37,11 @@
 @load base/protocols/ftp
 @load base/protocols/http
 @load base/protocols/irc
+@load base/protocols/modbus
 @load base/protocols/smtp
 @load base/protocols/socks
 @load base/protocols/ssh
 @load base/protocols/ssl
 @load base/protocols/syslog
+
+@load base/misc/find-checksum-offloading

scripts/base/misc/find-checksum-offloading.bro

@@ -0,0 +1,57 @@
+##! Discover cases where the local interface is sniffed and outbound packets
+##! have checksum offloading.  Load this script to receive a notice if it's
+##! likely that checksum offload effects are being seen on a live interface or
+##! in a packet trace file.
+
+@load base/frameworks/notice
+
+module ChecksumOffloading;
+
+export {
+	## The interval which is used for checking packet statistics
+	## to see if checksum offloading is affecting analysis.
+	const check_interval = 10secs &redef;
+}
+
+# Keep track of how many bad checksums have been seen.
+global bad_checksums = 0;
+
+# Track to see if this script is done so that messages aren't created multiple times.
+global done = F;
+
+event ChecksumOffloading::check()
+	{
+	if ( done )
+		return;
+
+	local pkts_recvd = net_stats()$pkts_recvd;
+	if ( (bad_checksums*1.0 / net_stats()$pkts_recvd*1.0) > 0.05 )
+		{
+		local packet_src = reading_traces() ? "trace file likely has" : "interface is likely receiving";
+		local message = fmt("Your %s invalid IP checksums, most likely from NIC checksum offloading.", packet_src);
+		Reporter::warning(message);
+		done = T;
+		}
+	else if ( pkts_recvd < 20 )
+		{
+		# Keep scheduling this event until we've seen some lower threshold of
+		# total packets.
+		schedule check_interval { ChecksumOffloading::check() };
+		}
+	}
+
+event bro_init()
+	{
+	schedule check_interval { ChecksumOffloading::check() };
+	}
+
+event net_weird(name: string)
+	{
+	if ( name == "bad_IP_checksum" )
+		++bad_checksums;
+	}
+
+event bro_done()
+	{
+	event ChecksumOffloading::check();
+	}

scripts/base/protocols/ftp/gridftp.bro

@@ -68,6 +68,16 @@ export {
 	const data_channel_initial_criteria: function(c: connection): bool &redef;
 }
 
+redef record FTP::Info += {
+	last_auth_requested: string &optional;
+};
+
+event ftp_request(c: connection, command: string, arg: string) &priority=4
+	{
+	if ( command == "AUTH" && c?$ftp )
+		c$ftp$last_auth_requested = arg;
+	}
+
 function size_callback(c: connection, cnt: count): interval
 	{
 	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
@@ -89,8 +99,10 @@ function size_callback(c: connection, cnt: count): interval
 
 event ssl_established(c: connection) &priority=5
 	{
-	# Add service label to control channels.
-	if ( "FTP" in c$service )
+	# If an FTP client requests AUTH GSSAPI and later an SSL handshake
+	# finishes, it's likely a GridFTP control channel, so add service label.
+	if ( c?$ftp && c$ftp?$last_auth_requested &&
+	     /GSSAPI/ in c$ftp$last_auth_requested )
 		add c$service["gridftp"];
 	}
 

scripts/base/protocols/modbus/__load__.bro

@@ -0,0 +1,2 @@
+@load ./consts
+@load ./main

scripts/base/protocols/modbus/consts.bro

@@ -0,0 +1,67 @@
+
+module Modbus;
+
+export {
+	## Standard defined Modbus function codes.
+	const function_codes = {
+		[0x01] = "READ_COILS",
+		[0x02] = "READ_DISCRETE_INPUTS",
+		[0x03] = "READ_HOLDING_REGISTERS",
+		[0x04] = "READ_INPUT_REGISTERS",
+		[0x05] = "WRITE_SINGLE_COIL",
+		[0x06] = "WRITE_SINGLE_REGISTER",
+		[0x07] = "READ_EXCEPTION_STATUS",
+		[0x08] = "DIAGNOSTICS",
+		[0x0B] = "GET_COMM_EVENT_COUNTER",
+		[0x0C] = "GET_COMM_EVENT_LOG",
+		[0x0F] = "WRITE_MULTIPLE_COILS",
+		[0x10] = "WRITE_MULTIPLE_REGISTERS",
+		[0x11] = "REPORT_SLAVE_ID",
+		[0x14] = "READ_FILE_RECORD",
+		[0x15] = "WRITE_FILE_RECORD",
+		[0x16] = "MASK_WRITE_REGISTER",
+		[0x17] = "READ_WRITE_MULTIPLE_REGISTERS",
+		[0x18] = "READ_FIFO_QUEUE",
+		[0x2B] = "ENCAP_INTERFACE_TRANSPORT",
+
+		# Machine/vendor/network specific functions
+		[0x09] = "PROGRAM_484",
+		[0x0A] = "POLL_484",
+		[0x0D] = "PROGRAM_584_984",
+		[0x0E] = "POLL_584_984",
+		[0x12] = "PROGRAM_884_U84",
+		[0x13] = "RESET_COMM_LINK_884_U84",
+		[0x28] = "PROGRAM_CONCEPT",
+		[0x7D] = "FIRMWARE_REPLACEMENT",
+		[0x7E] = "PROGRAM_584_984_2",
+		[0x7F] = "REPORT_LOCAL_ADDRESS",
+
+		# Exceptions
+		[0x81] = "READ_COILS_EXCEPTION",
+		[0x82] = "READ_DISCRETE_INPUTS_EXCEPTION",
+		[0x83] = "READ_HOLDING_REGISTERS_EXCEPTION",
+		[0x84] = "READ_INPUT_REGISTERS_EXCEPTION",
+		[0x85] = "WRITE_SINGLE_COIL_EXCEPTION",
+		[0x86] = "WRITE_SINGLE_REGISTER_EXCEPTION",
+		[0x87] = "READ_EXCEPTION_STATUS_EXCEPTION",
+		[0x8F] = "WRITE_MULTIPLE_COILS_EXCEPTION",
+		[0x90] = "WRITE_MULTIPLE_REGISTERS_EXCEPTION",
+		[0x94] = "READ_FILE_RECORD_EXCEPTION",
+		[0x95] = "WRITE_FILE_RECORD_EXCEPTION",
+		[0x96] = "MASK_WRITE_REGISTER_EXCEPTION",
+		[0x97] = "READ_WRITE_MULTIPLE_REGISTERS_EXCEPTION",
+		[0x98] = "READ_FIFO_QUEUE_EXCEPTION",
+	} &default=function(i: count):string { return fmt("unknown-%d", i); } &redef;
+
+	const exception_codes = {
+		[0x01] = "ILLEGAL_FUNCTION",
+		[0x02] = "ILLEGAL_DATA_ADDRESS",
+		[0x03] = "ILLEGAL_DATA_VALUE",
+		[0x04] = "SLAVE_DEVICE_FAILURE",
+		[0x05] = "ACKNOWLEDGE",
+		[0x06] = "SLAVE_DEVICE_BUSY",
+		[0x08] = "MEMORY_PARITY_ERROR",
+		[0x0A] = "GATEWAY_PATH_UNAVAILABLE",
+		[0x0B] = "GATEWAY_TARGET_DEVICE_FAILED_TO_RESPOND",
+	} &default=function(i: count):string { return fmt("unknown-%d", i); } &redef;
+}

scripts/base/protocols/modbus/main.bro

@@ -0,0 +1,71 @@
+##! Base Modbus analysis script.
+
+module Modbus;
+
+@load ./consts
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Time of the request.
+		ts:        time           &log;
+		## Unique identifier for the connnection.
+		uid:       string         &log;
+		## Identifier for the connection.
+		id:        conn_id        &log;
+		## The name of the function message that was sent.
+		func:      string         &log &optional;
+		## The exception if the response was a failure.
+		exception: string         &log &optional;
+	};
+
+	## Event that can be handled to access the Modbus record as it is sent on
+	## to the logging framework.
+	global log_modbus: event(rec: Info);
+}
+
+redef record connection += {
+	modbus: Info &optional;
+};
+
+# Configure DPD and the packet filter.
+redef capture_filters += { ["modbus"] = "tcp port 502" };
+redef dpd_config += { [ANALYZER_MODBUS] = [$ports = set(502/tcp)] };
+redef likely_server_ports += { 502/tcp };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Modbus::LOG, [$columns=Info, $ev=log_modbus]);
+	}
+
+event modbus_message(c: connection, headers: ModbusHeaders, is_orig: bool) &priority=5
+	{
+	if ( ! c?$modbus )
+		{
+		c$modbus = [$ts=network_time(), $uid=c$uid, $id=c$id];
+		}
+
+	c$modbus$ts   = network_time();
+	c$modbus$func = function_codes[headers$function_code];
+	}
+
+event modbus_message(c: connection, headers: ModbusHeaders, is_orig: bool) &priority=-5
+	{
+	# Only log upon replies.
+	# Also, don't log now if this is an exception (log in the exception event handler)
+	if ( ! is_orig && ( headers$function_code <= 0x81 || headers$function_code >= 0x98 ) )
+		Log::write(LOG, c$modbus);
+	}
+
+event modbus_exception(c: connection, headers: ModbusHeaders, code: count) &priority=5
+	{
+	c$modbus$exception = exception_codes[code];
+	}
+
+event modbus_exception(c: connection, headers: ModbusHeaders, code: count) &priority=-5
+	{
+	Log::write(LOG, c$modbus);
+	delete c$modbus$exception;
+	}
+

scripts/base/utils/patterns.bro

@@ -1,5 +1,7 @@
 ##! Functions for creating and working with patterns.
 
+module GLOBAL;
+
 ## Given a pattern as a string with two tildes (~~) contained in it, it will
 ## return a pattern with string set's elements OR'd together where the
 ## double-tilde was given (this function only works at or before init time).

scripts/base/utils/urls.bro

@@ -0,0 +1,25 @@
+## Functions for URL handling.
+
+## A regular expression for matching and extracting URLs.
+const url_regex = /^([a-zA-Z\-]{3,5})(:\/\/[^\/?#"'\r\n><]*)([^?#"'\r\n><]*)([^[:blank:]\r\n"'><]*|\??[^"'\r\n><]*)/ &redef;
+
+## Extracts URLs discovered in arbitrary text.
+function find_all_urls(s: string): string_set
+	{
+	return find_all(s, url_regex);
+	}
+
+## Extracts URLs discovered in arbitrary text without
+## the URL scheme included.
+function find_all_urls_without_scheme(s: string): string_set
+	{
+	local urls = find_all_urls(s);
+	local return_urls: set[string] = set();
+	for ( url in urls )
+		{
+		local no_scheme = sub(url, /^([a-zA-Z\-]{3,5})(:\/\/)/, "");
+		add return_urls[no_scheme];
+		}
+
+	return return_urls;
+	}

scripts/policy/frameworks/intel/__load__.bro

@@ -0,0 +1,8 @@
+@load ./conn-established
+@load ./dns
+@load ./http-host-header
+@load ./http-url
+@load ./http-user-agents
+@load ./ssl
+@load ./smtp
+@load ./smtp-url-extraction

scripts/policy/frameworks/intel/conn-established.bro

@@ -0,0 +1,8 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event connection_established(c: connection)
+	{
+	Intel::seen([$host=c$id$orig_h, $conn=c, $where=Conn::IN_ORIG]);
+	Intel::seen([$host=c$id$resp_h, $conn=c, $where=Conn::IN_RESP]);
+	}

scripts/policy/frameworks/intel/dns.bro

@@ -0,0 +1,10 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count)
+	{
+	Intel::seen([$str=query,
+	             $str_type=Intel::DOMAIN,
+	             $conn=c,
+	             $where=DNS::IN_REQUEST]);
+	}

scripts/policy/frameworks/intel/http-host-header.bro

@@ -0,0 +1,11 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event http_header(c: connection, is_orig: bool, name: string, value: string)
+	{
+	if ( is_orig && name == "HOST" )
+		Intel::seen([$str=value,
+		             $str_type=Intel::DOMAIN,
+		             $conn=c,
+		             $where=HTTP::IN_HOST_HEADER]);
+	}

scripts/policy/frameworks/intel/http-url.bro

@@ -0,0 +1,12 @@
+@load base/frameworks/intel
+@load base/protocols/http/utils
+@load ./where-locations
+
+event http_message_done(c: connection, is_orig: bool, stat: http_message_stat)
+	{
+	if ( is_orig && c?$http )
+		Intel::seen([$str=HTTP::build_url(c$http),
+		             $str_type=Intel::URL,
+		             $conn=c,
+		             $where=HTTP::IN_URL]);
+	}

scripts/policy/frameworks/intel/http-user-agents.bro

@@ -0,0 +1,12 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event http_header(c: connection, is_orig: bool, name: string, value: string)
+	{
+	if ( is_orig && name == "USER-AGENT" )
+		Intel::seen([$str=value,
+		             $str_type=Intel::USER_AGENT,
+		             $conn=c,
+		             $where=HTTP::IN_USER_AGENT_HEADER]);
+	}
+

scripts/policy/frameworks/intel/smtp-url-extraction.bro

@@ -0,0 +1,15 @@
+@load base/frameworks/intel
+@load base/utils/urls
+@load ./where-locations
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=3
+	{
+	local urls = find_all_urls_without_scheme(data);
+	for ( url in urls )
+		{
+		Intel::seen([$str=url,
+		             $str_type=Intel::URL,
+		             $conn=c,
+		             $where=SMTP::IN_MESSAGE]);
+		}
+	}

scripts/policy/frameworks/intel/smtp.bro

@@ -0,0 +1,71 @@
+@load base/frameworks/intel
+@load base/protocols/smtp
+@load ./where-locations
+
+event mime_end_entity(c: connection)
+	{
+	if ( c?$smtp )
+		{
+		if ( c$smtp?$path )
+			{
+			local path = c$smtp$path;
+			for ( i in path )
+				{
+				Intel::seen([$host=path[i],
+				             $conn=c,
+				             $where=SMTP::IN_RECEIVED_HEADER]);
+				}
+			}
+
+		if ( c$smtp?$user_agent )
+			Intel::seen([$str=c$smtp$user_agent,
+			             $str_type=Intel::USER_AGENT,
+			             $conn=c,
+			             $where=SMTP::IN_HEADER]);
+
+		if ( c$smtp?$x_originating_ip )
+			Intel::seen([$host=c$smtp$x_originating_ip,
+			             $conn=c,
+			             $where=SMTP::IN_X_ORIGINATING_IP_HEADER]);
+
+		if ( c$smtp?$mailfrom )
+			Intel::seen([$str=c$smtp$mailfrom,
+			             $str_type=Intel::EMAIL,
+			             $conn=c,
+			             $where=SMTP::IN_MAIL_FROM]);
+
+		if ( c$smtp?$rcptto )
+			{
+			for ( rcptto in c$smtp$rcptto )
+				{
+				Intel::seen([$str=rcptto,
+				             $str_type=Intel::EMAIL,
+				             $conn=c,
+				             $where=SMTP::IN_RCPT_TO]);
+				}
+			}
+
+		if ( c$smtp?$from )
+			Intel::seen([$str=c$smtp$from,
+			             $str_type=Intel::EMAIL,
+			             $conn=c,
+			             $where=SMTP::IN_FROM]);
+
+		if ( c$smtp?$to )
+			{
+			for ( email_to in c$smtp$to )
+				{
+				Intel::seen([$str=email_to,
+				             $str_type=Intel::EMAIL,
+				             $conn=c,
+				             $where=SMTP::IN_TO]);
+				}
+			}
+
+		if ( c$smtp?$reply_to )
+			Intel::seen([$str=c$smtp$reply_to,
+			             $str_type=Intel::EMAIL,
+			             $conn=c,
+			             $where=SMTP::IN_REPLY_TO]);
+		}
+	}

scripts/policy/frameworks/intel/ssl.bro

@@ -0,0 +1,34 @@
+@load base/frameworks/intel
+@load base/protocols/ssl
+@load ./where-locations
+
+event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string)
+	{
+	if ( chain_idx == 0 )
+		{
+		if ( /emailAddress=/ in cert$subject )
+			{
+			local email = sub(cert$subject, /^.*emailAddress=/, "");
+			email = sub(email, /,.*$/, "");
+			Intel::seen([$str=email,
+			             $str_type=Intel::EMAIL,
+			             $conn=c,
+			             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
+			}
+
+		Intel::seen([$str=sha1_hash(der_cert),
+		             $str_type=Intel::CERT_HASH,
+		             $conn=c,
+		             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
+		}
+	}
+
+event ssl_extension(c: connection, is_orig: bool, code: count, val: string)
+	{
+	if ( is_orig && SSL::extensions[code] == "server_name" && 
+	     c?$ssl && c$ssl?$server_name )
+		Intel::seen([$str=c$ssl$server_name,
+		             $str_type=Intel::DOMAIN,
+		             $conn=c,
+		             $where=SSL::IN_SERVER_NAME]);
+	}

scripts/policy/frameworks/intel/where-locations.bro

@@ -0,0 +1,25 @@
+@load base/frameworks/intel
+
+export {
+	redef enum Intel::Where += {
+		Conn::IN_ORIG,
+		Conn::IN_RESP,
+		DNS::IN_REQUEST,
+		DNS::IN_RESPONSE,
+		HTTP::IN_HOST_HEADER,
+		HTTP::IN_USER_AGENT_HEADER,
+		HTTP::IN_URL,
+		SMTP::IN_MAIL_FROM,
+		SMTP::IN_RCPT_TO,
+		SMTP::IN_FROM,
+		SMTP::IN_TO,
+		SMTP::IN_RECEIVED_HEADER,
+		SMTP::IN_REPLY_TO,
+		SMTP::IN_X_ORIGINATING_IP_HEADER,
+		SMTP::IN_MESSAGE,
+		SSL::IN_SERVER_CERT,
+		SSL::IN_CLIENT_CERT,
+		SSL::IN_SERVER_NAME,
+		SMTP::IN_HEADER,
+	};
+}

scripts/policy/integration/collective-intel/README

@@ -0,0 +1,4 @@
+The scripts in this module are for deeper integration with the
+Collective Intelligence Framework (CIF) since Bro's Intel framework
+doesn't natively behave the same as CIF nor does it store and maintain
+the same data in all cases.

scripts/policy/integration/collective-intel/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/policy/integration/collective-intel/main.bro

@@ -0,0 +1,15 @@
+
+@load base/frameworks/intel
+
+module Intel;
+
+## These are some fields to add extended compatibility between Bro and the Collective
+## Intelligence Framework
+redef record Intel::MetaData += {
+	## Maps to the Impact field in the Collective Intelligence Framework.
+	cif_impact:     string &optional;
+	## Maps to the Severity field in the Collective Intelligence Framework.
+	cif_severity:   string &optional;
+	## Maps to the Confidence field in the Collective Intelligence Framework.
+	cif_confidence: double &optional;
+};

scripts/policy/protocols/http/detect-MHR.bro

@@ -1,4 +1,4 @@
-##! Detect file downloads over HTTP that have MD5 sums matching files in Team 
+##! Detect file downloads over HTTP that have MD5 sums matching files in Team
 ##! Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
 ##! By default, not all file transfers will have MD5 sums calculated.  Read the
 ##! documentation for the :doc:base/protocols/http/file-hash.bro script to see
@@ -10,11 +10,17 @@
 module HTTP;
 
 export {
-	redef enum Notice::Type += { 
+	redef enum Notice::Type += {
 		## The MD5 sum of a file transferred over HTTP matched in the
 		## malware hash registry.
 		Malware_Hash_Registry_Match
 	};
+
+	## The malware hash registry runs each malware sample through several A/V engines.
+	## Team Cymru returns a percentage to indicate how many A/V engines flagged the
+	## sample as malicious. This threshold allows you to require a minimum detection
+	## rate (default: 50%).
+	const MHR_threshold = 50 &redef;
 }
 
 event log_http(rec: HTTP::Info)
@@ -22,14 +28,15 @@ event log_http(rec: HTTP::Info)
 	if ( rec?$md5 )
 		{
 		local hash_domain = fmt("%s.malware.hash.cymru.com", rec$md5);
-		when ( local addrs = lookup_hostname(hash_domain) )
+		when ( local MHR_result = lookup_hostname_txt(hash_domain) )
 			{
-			# 127.0.0.2 indicates that the md5 sum was found in the MHR.
-			if ( 127.0.0.2 in addrs )
+			# Data is returned as "<dateFirstDetected> <detectionRate>"
+			local MHR_answer = split1(MHR_result, / /);
+			if ( length(MHR_answer) == 2 && to_count(MHR_answer[2]) >= MHR_threshold )
 				{
 				local url = HTTP::build_url_http(rec);
 				local message = fmt("%s %s %s", rec$id$orig_h, rec$md5, url);
-				NOTICE([$note=Malware_Hash_Registry_Match, 
+				NOTICE([$note=Malware_Hash_Registry_Match,
 				        $msg=message, $id=rec$id, $URL=url]);
 				}
 			}

scripts/policy/protocols/http/detect-intel.bro

@@ -1,21 +0,0 @@
-##! Intelligence based HTTP detections.  Not yet working!
-
-@load base/protocols/http/main
-@load base/protocols/http/utils
-@load base/frameworks/intel/main
-
-module HTTP;
-
-event log_http(rec: Info)
-	{
-	local url = HTTP::build_url(rec);
-	local query = [$str=url, $subtype="url", $or_tags=set("malicious", "malware")];
-	if ( Intel::matcher(query) )
-		{
-		local msg = fmt("%s accessed a malicious URL from the intelligence framework", rec$id$orig_h);
-		NOTICE([$note=Intel::Detection, 
-		        $msg=msg, 
-		        $sub=HTTP::build_url_http(rec), 
-		        $id=rec$id]);
-		}
-	}

scripts/policy/protocols/modbus/known-masters-slaves.bro

@@ -0,0 +1,58 @@
+##! Script for tracking known Modbus masters and slaves.
+##!
+##! .. todo: This script needs a lot of work.  What might be more interesting is to track
+##!          master/slave relationships based on commands sent and successful (non-exception)
+##!          responses.
+
+@load base/protocols/modbus
+
+module Known;
+
+export {
+	redef enum Log::ID += { MODBUS_LOG };
+
+	type ModbusDeviceType: enum {
+		MODBUS_MASTER,
+		MODBUS_SLAVE,
+	};
+
+	type ModbusInfo: record {
+		## The time the device was discovered.
+		ts:          time             &log;
+		## The IP address of the host.
+		host:        addr             &log;
+		## The type of device being tracked.
+		device_type: ModbusDeviceType &log;
+	};
+
+	## The Modbus nodes being tracked.
+	global modbus_nodes: set[addr, ModbusDeviceType] &create_expire=1day &redef;
+
+	## Event that can be handled to access the loggable record as it is sent
+	## on to the logging framework.
+	global log_known_modbus: event(rec: ModbusInfo);
+}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Known::MODBUS_LOG, [$columns=ModbusInfo, $ev=log_known_modbus]);
+	}
+
+event modbus_message(c: connection, headers: ModbusHeaders, is_orig: bool)
+	{
+	local master = c$id$orig_h;
+	local slave  = c$id$resp_h;
+
+	if ( [master, MODBUS_MASTER] !in modbus_nodes )
+		{
+		add modbus_nodes[master, MODBUS_MASTER];
+		Log::write(MODBUS_LOG, [$ts=network_time(), $host=master, $device_type=MODBUS_MASTER]);
+		}
+
+	if ( [slave, MODBUS_SLAVE] !in modbus_nodes )
+		{
+		add modbus_nodes[slave, MODBUS_SLAVE];
+		Log::write(MODBUS_LOG, [$ts=network_time(), $host=slave, $device_type=MODBUS_SLAVE]);
+		}
+
+	}

scripts/policy/protocols/modbus/track-memmap.bro

@@ -0,0 +1,105 @@
+##! This script tracks the memory map of holding (read/write) registers and logs
+##! changes as they are discovered.
+##!
+##! .. todo: Not all register reads and write functions are being supported yet.
+
+@load base/protocols/modbus
+@load base/utils/directions-and-hosts
+
+module Modbus;
+
+export {
+	redef enum Log::ID += { Modbus::REGISTER_CHANGE_LOG };
+
+	## The hosts that should have memory mapping enabled.
+	const track_memmap: Host = ALL_HOSTS &redef;
+
+	type MemmapInfo: record {
+		## Timestamp for the detected register change
+		ts:        time     &log;
+		## Unique ID for the connection
+		uid:       string   &log;
+		## Connection ID.
+		id:        conn_id  &log;
+		## The device memory offset.
+		register:  count    &log;
+		## The old value stored in the register.
+		old_val:   count    &log;
+		## The new value stored in the register.
+		new_val:   count    &log;
+		## The time delta between when the 'old_val' and 'new_val' were seen.
+		delta:     interval &log;
+	};
+
+	type RegisterValue: record {
+		last_set: time;
+		value:    count;
+	};
+
+	## Indexed on the device register value and yielding the register value.
+	type Registers: table[count] of RegisterValue;
+
+	## The memory map of slaves is tracked with this variable.
+	global device_registers: table[addr] of Registers;
+
+	## This event is generated every time a register is seen to be different than
+	## it was previously seen to be.
+	global changed_register: event(c: connection, register: count, old_val: count, new_val: count, delta: interval);
+}
+
+redef record Modbus::Info += {
+	track_address: count &default=0;
+};
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Modbus::REGISTER_CHANGE_LOG, [$columns=MemmapInfo]);
+	}
+
+event modbus_read_holding_registers_request(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
+	{
+	c$modbus$track_address = start_address+1;
+	}
+
+event modbus_read_holding_registers_response(c: connection, headers: ModbusHeaders, registers: ModbusRegisters)
+	{
+	local slave = c$id$resp_h;
+
+	if ( ! addr_matches_host(slave, track_memmap ) )
+		return;
+
+	if ( slave !in device_registers )
+		device_registers[slave] = table();
+
+	local slave_regs = device_registers[slave];
+	for ( i in registers )
+		{
+		if ( c$modbus$track_address in slave_regs )
+			{
+			if ( slave_regs[c$modbus$track_address]$value != registers[i] )
+				{
+				local delta = network_time() - slave_regs[c$modbus$track_address]$last_set;
+				event Modbus::changed_register(c, c$modbus$track_address, 
+				                               slave_regs[c$modbus$track_address]$value, registers[i],
+				                               delta);
+				
+				slave_regs[c$modbus$track_address]$last_set = network_time();
+				slave_regs[c$modbus$track_address]$value = registers[i];
+				}
+			}
+		else
+			{
+			local tmp_reg: RegisterValue = [$last_set=network_time(), $value=registers[i]];
+			slave_regs[c$modbus$track_address] = tmp_reg;
+			}
+
+		++c$modbus$track_address;
+		}
+	}
+
+event Modbus::changed_register(c: connection, register: count, old_val: count, new_val: count, delta: interval)
+	{
+	local rec: MemmapInfo = [$ts=network_time(), $uid=c$uid, $id=c$id, 
+	                         $register=register, $old_val=old_val, $new_val=new_val, $delta=delta];
+	Log::write(REGISTER_CHANGE_LOG, rec);
+	}

scripts/test-all-policy.bro

@@ -14,6 +14,16 @@
 # @load frameworks/control/controller.bro
 @load frameworks/dpd/detect-protocols.bro
 @load frameworks/dpd/packet-segment-logging.bro
+@load frameworks/intel/__load__.bro
+@load frameworks/intel/conn-established.bro
+@load frameworks/intel/dns.bro
+@load frameworks/intel/http-host-header.bro
+@load frameworks/intel/http-url.bro
+@load frameworks/intel/http-user-agents.bro
+@load frameworks/intel/smtp-url-extraction.bro
+@load frameworks/intel/smtp.bro
+@load frameworks/intel/ssl.bro
+@load frameworks/intel/where-locations.bro
 @load frameworks/metrics/conn-example.bro
 @load frameworks/metrics/http-example.bro
 @load frameworks/metrics/ssl-example.bro
@@ -22,6 +32,8 @@
 @load integration/barnyard2/__load__.bro
 @load integration/barnyard2/main.bro
 @load integration/barnyard2/types.bro
+@load integration/collective-intel/__load__.bro
+@load integration/collective-intel/main.bro
 @load misc/analysis-groups.bro
 @load misc/capture-loss.bro
 @load misc/loaded-scripts.bro
@@ -35,7 +47,6 @@
 @load protocols/dns/detect-external-names.bro
 @load protocols/ftp/detect.bro
 @load protocols/ftp/software.bro
-@load protocols/http/detect-intel.bro
 @load protocols/http/detect-MHR.bro
 @load protocols/http/detect-sqli.bro
 @load protocols/http/detect-webapps.bro
@@ -44,6 +55,8 @@
 @load protocols/http/software.bro
 @load protocols/http/var-extraction-cookies.bro
 @load protocols/http/var-extraction-uri.bro
+@load protocols/modbus/known-masters-slaves.bro
+@load protocols/modbus/track-memmap.bro
 @load protocols/smtp/blocklists.bro
 @load protocols/smtp/detect-suspicious-orig.bro
 @load protocols/smtp/software.bro