Merge branch 'master' into fastpath

scripts/CMakeLists.txt

@@ -1,18 +1,27 @@
 include(InstallPackageConfigFile)
 
 install(DIRECTORY ./ DESTINATION ${POLICYDIR} FILES_MATCHING
-        PATTERN "all.bro" EXCLUDE
-        PATTERN "site/local.bro" EXCLUDE
-        PATTERN "bro.init"
+        PATTERN "site/local*" EXCLUDE
         PATTERN "*.bro"
         PATTERN "*.sig"
-        PATTERN "*.osf"
+        PATTERN "*.fp"
 )
 
-# Install as a config file since the local.bro script is meant to be
+# Install all local* scripts as config files since they are meant to be
 # user modify-able.
 InstallPackageConfigFile(
     ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro
     ${POLICYDIR}/site
     local.bro)
-
+InstallPackageConfigFile(
+    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro
+    ${POLICYDIR}/site
+    local-manager.bro)
+InstallPackageConfigFile(
+    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro
+    ${POLICYDIR}/site
+    local-proxy.bro)
+InstallPackageConfigFile(
+    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro
+    ${POLICYDIR}/site
+    local-worker.bro)

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -10,6 +10,9 @@
 
 @prefixes += cluster-manager
 
+## Load the script for local site configuration for the manager node.
+@load site/local-manager
+
 ## Turn off remote logging since this is the manager and should only log here.
 redef Log::enable_remote_logging = F;
 

scripts/base/frameworks/cluster/nodes/proxy.bro

@@ -1,6 +1,9 @@
 
 @prefixes += cluster-proxy
 
+## Load the script for local site configuration for proxy nodes.
+@load site/local-proxy
+
 ## The proxy only syncs state; does not forward events.
 redef forward_remote_events = F;
 redef forward_remote_state_changes = T;

scripts/base/frameworks/cluster/nodes/worker.bro

@@ -1,6 +1,9 @@
 
 @prefixes += cluster-worker
 
+## Load the script for local site configuration for the worker nodes.
+@load site/local-worker
+
 ## Don't do any local logging.
 redef Log::enable_local_logging = F;
 

scripts/base/frameworks/cluster/setup-connections.bro

@@ -1,3 +1,4 @@
+
 module Cluster;
 
 event bro_init() &priority=9

scripts/base/frameworks/notice/__load__.bro

@@ -6,7 +6,8 @@
 @load ./actions/drop
 @load ./actions/email_admin
 @load ./actions/page
+@load ./actions/add-geodata
 
-# Load the script to add hostnames to emails by default.
-# NOTE: this exposes a memleak in async DNS lookups.
-#@load ./extend-email/hostnames
+# There shouldn't be any defaul toverhead from loading these since they 
+# *should* only do anything when notices have the ACTION_EMAIL action applied.
+@load ./extend-email/hostnames

scripts/base/frameworks/notice/actions/add-geodata.bro

@@ -0,0 +1,47 @@
+##! This script adds geographic location data to notices for the "remote"
+##! host in a connection.  It does make the assumption that one of the 
+##! addresses in a connection is "local" and one is "remote" which is 
+##! probably a safe assumption to make in most cases.  If both addresses
+##! are remote, it will use the $src address.
+
+module Notice;
+
+export {
+	redef enum Action += {
+		## Indicates that the notice should have geodata added for the
+		## "remote" host.  :bro:id:`Site::local_nets` must be defined
+		## in order for this to work.
+		ACTION_ADD_GEODATA
+	};
+	
+	redef record Info += {
+		## If libGeoIP support is built in, notices can have geographic
+		## information attached to them.
+		remote_location: geo_location  &log &optional;
+	};
+	
+	## Notice types which should have the "remote" location looked up.
+	## If GeoIP support is not built in, this does nothing.
+	const lookup_location_types: set[Notice::Type] = {} &redef;
+	
+	## Add a helper to the notice policy for looking up GeoIP data.
+	redef Notice::policy += {
+		[$pred(n: Notice::Info) = { return (n$note in Notice::lookup_location_types); },
+		 $priority = 10],
+	};
+}
+
+# This is handled at a high priority in case other notice handlers 
+# want to use the data.
+event notice(n: Notice::Info) &priority=10
+	{
+	if ( ACTION_ADD_GEODATA in n$actions &&
+	     |Site::local_nets| > 0 &&
+	     ! n?$remote_location )
+		{
+		if ( n?$src && ! Site::is_local_addr(n$src) )
+			n$remote_location = lookup_location(n$src);
+		else if ( n?$dst && ! Site::is_local_addr(n$dst) )
+			n$remote_location = lookup_location(n$dst);
+		}
+	}

scripts/base/init-bare.bro

@@ -509,7 +509,7 @@ const tunnel_port = 0/udp &redef;
 # packets and IP-level bytes transfered by each endpoint. If
 # true, these values are returned in the connection's endpoint
 # record val.
-const use_conn_size_analyzer = F &redef;
+const use_conn_size_analyzer = T &redef;
 
 const UDP_INACTIVE = 0;
 const UDP_ACTIVE = 1;	# means we've seen something from this endpoint

scripts/base/protocols/http/main.bro

@@ -214,7 +214,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 			c$http$response_content_length = extract_count(value);
 		else if ( name == "CONTENT-DISPOSITION" &&
 		          /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value )
-			c$http$filename = sub(value, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
+			c$http$filename = extract_filename_from_content_disposition(value);
 		}
 	}
 	

scripts/base/protocols/mime/file-extract.bro

@@ -16,11 +16,13 @@ export {
 		extract_file:         bool    &default=F;
 	
 		## Store the file handle here for the file currently being extracted.
-		extraction_file:      file    &optional;
-		
+		extraction_file:      file    &log &optional;
+	};
+
+	redef record State += {
 		## Store a count of the number of files that have been transferred in
 		## this conversation to create unique file names on disk.
-		num_extracted_files:  count   &optional;
+		num_extracted_files:  count   &default=0;
 	};
 }
 
@@ -34,7 +36,7 @@ event mime_segment_data(c: connection, length: count, data: string) &priority=3
 	{
 	if ( c$mime$extract_file && c$mime$content_len == 0 )
 		{
-		local suffix = fmt("%d.dat", ++c$mime$num_extracted_files);
+		local suffix = fmt("%d.dat", ++c$mime_state$num_extracted_files);
 		local fname = generate_extraction_filename(extraction_prefix, c, suffix);
 		c$mime$extraction_file = open(fname);
 		enable_raw_output(c$mime$extraction_file);
@@ -57,4 +59,4 @@ event mime_end_entity(c: connection) &priority=-3
 	if ( c$mime?$extraction_file )
 		close(c$mime$extraction_file);
 	}
-	
+	

scripts/base/protocols/smtp/__load__.bro

@@ -1 +1,3 @@
 @load ./main
+@load ./entities
+@load ./entities-excerpt

scripts/base/protocols/smtp/entities-excerpt.bro

@@ -0,0 +1,52 @@
+##! This script is for optionally adding a body excerpt to the SMTP
+##! entities log.
+
+@load ./entities
+
+module SMTP;
+
+export {
+	redef record SMTP::EntityInfo += {
+		## The entity body excerpt.
+		excerpt:    string &log &default="";
+		
+		## Internal tracking to know how much of the body should be included
+		## in the excerpt.
+		excerpt_len: count &optional;
+	};
+	
+	## This is the default value for how much of the entity body should be
+	## included for all MIME entities.
+	const default_entity_excerpt_len = 0 &redef;
+	
+	## This table defines how much of various entity bodies should be 
+	## included in excerpts.
+	const entity_excerpt_len: table[string] of count = {} 
+		&redef
+		&default = default_entity_excerpt_len;
+}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-1
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$content_len == 0 )
+		c$smtp$current_entity$excerpt_len = entity_excerpt_len[c$smtp$current_entity$mime_type];
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-2
+	{
+	if ( ! c?$smtp ) return;
+	
+	local ent = c$smtp$current_entity;
+	if ( ent$content_len < ent$excerpt_len )
+		{
+		if ( ent$content_len + length < ent$excerpt_len )
+			ent$excerpt = cat(ent$excerpt, data);
+		else
+			{
+			local x_bytes = ent$excerpt_len - ent$content_len;
+			ent$excerpt = cat(ent$excerpt, sub_bytes(data, 1, x_bytes));
+			}
+		}
+	}

scripts/base/protocols/smtp/entities.bro

@@ -0,0 +1,234 @@
+##! Analysis and logging for MIME entities found in SMTP sessions.
+
+@load base/utils/strings
+@load base/utils/files
+@load ./main
+
+module SMTP;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates that an MD5 sum was calculated for a MIME message.
+		MD5,
+	};
+
+	redef enum Log::ID += { SMTP_ENTITIES };
+
+	type EntityInfo: record {
+		## This is the timestamp of when the MIME content transfer began.
+		ts:               time    &log;
+		uid:              string  &log;
+		id:               conn_id &log;
+		## Internally generated "message id" that ties back to the particular
+		## message in the SMTP log where this entity was seen.
+		mid:              string  &log;
+		## The filename seen in the Content-Disposition header.
+		filename:         string  &log &optional;
+		## Track how many bytes of the MIME encoded file have been seen.
+		content_len:      count   &log &default=0;
+		## The mime type of the entity discovered through magic bytes identification.
+		mime_type:        string  &log &optional;
+		
+		## The calculated MD5 sum for the MIME entity.
+		md5:              string  &log &optional;
+		## Optionally calculate the file's MD5 sum.  Must be set prior to the 
+		## first data chunk being see in an event.
+		calc_md5:         bool    &default=F;
+		## This boolean value indicates if an MD5 sum is being calculated 
+		## for the current file transfer.
+		calculating_md5:  bool    &default=F;
+		
+		## Optionally write the file to disk.  Must be set prior to first 
+		## data chunk being seen in an event.
+		extract_file:     bool    &default=F;
+		## Store the file handle here for the file currently being extracted.
+		extraction_file:  file    &log &optional;
+	};
+
+	redef record Info += {
+		## The in-progress entity information.
+		current_entity:   EntityInfo &optional;
+	};
+
+	redef record State += {
+		## Store a count of the number of files that have been transferred in
+		## a conversation to create unique file names on disk.
+		num_extracted_files:  count   &default=0;
+		## Track the number of MIME encoded files transferred during a session.
+		mime_level:           count   &default=0;
+	};
+
+	## Generate MD5 sums for these filetypes.
+	const generate_md5 = /application\/x-dosexec/    # Windows and DOS executables
+	                   | /application\/x-executable/ # *NIX executable binary
+	                   &redef;
+
+	## Pattern of file mime types to extract from MIME bodies.
+	const extract_file_types = /NO_DEFAULT/ &redef;
+
+	## The on-disk prefix for files to be extracted from MIME entity bodies.
+	const extraction_prefix = "smtp-entity" &redef;
+
+	global log_mime: event(rec: EntityInfo);
+}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(SMTP_ENTITIES, [$columns=EntityInfo, $ev=log_mime]);
+	}
+
+function set_session(c: connection, new_entity: bool)
+	{
+	if ( ! c$smtp?$current_entity || new_entity )
+		{
+		local info: EntityInfo;
+		info$ts=network_time();
+		info$uid=c$uid;
+		info$id=c$id;
+		info$mid=c$smtp$mid;
+		
+		c$smtp$current_entity = info;
+		++c$smtp_state$mime_level;
+		}
+	}
+
+event mime_begin_entity(c: connection) &priority=10
+	{
+	if ( ! c?$smtp ) return;
+	
+	set_session(c, T);
+	}
+
+# This has priority -10 because other handlers need to know the current
+# content_len before it's updated by this handler.
+event mime_segment_data(c: connection, length: count, data: string) &priority=-10
+	{
+	if ( ! c?$smtp ) return;
+	
+	c$smtp$current_entity$content_len = c$smtp$current_entity$content_len + length;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=7
+    {
+	if ( ! c?$smtp ) return;
+	if ( c$smtp$current_entity$content_len == 0 )
+		c$smtp$current_entity$mime_type = split1(identify_data(data, T), /;/)[1];
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+
+	if ( c$smtp$current_entity$content_len == 0 )
+		{
+		if ( generate_md5 in c$smtp$current_entity$mime_type )
+			c$smtp$current_entity$calc_md5 = T;
+
+		if ( c$smtp$current_entity$calc_md5 )
+			{
+			c$smtp$current_entity$calculating_md5 = T;
+			md5_hash_init(c$id);
+			}
+		}
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		md5_hash_update(c$id, data);
+}
+
+## In the event of a content gap during the MIME transfer, detect the state for
+## the MD5 sum calculation and stop calculating the MD5 since it would be
+## incorrect anyway.
+event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5
+	{
+	if ( is_orig || ! c?$smtp || ! c$smtp?$current_entity ) return;
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		{
+		c$smtp$current_entity$calculating_md5 = F;
+		md5_hash_finish(c$id);
+		}
+	}
+
+event mime_end_entity(c: connection) &priority=-3
+    {
+	# TODO: this check is only due to a bug in mime_end_entity that
+	#       causes the event to be generated twice for the same real event.
+	if ( ! c?$smtp || ! c$smtp?$current_entity )
+		return;
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		{
+		c$smtp$current_entity$md5 = md5_hash_finish(c$id);
+
+		NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h),
+				$sub=c$smtp$current_entity$md5, $conn=c]);
+		}
+	}
+
+event mime_one_header(c: connection, h: mime_header_rec)
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( h$name == "CONTENT-DISPOSITION" &&
+	     /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value )
+		c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value);
+	}
+
+event mime_end_entity(c: connection) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+
+	# This check and the delete below are just to cope with a bug where
+	# mime_end_entity can be generated multiple times for the same event.
+	if ( ! c$smtp?$current_entity )
+		return;
+
+	# Only log is there was some content.
+	if ( c$smtp$current_entity$content_len > 0 )
+		Log::write(SMTP_ENTITIES, c$smtp$current_entity);
+
+	delete c$smtp$current_entity;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=5
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( extract_file_types in c$smtp$current_entity$mime_type )
+		c$smtp$current_entity$extract_file = T;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=3
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$extract_file && 
+	     c$smtp$current_entity$content_len == 0 )
+		{
+		local suffix = fmt("%d.dat", ++c$smtp_state$num_extracted_files);
+		local fname = generate_extraction_filename(extraction_prefix, c, suffix);
+		c$smtp$current_entity$extraction_file = open(fname);
+		enable_raw_output(c$smtp$current_entity$extraction_file);
+		}
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$extract_file && c$smtp$current_entity?$extraction_file )
+		print c$smtp$current_entity$extraction_file, data;
+	}
+
+event mime_end_entity(c: connection) &priority=-3
+	{
+	if ( ! c?$smtp ) return;
+	
+	# TODO: this check is only due to a bug in mime_end_entity that
+	#       causes the event to be generated twice for the same real event.
+	if ( ! c$smtp?$current_entity )
+		return;
+
+	if ( c$smtp$current_entity?$extraction_file )
+		close(c$smtp$current_entity$extraction_file);
+	}

scripts/base/protocols/smtp/main.bro

@@ -4,17 +4,14 @@ module SMTP;
 export {
 	redef enum Log::ID += { SMTP };
 
-	redef enum Notice::Type += { 
-		## Indicates that the server sent a reply mentioning an SMTP block list.
-		BL_Error_Message, 
-		## Indicates the client's address is seen in the block list error message.
-		BL_Blocked_Host, 
-	};
-
 	type Info: record {
 		ts:                time            &log;
 		uid:               string          &log;
 		id:                conn_id         &log;
+		## This is an internally generated "message id" that can be used to
+		## map between SMTP messages and MIME entities in the SMTP entities
+		## log.
+		mid:               string          &log;
 		helo:              string          &log &optional;
 		mailfrom:          string          &log &optional;
 		rcptto:            set[string]     &log &optional;
@@ -30,19 +27,13 @@ export {
 		second_received:   string          &log &optional;
 		## The last message the server sent to the client.
 		last_reply:        string          &log &optional;
-		files:             set[string]     &log &optional;
 		path:              vector of addr  &log &optional;
 		user_agent:        string          &log &optional;
 		
-		## Indicate if this session is currently transmitting SMTP message 
-		## envelope headers.
-		in_headers:        bool            &default=F;
 		## Indicate if the "Received: from" headers should still be processed.
 		process_received_from: bool        &default=T;
-		## Maintain the current header for cases where there is header wrapping.
-		current_header:    string          &default="";
-		## Indicate when the message is logged and no longer applicable.
-		done:              bool            &default=F;
+		## Indicates if client activity has been seen, but not yet logged
+		has_client_activity:  bool            &default=F;
 	};
 	
 	type State: record {
@@ -61,26 +52,7 @@ export {
 	##    ALL_HOSTS - always capture the entire path.
 	##    NO_HOSTS - never capture the path.
 	const mail_path_capture = ALL_HOSTS &redef;
-	
-	# This matches content in SMTP error messages that indicate some
-	# block list doesn't like the connection/mail.
-	const bl_error_messages = 
-	    /spamhaus\.org\//
-	  | /sophos\.com\/security\//
-	  | /spamcop\.net\/bl/
-	  | /cbl\.abuseat\.org\// 
-	  | /sorbs\.net\// 
-	  | /bsn\.borderware\.com\//
-	  | /mail-abuse\.com\//
-	  | /b\.barracudacentral\.com\//
-	  | /psbl\.surriel\.com\// 
-	  | /antispam\.imp\.ch\// 
-	  | /dyndns\.com\/.*spam/
-	  | /rbl\.knology\.net\//
-	  | /intercept\.datapacket\.net\//
-	  | /uceprotect\.net\//
-	  | /hostkarma\.junkemailfilter\.com\// &redef;
-	
+		
 	global log_smtp: event(rec: Info);
 	
 	## Configure the default ports for SMTP analysis.
@@ -121,6 +93,7 @@ function new_smtp_log(c: connection): Info
 	l$ts=network_time();
 	l$uid=c$uid;
 	l$id=c$id;
+	l$mid=unique_id("@");
 	if ( c?$smtp_state && c$smtp_state?$helo )
 		l$helo = c$smtp_state$helo;
 	
@@ -136,26 +109,23 @@ function set_smtp_session(c: connection)
 	if ( ! c?$smtp_state )
 		c$smtp_state = [];
 	
-	if ( ! c?$smtp || c$smtp$done )
-		{
+	if ( ! c?$smtp )
 		c$smtp = new_smtp_log(c);
-		}
 	}
 
-
 function smtp_message(c: connection)
 	{
-	Log::write(SMTP, c$smtp);
-
-	c$smtp$done = T;
-	# Track the number of messages seen in this session.
-	++c$smtp_state$messages_transferred;
+	if ( c$smtp$has_client_activity )
+		Log::write(SMTP, c$smtp);
 	}
 	
 event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=5
 	{
 	set_smtp_session(c);
 	local upper_command = to_upper(command);
+
+	if ( upper_command != "QUIT" )
+		c$smtp$has_client_activity = T;
 	
 	if ( upper_command == "HELO" || upper_command == "EHLO" )
 		{
@@ -172,26 +142,11 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 
 	else if ( upper_command == "MAIL" && /^[fF][rR][oO][mM]:/ in arg )
 		{
-		# In case this is not the first message in a session we want to 
-		# essentially write out a log, clear the session tracking, and begin
-		# new session tracking.
-		if ( c$smtp_state$messages_transferred > 0 )
-			{
-			smtp_message(c);
-			set_smtp_session(c);
-			}
-		
 		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
 		c$smtp$mailfrom = split1(partially_done, /[[:blank:]]?/)[1];
 		}
-		
-	else if ( upper_command == "DATA" )
-		{
-		c$smtp$in_headers = T;
-		}
 	}
 	
-
 event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                  msg: string, cont_resp: bool) &priority=5
 	{
@@ -199,169 +154,98 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
 	
 	# This continually overwrites, but we want the last reply,
 	# so this actually works fine.
-	if ( code != 421 && code >= 400 )
+	c$smtp$last_reply = fmt("%d %s", code, msg);
+	}
+
+event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
+                 msg: string, cont_resp: bool) &priority=-5
+	{
+	set_smtp_session(c);
+	if ( cmd == "." )
 		{
-		c$smtp$last_reply = fmt("%d %s", code, msg);
-
-		# Raise a notice when an SMTP error about a block list is discovered.
-		if ( bl_error_messages in msg )
-			{
-			local note = BL_Error_Message;
-			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);
-
-			# Determine if the originator's IP address is in the message.
-			local ips = find_ip_addresses(msg);
-			local text_ip = "";
-			if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h )
-				{
-				note = BL_Blocked_Host;
-				message = fmt("%s is on an SMTP block list", c$id$orig_h);
-				}
-			
-			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]);
-			}
+		# Track the number of messages seen in this session.
+		++c$smtp_state$messages_transferred;
+		smtp_message(c);
+		c$smtp = new_smtp_log(c);
 		}
 	}
 
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=5
+event mime_one_header(c: connection, h: mime_header_rec) &priority=5
 	{
-	# Is there something we should be handling from the server?
-	if ( ! is_orig ) return;
-	
-	set_smtp_session(c);
-	
-	if ( ! c$smtp$in_headers )
-		{
-		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
-			{
-			if ( ! c$smtp?$files )
-				c$smtp$files = set();
-			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
-			add c$smtp$files[data];
-			}
-		return;
-		}
+	if ( ! c?$smtp ) return;
+	c$smtp$has_client_activity = T;
 
-	if ( /^[[:blank:]]*$/ in data )
-		c$smtp$in_headers = F;
+	if ( h$name == "MESSAGE-ID" )
+		c$smtp$msg_id = h$value;
 
-	# This is to reconstruct headers that tend to wrap around.
-	if ( /^[[:blank:]]/ in data )
-		{
-		# Remove all but a single space at the beginning (this seems to follow
-		# the most common behavior).
-		data = sub(data, /^[[:blank:]]*/, " ");
-		if ( c$smtp$current_header == "MESSAGE-ID" )
-			c$smtp$msg_id += data;
-		else if ( c$smtp$current_header == "RECEIVED" )
-			c$smtp$first_received += data;
-		else if ( c$smtp$current_header == "IN-REPLY-TO" )
-			c$smtp$in_reply_to += data;
-		else if ( c$smtp$current_header == "SUBJECCT" )
-			c$smtp$subject += data;
-		else if ( c$smtp$current_header == "FROM" )
-			c$smtp$from += data;
-		else if ( c$smtp$current_header == "REPLY-TO" )
-			c$smtp$reply_to += data;
-		else if ( c$smtp$current_header == "USER-AGENT" )
-			c$smtp$user_agent += data;
-		return;
-		}
-	# Once there isn't a line starting with a blank, we're not continuing a 
-	# header anymore.
-	c$smtp$current_header = "";
-	
-	local header_parts = split1(data, /:[[:blank:]]*/);
-	# TODO: do something in this case?  This would definitely be odd.
-	# Header wrapping needs to be handled more elegantly.  This will happen
-	# if the header value is wrapped immediately after the header key.
-	if ( |header_parts| != 2 )
-		return;
-	
-	local header_key = to_upper(header_parts[1]);
-	c$smtp$current_header = header_key;
-	
-	local header_val = header_parts[2];
-	
-	if ( header_key == "MESSAGE-ID" )
-		c$smtp$msg_id = header_val;
-	
-	else if ( header_key == "RECEIVED" )
+	else if ( h$name == "RECEIVED" )
 		{
 		if ( c$smtp?$first_received )
 			c$smtp$second_received = c$smtp$first_received;
-		c$smtp$first_received = header_val;
+		c$smtp$first_received = h$value;
 		}
-	
-	else if ( header_key == "IN-REPLY-TO" )
-		c$smtp$in_reply_to = header_val;
-	
-	else if ( header_key == "DATE" )
-		c$smtp$date = header_val;
-	
-	else if ( header_key == "FROM" )
-		c$smtp$from = header_val;
-	
-	else if ( header_key == "TO" )
+
+	else if ( h$name == "IN-REPLY-TO" )
+		c$smtp$in_reply_to = h$value;
+
+	else if ( h$name == "SUBJECT" )
+		c$smtp$subject = h$value;
+
+	else if ( h$name == "FROM" )
+		c$smtp$from = h$value;
+
+	else if ( h$name == "REPLY-TO" )
+		c$smtp$reply_to = h$value;
+
+	else if ( h$name == "DATE" )
+		c$smtp$date = h$value;
+
+	else if ( h$name == "TO" )
 		{
 		if ( ! c$smtp?$to )
 			c$smtp$to = set();
-		add c$smtp$to[header_val];
+		add c$smtp$to[h$value];
 		}
-	
-	else if ( header_key == "REPLY-TO" )
-		c$smtp$reply_to = header_val;
-	
-	else if ( header_key == "SUBJECT" )
-		c$smtp$subject = header_val;
 
-	else if ( header_key == "X-ORIGINATING-IP" )
+	else if ( h$name == "X-ORIGINATING-IP" )
 		{
-		local addresses = find_ip_addresses(header_val);
+		local addresses = find_ip_addresses(h$value);
 		if ( 1 in addresses )
 			c$smtp$x_originating_ip = to_addr(addresses[1]);
 		}
 	
-	else if ( header_key == "X-MAILER" || 
-	          header_key == "USER-AGENT" ||
-	          header_key == "X-USER-AGENT" )
-		{
-		c$smtp$user_agent = header_val;
-		# Explicitly set the current header here because there are several 
-		# headers bulked under this same key.
-		c$smtp$current_header = "USER-AGENT";
-		}
+	else if ( h$name == "X-MAILER" ||
+	          h$name == "USER-AGENT" ||
+	          h$name == "X-USER-AGENT" )
+		c$smtp$user_agent = h$value;
 	}
 	
 # This event handler builds the "Received From" path by reading the 
 # headers in the mail
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=3
+event mime_one_header(c: connection, h: mime_header_rec) &priority=3
 	{
 	# If we've decided that we're done watching the received headers for
 	# whatever reason, we're done.  Could be due to only watching until 
 	# local addresses are seen in the received from headers.
-	if ( c$smtp$current_header != "RECEIVED" ||
-	     ! c$smtp$process_received_from )
+	if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from )
 		return;
-		
-	local text_ip = find_address_in_smtp_header(data);
+
+	local text_ip = find_address_in_smtp_header(h$value);
 	if ( text_ip == "" )
 		return;
 	local ip = to_addr(text_ip);
-	
+
 	if ( ! addr_matches_host(ip, mail_path_capture) && 
 	     ! Site::is_private_addr(ip) )
 		{
 		c$smtp$process_received_from = F;
 		}
-	
 	if ( c$smtp$path[|c$smtp$path|-1] != ip )
 		c$smtp$path[|c$smtp$path|] = ip;
 	}
 
-
 event connection_state_remove(c: connection) &priority=-5
 	{
-	if ( c?$smtp && ! c$smtp$done )
+	if ( c?$smtp )
 		smtp_message(c);
 	}

scripts/base/protocols/ssh/main.bro

@@ -1,74 +1,58 @@
+##! Base SSH analysis script.  The heuristic to blindly determine success or 
+##! failure for SSH connections is implemented here.  At this time, it only
+##! uses the size of the data being returned from the server to make the
+##! heuristic determination about success of the connection.  
+##! Requires that :bro:id:`use_conn_size_analyzer` is set to T!  The heuristic
+##! is not attempted if the connection size analyzer isn't enabled.
 
 module SSH;
 
 export {
 	redef enum Log::ID += { SSH };
 
-	redef enum Notice::Type += {
-		Login,
-		Password_Guessing,
-		Login_By_Password_Guesser,
-		Login_From_Interesting_Hostname,
-		Bytecount_Inconsistency,
-	};
-
 	type Info: record {
 		ts:              time         &log;
 		uid:             string       &log;
 		id:              conn_id      &log;
+		## Indicates if the login was heuristically guessed to be "success"
+		## or "failure".
 		status:          string       &log &optional;
-		direction:       string       &log &optional;
-		remote_location: geo_location &log &optional;
+		## Direction of the connection.  If the client was a local host 
+		## logging into an external host, this would be OUTBOUD.  INBOUND
+		## would be set for the opposite situation.
+		# TODO: handle local-local and remote-remote better.
+		direction:       Direction    &log &optional;
+		## The software string given by the client.
 		client:          string       &log &optional;
+		## The software string given by the server.
 		server:          string       &log &optional;
+		## The amount of data returned from the server.  This is currently
+		## the only measure of the success heuristic and it is logged to 
+		## assist analysts looking at the logs to make their own determination
+		## about the success on a case-by-case basis.
 		resp_size:       count        &log &default=0;
 		
 		## Indicate if the SSH session is done being watched.
 		done:            bool         &default=F;
 	};
-
-	const password_guesses_limit = 30 &redef;
 	
-	# The size in bytes at which the SSH connection is presumed to be
-	# successful.
+	## The size in bytes at which the SSH connection is presumed to be
+	## successful.
 	const authentication_data_size = 5500 &redef;
 	
-	# The amount of time to remember presumed non-successful logins to build
-	# model of a password guesser.
-	const guessing_timeout = 30 mins &redef;
-
-	# The set of countries for which you'd like to throw notices upon successful login
-	#   requires Bro compiled with libGeoIP support
-	const watched_countries: set[string] = {"RO"} &redef;
-
-	# Strange/bad host names to originate successful SSH logins
-	const interesting_hostnames =
-			/^d?ns[0-9]*\./ |
-			/^smtp[0-9]*\./ |
-			/^mail[0-9]*\./ |
-			/^pop[0-9]*\./  |
-			/^imap[0-9]*\./ |
-			/^www[0-9]*\./  |
-			/^ftp[0-9]*\./  &redef;
-
-	# This is a table with orig subnet as the key, and subnet as the value.
-	const ignore_guessers: table[subnet] of subnet &redef;
-	
-	# If true, we tell the event engine to not look at further data
-	# packets after the initial SSH handshake. Helps with performance
-	# (especially with large file transfers) but precludes some
-	# kinds of analyses (e.g., tracking connection size).
+	## If true, we tell the event engine to not look at further data
+	## packets after the initial SSH handshake. Helps with performance
+	## (especially with large file transfers) but precludes some
+	## kinds of analyses (e.g., tracking connection size).
 	const skip_processing_after_detection = F &redef;
 	
-	# Keeps count of how many rejections a host has had
-	global password_rejections: table[addr] of TrackCount 
-		&write_expire=guessing_timeout
-		&synchronized;
-
-	# Keeps track of hosts identified as guessing passwords
-	# TODO: guessing_timeout doesn't work correctly here.  If a user redefs
-	#       the variable, it won't take effect.
-	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized;
+	## This event is generated when the heuristic thinks that a login
+	## was successful.
+	global heuristic_successful_login: event(c: connection);
+	
+	## This event is generated when the heuristic thinks that a login
+	## failed.
+	global heuristic_failed_login: event(c: connection);
 	
 	global log_ssh: event(rec: Info);
 }
@@ -106,116 +90,51 @@ function check_ssh_connection(c: connection, done: bool)
 	
 	# If this is still a live connection and the byte count has not
 	# crossed the threshold, just return and let the resheduled check happen later.
-	if ( !done && c$resp$size < authentication_data_size )
+	if ( !done && c$resp$num_bytes_ip < authentication_data_size )
 		return;
 
 	# Make sure the server has sent back more than 50 bytes to filter out
 	# hosts that are just port scanning.  Nothing is ever logged if the server
 	# doesn't send back at least 50 bytes.
-	if ( c$resp$size < 50 )
+	if ( c$resp$num_bytes_ip < 50 )
 		return;
 
-	local status = "failure";
-	local direction = Site::is_local_addr(c$id$orig_h) ? "to" : "from";
-	local location: geo_location;
-	location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
+	c$ssh$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND : INBOUND;
+	c$ssh$resp_size = c$resp$num_bytes_ip;
 	
-	if ( done && c$resp$size < authentication_data_size )
+	if ( c$resp$num_bytes_ip < authentication_data_size )
 		{
-		# presumed failure
-		if ( c$id$orig_h !in password_rejections )
-			password_rejections[c$id$orig_h] = new_track_count();
-			
-		# Track the number of rejections
-		if ( !(c$id$orig_h in ignore_guessers &&
-		       c$id$resp_h in ignore_guessers[c$id$orig_h]) )
-			++password_rejections[c$id$orig_h]$n;
-			
-		if ( default_check_threshold(password_rejections[c$id$orig_h]) )
-			{
-			add password_guessers[c$id$orig_h];
-			NOTICE([$note=Password_Guessing,
-			        $conn=c,
-			        $msg=fmt("SSH password guessing by %s", c$id$orig_h),
-			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n),
-			        $n=password_rejections[c$id$orig_h]$n]);
-			}
-		} 
-	# TODO: This is to work around a quasi-bug in Bro which occasionally 
-	#       causes the byte count to be oversized.
-	#   Watch for Gregors work that adds an actual counter of bytes transferred.
-	else if ( c$resp$size < 20000000 ) 
+		c$ssh$status  = "failure";
+		event SSH::heuristic_failed_login(c);
+		}
+	else
 		{ 
 		# presumed successful login
-		status = "success";
-		c$ssh$done = T;
-
-		if ( c$id$orig_h in password_rejections &&
-		     password_rejections[c$id$orig_h]$n > password_guesses_limit &&
-		     c$id$orig_h !in password_guessers )
-			{
-			add password_guessers[c$id$orig_h];
-			NOTICE([$note=Login_By_Password_Guesser,
-			        $conn=c,
-			        $n=password_rejections[c$id$orig_h]$n,
-			        $msg=fmt("Successful SSH login by password guesser %s", c$id$orig_h),
-			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n)]);
-			}
-		
-		local message = fmt("SSH login %s %s \"%s\" \"%s\" %f %f %s (triggered with %d bytes)",
-		              direction, location$country_code, location$region, location$city,
-		              location$latitude, location$longitude,
-		              id_string(c$id), c$resp$size);
-		NOTICE([$note=Login,
-		        $conn=c,
-		        $msg=message,
-		        $sub=location$country_code]);
-		
-		# Check to see if this login came from an interesting hostname
-		when ( local hostname = lookup_addr(c$id$orig_h) )
-			{
-			if ( interesting_hostnames in hostname )
-				{
-				NOTICE([$note=Login_From_Interesting_Hostname,
-				        $conn=c,
-				        $msg=fmt("Strange login from %s", hostname),
-				        $sub=hostname]);
-				}
-			}
-			
-		if ( location$country_code in watched_countries )
-			{
-			
-			}
-			
+		c$ssh$status = "success";
+		event SSH::heuristic_successful_login(c);
 		}
-	else if ( c$resp$size >= 200000000 ) 
-		{
-		NOTICE([$note=Bytecount_Inconsistency,
-		        $conn=c,
-		        $msg="During byte counting in SSH analysis, an overly large value was seen.",
-		        $sub=fmt("%d",c$resp$size)]);
-		}
-
-	c$ssh$remote_location = location;
-	c$ssh$status = status;
-	c$ssh$direction = direction;
-	c$ssh$resp_size = c$resp$size;
-	
-	Log::write(SSH, c$ssh);
 	
 	# Set the "done" flag to prevent the watching event from rescheduling
 	# after detection is done.
-	c$ssh$done;
+	c$ssh$done=T;
 	
-	# Stop watching this connection, we don't care about it anymore.
 	if ( skip_processing_after_detection )
 		{
+		# Stop watching this connection, we don't care about it anymore.
 		skip_further_processing(c$id);
 		set_record_packets(c$id, F);
 		}
 	}
 
+event SSH::heuristic_successful_login(c: connection) &priority=-5
+	{
+	Log::write(SSH, c$ssh);
+	}
+event SSH::heuristic_failed_login(c: connection) &priority=-5
+	{
+	Log::write(SSH, c$ssh);
+	}
+
 event connection_state_remove(c: connection) &priority=-5
 	{
 	if ( c?$ssh )
@@ -226,7 +145,7 @@ event ssh_watcher(c: connection)
 	{
 	local id = c$id;
 	# don't go any further if this connection is gone already!
-	if ( !connection_exists(id) )
+	if ( ! connection_exists(id) )
 		return;
 
 	check_ssh_connection(c, F);
@@ -244,5 +163,9 @@ event ssh_client_version(c: connection, version: string) &priority=5
 	{
 	set_session(c);
 	c$ssh$client = version;
-	schedule +15secs { ssh_watcher(c) };
+	
+	# The heuristic detection for SSH relies on the ConnSize analyzer.
+	# Don't do the heuristics if it's disabled.
+	if ( use_conn_size_analyzer )
+		schedule +15secs { ssh_watcher(c) };
 	}

scripts/base/utils/files.bro

@@ -12,4 +12,15 @@ function generate_extraction_filename(prefix: string, c: connection, suffix: str
 		conn_info = fmt("%s_%s", conn_info, suffix);
 		
 	return conn_info;
-	}
+	}
+	
+## For CONTENT-DISPOSITION headers, this function can be used to extract 
+## the filename.
+function extract_filename_from_content_disposition(data: string): string
+	{
+	local filename = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
+	# Remove quotes around the filename if they are there.
+	if ( /^\"/ in filename )
+		filename =  split_n(filename, /\"/, F, 2)[2];
+	return filename;
+	}

scripts/policy/misc/profiling.bro

@@ -0,0 +1,19 @@
+##! Turns on profiling of Bro resource consumption.
+
+module Profiling;
+
+redef profiling_file = open_log_file("prof");
+
+export {
+	## Cheap profiling every 15 seconds.
+	redef profiling_interval = 15 secs &redef;
+}
+
+# Expensive profiling every 5 minutes.
+redef expensive_profiling_multiple = 20;
+
+event bro_init()
+	{
+	set_buf(profiling_file, F);
+	}
+

scripts/policy/protocols/http/detect-MHR.bro

@@ -1,7 +1,7 @@
 ##! This script takes MD5 sums of files transferred over HTTP and checks them with
 ##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
 ##! By default, not all file transfers will have MD5 sums calculated.  Read the
-##! documentation for the protocols/http/file-hash.bro script to see how to 
+##! documentation for the base/protocols/http/file-hash.bro script to see how to 
 ##! configure which transfers will have hashes calculated.
 
 export {

scripts/policy/protocols/smtp/blocklists.bro

@@ -0,0 +1,58 @@
+
+@load base/protocols/smtp
+
+module SMTP;
+
+export {
+	redef enum Notice::Type += { 
+		## Indicates that the server sent a reply mentioning an SMTP block list.
+		Blocklist_Error_Message,
+		## Indicates the client's address is seen in the block list error message.
+		Blocklist_Blocked_Host,
+	};
+
+	# This matches content in SMTP error messages that indicate some
+	# block list doesn't like the connection/mail.
+	const blocklist_error_messages = 
+		  /spamhaus\.org\//
+		| /sophos\.com\/security\//
+		| /spamcop\.net\/bl/
+		| /cbl\.abuseat\.org\// 
+		| /sorbs\.net\// 
+		| /bsn\.borderware\.com\//
+		| /mail-abuse\.com\//
+		| /b\.barracudacentral\.com\//
+		| /psbl\.surriel\.com\// 
+		| /antispam\.imp\.ch\// 
+		| /dyndns\.com\/.*spam/
+		| /rbl\.knology\.net\//
+		| /intercept\.datapacket\.net\//
+		| /uceprotect\.net\//
+		| /hostkarma\.junkemailfilter\.com\// &redef;
+	
+}
+
+event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
+                 msg: string, cont_resp: bool) &priority=3
+	{
+	if ( code >= 400 && code != 421 )
+		{
+		# Raise a notice when an SMTP error about a block list is discovered.
+		if ( blocklist_error_messages in msg )
+			{
+			local note = Blocklist_Error_Message;
+			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);
+
+			# Determine if the originator's IP address is in the message.
+			local ips = find_ip_addresses(msg);
+			local text_ip = "";
+			if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h )
+				{
+				note = Blocklist_Blocked_Host;
+				message = fmt("%s is on an SMTP block list", c$id$orig_h);
+				}
+			
+			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]);
+			}
+		}
+	}

scripts/policy/protocols/smtp/software.bro

@@ -43,10 +43,10 @@ export {
 	                   | /ZimbraWebClient/ &redef;
 }
 
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=4
+event mime_one_header(c: connection, h: mime_header_rec) &priority=4
 	{
-	if ( c$smtp$current_header == "USER-AGENT" &&
-	     webmail_user_agents in c$smtp$user_agent )
+	if ( ! c?$smtp ) return;
+	if ( h$name == "USER-AGENT" && webmail_user_agents in c$smtp$user_agent )
 		c$smtp$is_webmail = T;
 	}
 

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -0,0 +1,79 @@
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates that a host has been identified as crossing the 
+		## :bro:id:`password_guesses_limit` threshold with heuristically
+		## determined failed logins.
+		Password_Guessing,
+		## Indicates that a host previously identified as a "password guesser"
+		## has now had a heuristically successful login attempt.
+		Login_By_Password_Guesser,
+	};
+
+	## The number of failed SSH connections before a host is designated as
+	## guessing passwords.
+	const password_guesses_limit = 30 &redef;
+
+	## The amount of time to remember presumed non-successful logins to build
+	## model of a password guesser.
+	const guessing_timeout = 30 mins &redef;
+
+	## This value can be used to exclude hosts or entire networks from being 
+	## tracked as potential "guessers".  There are cases where the success
+	## heuristic fails and this acts as the whitelist.  The index represents 
+	## client subnets and the yield value represents server subnets.
+	const ignore_guessers: table[subnet] of subnet &redef;
+
+	## Keeps count of how many rejections a host has had.
+	global password_rejections: table[addr] of TrackCount 
+		&write_expire=guessing_timeout
+		&synchronized;
+
+	## Keeps track of hosts identified as guessing passwords.
+	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized;
+}
+
+event SSH::heuristic_successful_login(c: connection)
+	{
+	local id = c$id;
+	
+	# TODO: this should be migrated to the metrics framework.
+	if ( id$orig_h in password_rejections &&
+	     password_rejections[id$orig_h]$n > password_guesses_limit &&
+	     id$orig_h !in password_guessers )
+		{
+		add password_guessers[id$orig_h];
+		NOTICE([$note=Login_By_Password_Guesser,
+		        $conn=c,
+		        $n=password_rejections[id$orig_h]$n,
+		        $msg=fmt("Successful SSH login by password guesser %s", id$orig_h),
+		        $sub=fmt("%d failed logins", password_rejections[id$orig_h]$n)]);
+		}
+	}
+
+event SSH::heuristic_failed_login(c: connection)
+	{
+	local id = c$id;
+	
+	# presumed failure
+	if ( id$orig_h !in password_rejections )
+		password_rejections[id$orig_h] = new_track_count();
+	
+	# Track the number of rejections
+	# TODO: this should be migrated to the metrics framework.
+	if ( ! (id$orig_h in ignore_guessers &&
+	        id$resp_h in ignore_guessers[id$orig_h]) )
+		++password_rejections[id$orig_h]$n;
+	
+	if ( default_check_threshold(password_rejections[id$orig_h]) )
+		{
+		add password_guessers[id$orig_h];
+		NOTICE([$note=Password_Guessing,
+		        $conn=c,
+		        $msg=fmt("SSH password guessing by %s", id$orig_h),
+		        $sub=fmt("%d apparently failed logins", password_rejections[id$orig_h]$n),
+		        $n=password_rejections[id$orig_h]$n]);
+		}
+	}

scripts/policy/protocols/ssh/geo-data.bro

@@ -0,0 +1,39 @@
+##! This implements all of the additional information and geodata detections 
+##! for SSH analysis.
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## If an SSH login is seen to or from a "watched" country based on the
+		## :bro:id:`SSH::watched_countries` variable then this notice will
+		## be generated.
+		Login_From_Watched_Country,
+	};
+	
+	## The set of countries for which you'd like to throw notices upon 
+	## successful login
+	const watched_countries: set[string] = {"RO"} &redef;
+
+	redef record Info += {
+		## Add geographic data related to the "remote" host of the connection.
+		remote_location: geo_location &log &optional;
+	};
+}
+
+event SSH::heuristic_successful_login(c: connection) &priority=5
+	{
+	local location: geo_location;
+	location = (c$ssh$direction == OUTBOUND) ? 
+		lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
+	
+	# Add the location data to the SSH record.
+	c$ssh$remote_location = location;
+	
+	if ( location$country_code in watched_countries )
+		{
+		NOTICE([$note=Login_From_Watched_Country,
+		        $conn=c,
+		        $msg=fmt("SSH login from watched country: %s", location$country_code)]);
+		}
+	}

scripts/policy/protocols/ssh/interesting-hostnames.bro

@@ -0,0 +1,50 @@
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## Generated if a login originates from a host matched by the 
+		## :bro:id:`interesting_hostnames` regular expression.
+		Login_From_Interesting_Hostname,
+		## Generated if a login goes to a host matched by the 
+		## :bro:id:`interesting_hostnames` regular expression.
+		Login_To_Interesting_Hostname,
+	};
+	
+	## Strange/bad host names to see successful SSH logins from or to.
+	const interesting_hostnames =
+			/^d?ns[0-9]*\./ |
+			/^smtp[0-9]*\./ |
+			/^mail[0-9]*\./ |
+			/^pop[0-9]*\./  |
+			/^imap[0-9]*\./ |
+			/^www[0-9]*\./  |
+			/^ftp[0-9]*\./  &redef;
+}
+
+event SSH::heuristic_successful_login(c: connection)
+	{
+	# Check to see if this login came from an interesting hostname.
+	when ( local orig_hostname = lookup_addr(c$id$orig_h) )
+		{
+		if ( interesting_hostnames in orig_hostname )
+			{
+			NOTICE([$note=Login_From_Interesting_Hostname,
+			        $conn=c,
+			        $msg=fmt("Interesting login from hostname: %s", orig_hostname),
+			        $sub=orig_hostname]);
+			}
+		}
+	# Check to see if this login went to an interesting hostname.
+	when ( local resp_hostname = lookup_addr(c$id$orig_h) )
+		{
+		if ( interesting_hostnames in resp_hostname )
+			{
+			NOTICE([$note=Login_To_Interesting_Hostname,
+			        $conn=c,
+			        $msg=fmt("Interesting login to hostname: %s", resp_hostname),
+			        $sub=resp_hostname]);
+			}
+		}
+	}
+

scripts/policy/protocols/ssh/software.bro

@@ -3,8 +3,8 @@ module SSH;
 
 export {
 	redef enum Software::Type += {
-		SSH_SERVER,
-		SSH_CLIENT,
+		SERVER,
+		CLIENT,
 	};
 }
 
@@ -12,7 +12,7 @@ event ssh_client_version(c: connection, version: string) &priority=4
 	{
 	# Get rid of the protocol information when passing to the software framework.
 	local cleaned_version = sub(version, /^SSH[0-9\.\-]+/, "");
-	local si = Software::parse(cleaned_version, c$id$orig_h, SSH_CLIENT);
+	local si = Software::parse(cleaned_version, c$id$orig_h, CLIENT);
 	Software::found(c$id, si);
 	}
 
@@ -20,6 +20,6 @@ event ssh_server_version(c: connection, version: string) &priority=4
 	{
 	# Get rid of the protocol information when passing to the software framework.
 	local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, "");
-	local si = Software::parse(cleaned_version, c$id$resp_h, SSH_SERVER);
+	local si = Software::parse(cleaned_version, c$id$resp_h, SERVER);
 	Software::found(c$id, si);
 	}

scripts/site/local-manager.bro

@@ -0,0 +1,7 @@
+##! Local site policy loaded only by the manager in a cluster.
+
+# If you are running a cluster you should define your Notice::policy here 
+# so that notice processing occurs on the manager.
+redef Notice::policy += {
+
+};

scripts/site/local-proxy.bro

@@ -0,0 +1,2 @@
+##! Local site policy loaded only by the proxies if Bro is running as a cluster.
+

scripts/site/local-worker.bro

@@ -0,0 +1 @@
+##! Local site policy loaded only by the workers if Bro is running as a cluster.

scripts/site/local.bro

@@ -22,6 +22,7 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 
 # Load all of the scripts that detect software in various protocols.
 @load protocols/http/software
+#@load protocols/http/detect-webapps
 @load protocols/ftp/software
 @load protocols/smtp/software
 @load protocols/ssh/software
@@ -43,4 +44,25 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 @load protocols/ssl/known-certs
 
 # Load the script to enable SSL/TLS certificate validation.
-@load protocols/ssl/validate-certs
+@load protocols/ssl/validate-certs
+
+# If you have libGeoIP support built in, do some geographic detections and 
+# logging for SSH traffic.
+@load protocols/ssh/geo-data
+# Detect hosts doing SSH bruteforce attacks.
+@load protocols/ssh/detect-bruteforcing
+# Detect logins using "interesting" hostnames.
+@load protocols/ssh/interesting-hostnames
+
+# Detect MD5 sums in Team Cymru's Malware Hash Registry.
+@load protocols/http/detect-MHR
+# Detect SQL injection attacks
+@load protocols/http/detect-sqli
+
+# Uncomment this redef if you want to extract SMTP MIME entities for 
+# some file types.  The numbers given indicate how many bytes to extract for
+# the various mime types.
+redef SMTP::entity_excerpt_len += {
+#	["text/plain"] = 1024,
+#	["text/html"] = 1024,
+};

testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log

@@ -0,0 +1,2 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	helo	mailfrom	rcptto	date	from	to	reply_to	msg_id	in_reply_to	subject	x_originating_ip	first_received	second_received	last_reply	path	user_agent
+1254722768.219663	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	GP	<gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	Mon, 5 Oct 2009 11:36:07 +0530	"Gurpartap Singh" <gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	-	<000301ca4581$ef9e57f0$cedb07d0$@in>	-	SMTP	-	-	-	250 OK id=1Mugho-0003Dg-Un	74.53.140.153,10.10.1.4	Microsoft Office Outlook 12.0

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat

@@ -0,0 +1,13 @@
+Hello
+
+ 
+
+I send u smtp pcap file 
+
+Find the attachment
+
+ 
+
+GPS
+
+

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat

@@ -0,0 +1,264 @@
+Version 4.9.9.1
+* Many bug fixes
+* Improved editor
+
+Version 4.9.9.0
+* Support for latest Mingw compiler system builds
+* Bug fixes
+
+Version 4.9.8.9
+* New code tooltip display
+* Improved Indent/Unindent and Remove Comment
+* Improved automatic indent
+* Added support for the "interface" keyword
+* WebUpdate should now report installation problems from PackMan
+* New splash screen and association icons
+* Improved installer
+* Many bug fixes
+
+Version 4.9.8.7
+* Added support for GCC > 3.2
+* Debug variables are now resent during next debug session
+* Watched Variables not in correct context are now kept and updated when it is needed
+* Added new compiler/linker options: 20
+  - Strip executable
+  - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20
+    k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2)
+  - Enable use of processor specific built-in functions (mmmx, sse, sse2, pni, 3dnow)
+* "Default" button in Compiler Options is back
+* Error messages parsing improved
+* Bug fixes
+
+Version 4.9.8.5
+* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value")
+* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features.
+* Many bug fixes
+
+Version 4.9.8.4
+* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup
+* Improved code completion cache
+* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP
+* Big speed up in function parameters listing while editing
+* Bug fixes
+
+Version 4.9.8.3
+* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20
+  include files can now be generated.
+* Improved WebUpdate module
+* Many bug fixes
+
+Version 4.9.8.2
+* New debug feature for DLLs: attach to a running process
+* New project option: Use custom Makefile. 20
+* New WebUpdater module.
+* Allow user to specify an alternate configuration file in Environment Options 20
+  (still can be overriden by using "-c" command line parameter).
+* Lots of bug fixes.
+
+Version 4.9.8.1
+* When creating a DLL, the created static lib respects now the project-defined output directory
+
+Version 4.9.8.0
+* Changed position of compiler/linker parameters in Project Options.
+* Improved help file
+* Bug fixes
+
+Version 4.9.7.9
+* Resource errors are now reported in the Resource sheet
+* Many bug fixes
+
+Version 4.9.7.8
+* Made whole bottom report control floating instead of only debug output.
+* Many bug fixes
+
+Version 4.9.7.7
+* Printing settings are now saved
+* New environment options : "watch variable under mouse" and "Report watch errors"
+* Bug fixes
+
+Version 4.9.7.6
+* Debug variable browser
+* Added possibility to include in a Template the Project's directories (include, libs and ressources)
+* Changed tint of Class browser pictures colors to match the New Look style
+* Bug fixes
+
+Version 4.9.7.5
+* Bug fixes
+
+Version 4.9.7.4
+* When compiling with debugging symbols, an extra definition is passed to the
+  compiler: -D__DEBUG__
+* Each project creates a <project_name>_private.h file containing version
+  information definitions
+* When compiling the current file only, no dependency checks are performed
+* ~300% Speed-up in class parser
+* Added "External programs" in Tools/Environment Options (for units "Open with")
+* Added "Open with" in project units context menu
+* Added "Classes" toolbar
+* Fixed pre-compilation dependency checks to work correctly
+* Added new file menu entry: Save Project As
+* Bug-fix for double quotes in devcpp.cfg file read by vUpdate
+* Other bug fixes
+
+Version 4.9.7.3
+* When adding debugging symbols on request, remove "-s" option from linker
+* Compiling progress window
+* Environment options : "Show progress window" and "Auto-close progress window"
+* Bug fixes
+
+Version 4.9.7.2
+* Bug fixes
+
+Version 4.9.7.1
+* "Build priority" per-unit
+* "Include file in linking process" per-unit
+* New feature: compile current file only
+* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744)
+* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744)
+* Necessary UI changes in Project Options
+* Added display of project filename, project output and a summary of the project files in Project Options General tab.
+* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0
+* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings)
+* Resource files are treated as ordinary files now
+* Updates in "Project Options/Files" code
+* MSVC import now creates the folders structure of the original VC project
+* Bug fixes
+
+Version 4.9.7.0
+* Allow customizing of per-unit compile command in projects
+* Added two new macros: <DATE> and <DATETIME>
+* Added support for macros in the "default source code" (Tools/Editor Options/Code)
+* Separated layout info from project file. It is now kept in a different file
+  (the same filename as the project's but with extension ".layout"). If you
+  have your project under CVS control, you ''ll know why this had to happen...
+* Compiler settings per-project
+* Compiler set per-project
+* Implemented new compiler settings framework
+* "Compile as C++" per-unit
+* "Include file in compilation process" per-unit
+* Project version info (creates the relevant VERSIONINFO struct in the private
+  resource)
+* Support XP Themes (creates the CommonControls 6.0 manifest file and includes
+  it in the private resource)
+* Added CVS "login" and "logout" commands
+* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows.
+* Added "Add Library" button in Project Options
+* Bug fixes
+
+Version 4.9.6.9
+* Implemented search in help files for the word at cursor (context sensitive help)
+* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2)
+* Added "Files" tab in CVS form to allow selection of more than one file for
+  the requested CVS action
+  20
+Version 4.9.6.8
+* support for DLL application hosting, for debugging and executing DLLs under Dev-C++.
+* New class browser option: "Show inherited members"
+* Added support for the '::' member access operator in code-completion
+* Added *working* function arguments hint
+* Added bracket highlighting. When the caret is on a bracket, that bracket and
+  its counterpart are highlighted
+* Nested folders in project view
+
+Version 4.9.6.7
+* XP Theme support
+* Added CVS commands "Add" and "Remove"
+* Added configuration option for "Templates Directory" in "Environment Options"
+* Code-completion updates
+* Bug fixes
+
+Version 4.9.6.6
+* Editor colors are initialized properly on Dev-C++ first-run
+* Added doxygen-style comments in NewClass, NewMemberFunction and NewMemberVariable wizards
+* Added file's date/time stamp in File/Properties window
+* Current windows listing in Window menu
+* Bug fixes
+
+Version 4.9.6.5
+* CVS support
+* Window list (in Window menu)
+* bug fixes
+
+version 4.9.6.4
+* added ENTER key for opening file in project browser, DEL to delete from the project.
+* bug fixes
+
+version 4.9.6.3
+* Bug fixes
+
+version 4.9.6.2
+* Bug fixes
+
+version 4.9.6.1
+* New "Abort compilation" button
+* Bug fixes
+* Now checks for vRoach existance when sending a crash report
+
+Version 4.9.5.5
+* New option in Editor Options: Show editor hints. User can disable the hints
+  displayed in the editor when the mouse moves over a word. Since this was the
+  cause of many errors (although it should be fixed by now), we are giving the
+  user the option to disable this feature.
+* New option in Editor Options (code-completion): Use code-completion cache.
+  Well, it adds caching to code-completion. Depending on the cache size,
+  the program may take a bit longer to start-up, but provides very fast
+  code-completion and the user has all the commands (belonging to the files
+  he added in the cache) at his fingertips. If, for example, the user adds
+  "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of
+  wxWindows! You get the picture...
+* Removed "Only show classes from current file" option in class browser settings.
+  It used to be a checkbox, allowing only two states (on or off), but there is
+  a third relevant option now: "Project classes" so it didn't fit the purpose...
+  The user can define this in the class browser's context menu under "View mode".
+* Fixed the dreaded "Clock skew detected" compiler warning!
+* Fixed many class browser bugs, including some that had to do with class folders.
+
+Version 4.9.5.4
+* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data)
+
+Version 4.9.5.3
+* Added ExceptionsAnalyzer. If the devcpp.map file is in the devcpp.exe directory
+  then we even get a stack trace in the bug report!
+* Added new WebUpdate module (inactive temporarily).
+* Added new code for code-completion caching of files (disabled - work in progress).
+
+Version 4.9.5.2
+* Added new option in class-browser: Use colors
+  (available when right-clicking the class-browser
+  and selecting "View mode").
+* Dev-C++ now traps access violation of your programs (and of itself too ;)
+
+Version 4.9.5.1
+* Implemented the "File/Export/Project to HTML" function.
+* Added "Tip of the day" system.
+* When running a source file in explorer, don't spawn new instance.
+  Instead open the file in an already launched Dev-C++.
+* Class-parser speed-up (50% to 85% improvement timed!!!)
+* Many code-completion updates. Now takes into account context,
+  class inheritance and visibility (shows items only from files
+  #included directly or indirectly)!
+* Caching of result set of code-completion for speed-up.
+* New option "Execution/Parameters" (and "Debug/Parameters").
+
+Version 4.9.5.0 (5.0 beta 5):
+* CPU Window (still in development)
+* ToDo  list
+* Backtrace in debugging
+* Run to cursor
+* Folders in Project and Class Browser
+* Send custom commands to GDB
+* Makefile can now be customized.
+* Modified the behaviour of the -c param : 20
+  -c <config file directory>
+* Saving of custom syntax parameter group
+* Possibility of changing compilers and tools filename.
+* Many bug fixes
+
+
+Version 4.9.4.1 (5.0 beta 4.1):
+
+* back to gcc 2.95.3
+* Profiling support
+* new update/packages checker (vUpdate)
+* Lots of bugfixes
+

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_entities.log

@@ -0,0 +1,4 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	filename	content_len	mime_type	md5	extraction_file	excerpt
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	79	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat	-
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	1918	FAKE_MIME	-	-	-
+1254722770.692804	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	NEWS.txt	10823	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat	-

testing/btest/Baseline/policy.protocols.smtp.mime/smtp_entities.log

@@ -0,0 +1,4 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	filename	content_len	mime_type	md5	extraction_file	excerpt
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	79	FAKE_MIME	92bca2e6cdcde73647125da7dccbdd07	-	-
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	1918	FAKE_MIME	-	-	-
+1254722770.692804	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	NEWS.txt	10823	FAKE_MIME	a968bb0f9f9d95835b2e74c845877e87	-	-

testing/btest/policy/protocols/smtp/basic.test

@@ -0,0 +1,4 @@
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp.log
+
+@load base/protocols/smtp

testing/btest/policy/protocols/smtp/mime-extract.test

@@ -0,0 +1,25 @@
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp_entities.log
+# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat
+# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test"
+# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_1.dat
+# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_2.dat
+
+@load base/protocols/smtp
+
+redef SMTP::extract_file_types=/text\/plain/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(SMTP::SMTP_ENTITIES);
+	Log::add_filter(SMTP::SMTP_ENTITIES, [$name="normalized-mime-types",
+									  	$pred=function(rec: SMTP::EntityInfo): bool
+		{
+		if ( rec?$mime_type )
+			rec$mime_type = "FAKE_MIME";
+		return T;
+		}
+	]);
+	}

testing/btest/policy/protocols/smtp/mime.test

@@ -0,0 +1,23 @@
+# Checks logging of mime types and md5 calculation.  Mime type in the log
+# is normalized to prevent sensitivity to libmagic version.
+
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp_entities.log
+
+@load base/protocols/smtp
+
+redef SMTP::generate_md5=/text\/plain/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(SMTP::SMTP_ENTITIES);
+	Log::add_filter(SMTP::SMTP_ENTITIES, [$name="normalized-mime-types",
+									  $pred=function(rec: SMTP::EntityInfo): bool
+		{
+		if ( rec?$mime_type )
+			rec$mime_type = "FAKE_MIME";
+		return T;
+		}
+	]);
+	}