Merge remote-tracking branch 'origin/topic/jsiwek/smtp-refactor'

- While updating, I did some further work on the branch.

- New function in the base/utils/files for extracting filenames
  from content-dispositions.

- New script for entity excerpt extraction if you aren't interested
  in full extraction.  The data goes a log field too.

- Some renaming and reorganization of types.

- Updated tests to work with new code.

* origin/topic/jsiwek/smtp-refactor:
  Make the doc.coverage test happy.
  SMTP script refactor. (addresses #509)

Conflicts:
	doc/scripts/DocSourcesList.cmake
	policy/protocols/smtp/__load__.bro
	policy/protocols/smtp/base/__load__.bro

scripts/base/protocols/mime/file-extract.bro

@@ -16,11 +16,13 @@ export {
 		extract_file:         bool    &default=F;
 	
 		## Store the file handle here for the file currently being extracted.
-		extraction_file:      file    &optional;
-		
+		extraction_file:      file    &log &optional;
+	};
+
+	redef record State += {
 		## Store a count of the number of files that have been transferred in
 		## this conversation to create unique file names on disk.
-		num_extracted_files:  count   &optional;
+		num_extracted_files:  count   &default=0;
 	};
 }
 
@@ -34,7 +36,7 @@ event mime_segment_data(c: connection, length: count, data: string) &priority=3
 	{
 	if ( c$mime$extract_file && c$mime$content_len == 0 )
 		{
-		local suffix = fmt("%d.dat", ++c$mime$num_extracted_files);
+		local suffix = fmt("%d.dat", ++c$mime_state$num_extracted_files);
 		local fname = generate_extraction_filename(extraction_prefix, c, suffix);
 		c$mime$extraction_file = open(fname);
 		enable_raw_output(c$mime$extraction_file);
@@ -57,4 +59,4 @@ event mime_end_entity(c: connection) &priority=-3
 	if ( c$mime?$extraction_file )
 		close(c$mime$extraction_file);
 	}
-	
+	

scripts/base/protocols/smtp/__load__.bro

@@ -1 +1,3 @@
 @load ./main
+@load ./entities
+@load ./entities-excerpt

scripts/base/protocols/smtp/entities-excerpt.bro

@@ -0,0 +1,52 @@
+##! This script is for optionally adding a body excerpt to the SMTP
+##! entities log.
+
+@load ./entities
+
+module SMTP;
+
+export {
+	redef record SMTP::EntityInfo += {
+		## The entity body excerpt.
+		excerpt:    string &log &default="";
+		
+		## Internal tracking to know how much of the body should be included
+		## in the excerpt.
+		excerpt_len: count &optional;
+	};
+	
+	## This is the default value for how much of the entity body should be
+	## included for all MIME entities.
+	const default_entity_excerpt_len = 0 &redef;
+	
+	## This table defines how much of various entity bodies should be 
+	## included in excerpts.
+	const entity_excerpt_len: table[string] of count = {} 
+		&redef
+		&default = default_entity_excerpt_len;
+}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-1
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$content_len == 0 )
+		c$smtp$current_entity$excerpt_len = entity_excerpt_len[c$smtp$current_entity$mime_type];
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-2
+	{
+	if ( ! c?$smtp ) return;
+	
+	local ent = c$smtp$current_entity;
+	if ( ent$content_len < ent$excerpt_len )
+		{
+		if ( ent$content_len + length < ent$excerpt_len )
+			ent$excerpt = cat(ent$excerpt, data);
+		else
+			{
+			local x_bytes = ent$excerpt_len - ent$content_len;
+			ent$excerpt = cat(ent$excerpt, sub_bytes(data, 1, x_bytes));
+			}
+		}
+	}

scripts/base/protocols/smtp/entities.bro

@@ -0,0 +1,234 @@
+##! Analysis and logging for MIME entities found in SMTP sessions.
+
+@load base/utils/strings
+@load base/utils/files
+@load ./main
+
+module SMTP;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates that an MD5 sum was calculated for a MIME message.
+		MD5,
+	};
+
+	redef enum Log::ID += { SMTP_ENTITIES };
+
+	type EntityInfo: record {
+		## This is the timestamp of when the MIME content transfer began.
+		ts:               time    &log;
+		uid:              string  &log;
+		id:               conn_id &log;
+		## Internally generated "message id" that ties back to the particular
+		## message in the SMTP log where this entity was seen.
+		mid:              string  &log;
+		## The filename seen in the Content-Disposition header.
+		filename:         string  &log &optional;
+		## Track how many bytes of the MIME encoded file have been seen.
+		content_len:      count   &log &default=0;
+		## The mime type of the entity discovered through magic bytes identification.
+		mime_type:        string  &log &optional;
+		
+		## The calculated MD5 sum for the MIME entity.
+		md5:              string  &log &optional;
+		## Optionally calculate the file's MD5 sum.  Must be set prior to the 
+		## first data chunk being see in an event.
+		calc_md5:         bool    &default=F;
+		## This boolean value indicates if an MD5 sum is being calculated 
+		## for the current file transfer.
+		calculating_md5:  bool    &default=F;
+		
+		## Optionally write the file to disk.  Must be set prior to first 
+		## data chunk being seen in an event.
+		extract_file:     bool    &default=F;
+		## Store the file handle here for the file currently being extracted.
+		extraction_file:  file    &log &optional;
+	};
+
+	redef record Info += {
+		## The in-progress entity information.
+		current_entity:   EntityInfo &optional;
+	};
+
+	redef record State += {
+		## Store a count of the number of files that have been transferred in
+		## a conversation to create unique file names on disk.
+		num_extracted_files:  count   &default=0;
+		## Track the number of MIME encoded files transferred during a session.
+		mime_level:           count   &default=0;
+	};
+
+	## Generate MD5 sums for these filetypes.
+	const generate_md5 = /application\/x-dosexec/    # Windows and DOS executables
+	                   | /application\/x-executable/ # *NIX executable binary
+	                   &redef;
+
+	## Pattern of file mime types to extract from MIME bodies.
+	const extract_file_types = /NO_DEFAULT/ &redef;
+
+	## The on-disk prefix for files to be extracted from MIME entity bodies.
+	const extraction_prefix = "smtp-entity" &redef;
+
+	global log_mime: event(rec: EntityInfo);
+}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(SMTP_ENTITIES, [$columns=EntityInfo, $ev=log_mime]);
+	}
+
+function set_session(c: connection, new_entity: bool)
+	{
+	if ( ! c$smtp?$current_entity || new_entity )
+		{
+		local info: EntityInfo;
+		info$ts=network_time();
+		info$uid=c$uid;
+		info$id=c$id;
+		info$mid=c$smtp$mid;
+		
+		c$smtp$current_entity = info;
+		++c$smtp_state$mime_level;
+		}
+	}
+
+event mime_begin_entity(c: connection) &priority=10
+	{
+	if ( ! c?$smtp ) return;
+	
+	set_session(c, T);
+	}
+
+# This has priority -10 because other handlers need to know the current
+# content_len before it's updated by this handler.
+event mime_segment_data(c: connection, length: count, data: string) &priority=-10
+	{
+	if ( ! c?$smtp ) return;
+	
+	c$smtp$current_entity$content_len = c$smtp$current_entity$content_len + length;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=7
+    {
+	if ( ! c?$smtp ) return;
+	if ( c$smtp$current_entity$content_len == 0 )
+		c$smtp$current_entity$mime_type = split1(identify_data(data, T), /;/)[1];
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+
+	if ( c$smtp$current_entity$content_len == 0 )
+		{
+		if ( generate_md5 in c$smtp$current_entity$mime_type )
+			c$smtp$current_entity$calc_md5 = T;
+
+		if ( c$smtp$current_entity$calc_md5 )
+			{
+			c$smtp$current_entity$calculating_md5 = T;
+			md5_hash_init(c$id);
+			}
+		}
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		md5_hash_update(c$id, data);
+}
+
+## In the event of a content gap during the MIME transfer, detect the state for
+## the MD5 sum calculation and stop calculating the MD5 since it would be
+## incorrect anyway.
+event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5
+	{
+	if ( is_orig || ! c?$smtp || ! c$smtp?$current_entity ) return;
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		{
+		c$smtp$current_entity$calculating_md5 = F;
+		md5_hash_finish(c$id);
+		}
+	}
+
+event mime_end_entity(c: connection) &priority=-3
+    {
+	# TODO: this check is only due to a bug in mime_end_entity that
+	#       causes the event to be generated twice for the same real event.
+	if ( ! c?$smtp || ! c$smtp?$current_entity )
+		return;
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		{
+		c$smtp$current_entity$md5 = md5_hash_finish(c$id);
+
+		NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h),
+				$sub=c$smtp$current_entity$md5, $conn=c]);
+		}
+	}
+
+event mime_one_header(c: connection, h: mime_header_rec)
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( h$name == "CONTENT-DISPOSITION" &&
+	     /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value )
+		c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value);
+	}
+
+event mime_end_entity(c: connection) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+
+	# This check and the delete below are just to cope with a bug where
+	# mime_end_entity can be generated multiple times for the same event.
+	if ( ! c$smtp?$current_entity )
+		return;
+
+	# Only log is there was some content.
+	if ( c$smtp$current_entity$content_len > 0 )
+		Log::write(SMTP_ENTITIES, c$smtp$current_entity);
+
+	delete c$smtp$current_entity;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=5
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( extract_file_types in c$smtp$current_entity$mime_type )
+		c$smtp$current_entity$extract_file = T;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=3
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$extract_file && 
+	     c$smtp$current_entity$content_len == 0 )
+		{
+		local suffix = fmt("%d.dat", ++c$smtp_state$num_extracted_files);
+		local fname = generate_extraction_filename(extraction_prefix, c, suffix);
+		c$smtp$current_entity$extraction_file = open(fname);
+		enable_raw_output(c$smtp$current_entity$extraction_file);
+		}
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$extract_file && c$smtp$current_entity?$extraction_file )
+		print c$smtp$current_entity$extraction_file, data;
+	}
+
+event mime_end_entity(c: connection) &priority=-3
+	{
+	if ( ! c?$smtp ) return;
+	
+	# TODO: this check is only due to a bug in mime_end_entity that
+	#       causes the event to be generated twice for the same real event.
+	if ( ! c$smtp?$current_entity )
+		return;
+
+	if ( c$smtp$current_entity?$extraction_file )
+		close(c$smtp$current_entity$extraction_file);
+	}

scripts/base/protocols/smtp/main.bro

@@ -15,6 +15,10 @@ export {
 		ts:                time            &log;
 		uid:               string          &log;
 		id:                conn_id         &log;
+		## This is an internally generated "message id" that can be used to
+		## map between SMTP messages and MIME entities in the SMTP entities
+		## log.
+		mid:               string          &log;
 		helo:              string          &log &optional;
 		mailfrom:          string          &log &optional;
 		rcptto:            set[string]     &log &optional;
@@ -30,19 +34,13 @@ export {
 		second_received:   string          &log &optional;
 		## The last message the server sent to the client.
 		last_reply:        string          &log &optional;
-		files:             set[string]     &log &optional;
 		path:              vector of addr  &log &optional;
 		user_agent:        string          &log &optional;
 		
-		## Indicate if this session is currently transmitting SMTP message 
-		## envelope headers.
-		in_headers:        bool            &default=F;
 		## Indicate if the "Received: from" headers should still be processed.
 		process_received_from: bool        &default=T;
-		## Maintain the current header for cases where there is header wrapping.
-		current_header:    string          &default="";
-		## Indicate when the message is logged and no longer applicable.
-		done:              bool            &default=F;
+		## Indicates if client activity has been seen, but not yet logged
+		has_client_activity:  bool            &default=F;
 	};
 	
 	type State: record {
@@ -121,6 +119,7 @@ function new_smtp_log(c: connection): Info
 	l$ts=network_time();
 	l$uid=c$uid;
 	l$id=c$id;
+	l$mid=unique_id("@");
 	if ( c?$smtp_state && c$smtp_state?$helo )
 		l$helo = c$smtp_state$helo;
 	
@@ -136,26 +135,23 @@ function set_smtp_session(c: connection)
 	if ( ! c?$smtp_state )
 		c$smtp_state = [];
 	
-	if ( ! c?$smtp || c$smtp$done )
-		{
+	if ( ! c?$smtp )
 		c$smtp = new_smtp_log(c);
-		}
 	}
 
-
 function smtp_message(c: connection)
 	{
-	Log::write(SMTP, c$smtp);
-
-	c$smtp$done = T;
-	# Track the number of messages seen in this session.
-	++c$smtp_state$messages_transferred;
+	if ( c$smtp$has_client_activity )
+		Log::write(SMTP, c$smtp);
 	}
 	
 event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=5
 	{
 	set_smtp_session(c);
 	local upper_command = to_upper(command);
+
+	if ( upper_command != "QUIT" )
+		c$smtp$has_client_activity = T;
 	
 	if ( upper_command == "HELO" || upper_command == "EHLO" )
 		{
@@ -172,26 +168,11 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 
 	else if ( upper_command == "MAIL" && /^[fF][rR][oO][mM]:/ in arg )
 		{
-		# In case this is not the first message in a session we want to 
-		# essentially write out a log, clear the session tracking, and begin
-		# new session tracking.
-		if ( c$smtp_state$messages_transferred > 0 )
-			{
-			smtp_message(c);
-			set_smtp_session(c);
-			}
-		
 		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
 		c$smtp$mailfrom = split1(partially_done, /[[:blank:]]?/)[1];
 		}
-		
-	else if ( upper_command == "DATA" )
-		{
-		c$smtp$in_headers = T;
-		}
 	}
 	
-
 event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                  msg: string, cont_resp: bool) &priority=5
 	{
@@ -199,10 +180,10 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
 	
 	# This continually overwrites, but we want the last reply,
 	# so this actually works fine.
+	c$smtp$last_reply = fmt("%d %s", code, msg);
+
 	if ( code != 421 && code >= 400 )
 		{
-		c$smtp$last_reply = fmt("%d %s", code, msg);
-
 		# Raise a notice when an SMTP error about a block list is discovered.
 		if ( bl_error_messages in msg )
 			{
@@ -223,145 +204,95 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
 		}
 	}
 
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=5
+event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
+                 msg: string, cont_resp: bool) &priority=-5
 	{
-	# Is there something we should be handling from the server?
-	if ( ! is_orig ) return;
-	
 	set_smtp_session(c);
-	
-	if ( ! c$smtp$in_headers )
+	if ( cmd == "." )
 		{
-		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
-			{
-			if ( ! c$smtp?$files )
-				c$smtp$files = set();
-			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
-			add c$smtp$files[data];
-			}
-		return;
+		# Track the number of messages seen in this session.
+		++c$smtp_state$messages_transferred;
+		smtp_message(c);
+		c$smtp = new_smtp_log(c);
 		}
+	}
 
-	if ( /^[[:blank:]]*$/ in data )
-		c$smtp$in_headers = F;
+event mime_one_header(c: connection, h: mime_header_rec) &priority=5
+	{
+	if ( ! c?$smtp ) return;
+	c$smtp$has_client_activity = T;
 
-	# This is to reconstruct headers that tend to wrap around.
-	if ( /^[[:blank:]]/ in data )
-		{
-		# Remove all but a single space at the beginning (this seems to follow
-		# the most common behavior).
-		data = sub(data, /^[[:blank:]]*/, " ");
-		if ( c$smtp$current_header == "MESSAGE-ID" )
-			c$smtp$msg_id += data;
-		else if ( c$smtp$current_header == "RECEIVED" )
-			c$smtp$first_received += data;
-		else if ( c$smtp$current_header == "IN-REPLY-TO" )
-			c$smtp$in_reply_to += data;
-		else if ( c$smtp$current_header == "SUBJECCT" )
-			c$smtp$subject += data;
-		else if ( c$smtp$current_header == "FROM" )
-			c$smtp$from += data;
-		else if ( c$smtp$current_header == "REPLY-TO" )
-			c$smtp$reply_to += data;
-		else if ( c$smtp$current_header == "USER-AGENT" )
-			c$smtp$user_agent += data;
-		return;
-		}
-	# Once there isn't a line starting with a blank, we're not continuing a 
-	# header anymore.
-	c$smtp$current_header = "";
-	
-	local header_parts = split1(data, /:[[:blank:]]*/);
-	# TODO: do something in this case?  This would definitely be odd.
-	# Header wrapping needs to be handled more elegantly.  This will happen
-	# if the header value is wrapped immediately after the header key.
-	if ( |header_parts| != 2 )
-		return;
-	
-	local header_key = to_upper(header_parts[1]);
-	c$smtp$current_header = header_key;
-	
-	local header_val = header_parts[2];
-	
-	if ( header_key == "MESSAGE-ID" )
-		c$smtp$msg_id = header_val;
-	
-	else if ( header_key == "RECEIVED" )
+	if ( h$name == "MESSAGE-ID" )
+		c$smtp$msg_id = h$value;
+
+	else if ( h$name == "RECEIVED" )
 		{
 		if ( c$smtp?$first_received )
 			c$smtp$second_received = c$smtp$first_received;
-		c$smtp$first_received = header_val;
+		c$smtp$first_received = h$value;
 		}
-	
-	else if ( header_key == "IN-REPLY-TO" )
-		c$smtp$in_reply_to = header_val;
-	
-	else if ( header_key == "DATE" )
-		c$smtp$date = header_val;
-	
-	else if ( header_key == "FROM" )
-		c$smtp$from = header_val;
-	
-	else if ( header_key == "TO" )
+
+	else if ( h$name == "IN-REPLY-TO" )
+		c$smtp$in_reply_to = h$value;
+
+	else if ( h$name == "SUBJECT" )
+		c$smtp$subject = h$value;
+
+	else if ( h$name == "FROM" )
+		c$smtp$from = h$value;
+
+	else if ( h$name == "REPLY-TO" )
+		c$smtp$reply_to = h$value;
+
+	else if ( h$name == "DATE" )
+		c$smtp$date = h$value;
+
+	else if ( h$name == "TO" )
 		{
 		if ( ! c$smtp?$to )
 			c$smtp$to = set();
-		add c$smtp$to[header_val];
+		add c$smtp$to[h$value];
 		}
-	
-	else if ( header_key == "REPLY-TO" )
-		c$smtp$reply_to = header_val;
-	
-	else if ( header_key == "SUBJECT" )
-		c$smtp$subject = header_val;
 
-	else if ( header_key == "X-ORIGINATING-IP" )
+	else if ( h$name == "X-ORIGINATING-IP" )
 		{
-		local addresses = find_ip_addresses(header_val);
+		local addresses = find_ip_addresses(h$name);
 		if ( 1 in addresses )
 			c$smtp$x_originating_ip = to_addr(addresses[1]);
 		}
 	
-	else if ( header_key == "X-MAILER" || 
-	          header_key == "USER-AGENT" ||
-	          header_key == "X-USER-AGENT" )
-		{
-		c$smtp$user_agent = header_val;
-		# Explicitly set the current header here because there are several 
-		# headers bulked under this same key.
-		c$smtp$current_header = "USER-AGENT";
-		}
+	else if ( h$name == "X-MAILER" ||
+	          h$name == "USER-AGENT" ||
+	          h$name == "X-USER-AGENT" )
+		c$smtp$user_agent = h$value;
 	}
 	
 # This event handler builds the "Received From" path by reading the 
 # headers in the mail
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=3
+event mime_one_header(c: connection, h: mime_header_rec) &priority=3
 	{
 	# If we've decided that we're done watching the received headers for
 	# whatever reason, we're done.  Could be due to only watching until 
 	# local addresses are seen in the received from headers.
-	if ( c$smtp$current_header != "RECEIVED" ||
-	     ! c$smtp$process_received_from )
+	if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from)
 		return;
-		
-	local text_ip = find_address_in_smtp_header(data);
+
+	local text_ip = find_address_in_smtp_header(h$value);
 	if ( text_ip == "" )
 		return;
 	local ip = to_addr(text_ip);
-	
+
 	if ( ! addr_matches_host(ip, mail_path_capture) && 
 	     ! Site::is_private_addr(ip) )
 		{
 		c$smtp$process_received_from = F;
 		}
-	
 	if ( c$smtp$path[|c$smtp$path|-1] != ip )
 		c$smtp$path[|c$smtp$path|] = ip;
 	}
 
-
 event connection_state_remove(c: connection) &priority=-5
 	{
-	if ( c?$smtp && ! c$smtp$done )
+	if ( c?$smtp )
 		smtp_message(c);
 	}

scripts/base/utils/files.bro

@@ -12,4 +12,15 @@ function generate_extraction_filename(prefix: string, c: connection, suffix: str
 		conn_info = fmt("%s_%s", conn_info, suffix);
 		
 	return conn_info;
-	}
+	}
+	
+## For CONTENT-DISPOSITION headers, this function can be used to extract 
+## the filename.
+function extract_filename_from_content_disposition(data: string): string
+	{
+	local filename = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
+	# Remove quotes around the filename if they are there.
+	if ( /^\"/ in filename )
+		filename =  split_n(filename, /\"/, F, 2)[2];
+	return filename;
+	}

scripts/policy/protocols/smtp/software.bro

@@ -43,10 +43,10 @@ export {
 	                   | /ZimbraWebClient/ &redef;
 }
 
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=4
+event mime_one_header(c: connection, h: mime_header_rec) &priority=4
 	{
-	if ( c$smtp$current_header == "USER-AGENT" &&
-	     webmail_user_agents in c$smtp$user_agent )
+	if ( ! c?$smtp ) return;
+	if ( h$name == "USER-AGENT" && webmail_user_agents in c$smtp$user_agent )
 		c$smtp$is_webmail = T;
 	}
 

scripts/site/local.bro

@@ -43,4 +43,12 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 @load protocols/ssl/known-certs
 
 # Load the script to enable SSL/TLS certificate validation.
-@load protocols/ssl/validate-certs
+@load protocols/ssl/validate-certs
+
+# Uncomment this redef if you want to extract SMTP MIME entities for 
+# some file types.  The numbers given indicate how many bytes to extract for
+# the various mime types.
+redef SMTP::entity_excerpt_len += {
+#	["text/plain"] = 1024,
+#	["text/html"] = 1024,
+};

testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log

@@ -0,0 +1,2 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	helo	mailfrom	rcptto	date	from	to	reply_to	msg_id	in_reply_to	subject	x_originating_ip	first_received	second_received	last_reply	path	user_agent
+1254722768.219663	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	GP	<gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	Mon, 5 Oct 2009 11:36:07 +0530	"Gurpartap Singh" <gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	-	<000301ca4581$ef9e57f0$cedb07d0$@in>	-	SMTP	-	-	-	250 OK id=1Mugho-0003Dg-Un	74.53.140.153,10.10.1.4	Microsoft Office Outlook 12.0

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat

@@ -0,0 +1,13 @@
+Hello
+
+ 
+
+I send u smtp pcap file 
+
+Find the attachment
+
+ 
+
+GPS
+
+

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat

@@ -0,0 +1,264 @@
+Version 4.9.9.1
+* Many bug fixes
+* Improved editor
+
+Version 4.9.9.0
+* Support for latest Mingw compiler system builds
+* Bug fixes
+
+Version 4.9.8.9
+* New code tooltip display
+* Improved Indent/Unindent and Remove Comment
+* Improved automatic indent
+* Added support for the "interface" keyword
+* WebUpdate should now report installation problems from PackMan
+* New splash screen and association icons
+* Improved installer
+* Many bug fixes
+
+Version 4.9.8.7
+* Added support for GCC > 3.2
+* Debug variables are now resent during next debug session
+* Watched Variables not in correct context are now kept and updated when it is needed
+* Added new compiler/linker options: 20
+  - Strip executable
+  - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20
+    k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2)
+  - Enable use of processor specific built-in functions (mmmx, sse, sse2, pni, 3dnow)
+* "Default" button in Compiler Options is back
+* Error messages parsing improved
+* Bug fixes
+
+Version 4.9.8.5
+* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value")
+* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features.
+* Many bug fixes
+
+Version 4.9.8.4
+* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup
+* Improved code completion cache
+* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP
+* Big speed up in function parameters listing while editing
+* Bug fixes
+
+Version 4.9.8.3
+* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20
+  include files can now be generated.
+* Improved WebUpdate module
+* Many bug fixes
+
+Version 4.9.8.2
+* New debug feature for DLLs: attach to a running process
+* New project option: Use custom Makefile. 20
+* New WebUpdater module.
+* Allow user to specify an alternate configuration file in Environment Options 20
+  (still can be overriden by using "-c" command line parameter).
+* Lots of bug fixes.
+
+Version 4.9.8.1
+* When creating a DLL, the created static lib respects now the project-defined output directory
+
+Version 4.9.8.0
+* Changed position of compiler/linker parameters in Project Options.
+* Improved help file
+* Bug fixes
+
+Version 4.9.7.9
+* Resource errors are now reported in the Resource sheet
+* Many bug fixes
+
+Version 4.9.7.8
+* Made whole bottom report control floating instead of only debug output.
+* Many bug fixes
+
+Version 4.9.7.7
+* Printing settings are now saved
+* New environment options : "watch variable under mouse" and "Report watch errors"
+* Bug fixes
+
+Version 4.9.7.6
+* Debug variable browser
+* Added possibility to include in a Template the Project's directories (include, libs and ressources)
+* Changed tint of Class browser pictures colors to match the New Look style
+* Bug fixes
+
+Version 4.9.7.5
+* Bug fixes
+
+Version 4.9.7.4
+* When compiling with debugging symbols, an extra definition is passed to the
+  compiler: -D__DEBUG__
+* Each project creates a <project_name>_private.h file containing version
+  information definitions
+* When compiling the current file only, no dependency checks are performed
+* ~300% Speed-up in class parser
+* Added "External programs" in Tools/Environment Options (for units "Open with")
+* Added "Open with" in project units context menu
+* Added "Classes" toolbar
+* Fixed pre-compilation dependency checks to work correctly
+* Added new file menu entry: Save Project As
+* Bug-fix for double quotes in devcpp.cfg file read by vUpdate
+* Other bug fixes
+
+Version 4.9.7.3
+* When adding debugging symbols on request, remove "-s" option from linker
+* Compiling progress window
+* Environment options : "Show progress window" and "Auto-close progress window"
+* Bug fixes
+
+Version 4.9.7.2
+* Bug fixes
+
+Version 4.9.7.1
+* "Build priority" per-unit
+* "Include file in linking process" per-unit
+* New feature: compile current file only
+* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744)
+* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744)
+* Necessary UI changes in Project Options
+* Added display of project filename, project output and a summary of the project files in Project Options General tab.
+* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0
+* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings)
+* Resource files are treated as ordinary files now
+* Updates in "Project Options/Files" code
+* MSVC import now creates the folders structure of the original VC project
+* Bug fixes
+
+Version 4.9.7.0
+* Allow customizing of per-unit compile command in projects
+* Added two new macros: <DATE> and <DATETIME>
+* Added support for macros in the "default source code" (Tools/Editor Options/Code)
+* Separated layout info from project file. It is now kept in a different file
+  (the same filename as the project's but with extension ".layout"). If you
+  have your project under CVS control, you ''ll know why this had to happen...
+* Compiler settings per-project
+* Compiler set per-project
+* Implemented new compiler settings framework
+* "Compile as C++" per-unit
+* "Include file in compilation process" per-unit
+* Project version info (creates the relevant VERSIONINFO struct in the private
+  resource)
+* Support XP Themes (creates the CommonControls 6.0 manifest file and includes
+  it in the private resource)
+* Added CVS "login" and "logout" commands
+* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows.
+* Added "Add Library" button in Project Options
+* Bug fixes
+
+Version 4.9.6.9
+* Implemented search in help files for the word at cursor (context sensitive help)
+* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2)
+* Added "Files" tab in CVS form to allow selection of more than one file for
+  the requested CVS action
+  20
+Version 4.9.6.8
+* support for DLL application hosting, for debugging and executing DLLs under Dev-C++.
+* New class browser option: "Show inherited members"
+* Added support for the '::' member access operator in code-completion
+* Added *working* function arguments hint
+* Added bracket highlighting. When the caret is on a bracket, that bracket and
+  its counterpart are highlighted
+* Nested folders in project view
+
+Version 4.9.6.7
+* XP Theme support
+* Added CVS commands "Add" and "Remove"
+* Added configuration option for "Templates Directory" in "Environment Options"
+* Code-completion updates
+* Bug fixes
+
+Version 4.9.6.6
+* Editor colors are initialized properly on Dev-C++ first-run
+* Added doxygen-style comments in NewClass, NewMemberFunction and NewMemberVariable wizards
+* Added file's date/time stamp in File/Properties window
+* Current windows listing in Window menu
+* Bug fixes
+
+Version 4.9.6.5
+* CVS support
+* Window list (in Window menu)
+* bug fixes
+
+version 4.9.6.4
+* added ENTER key for opening file in project browser, DEL to delete from the project.
+* bug fixes
+
+version 4.9.6.3
+* Bug fixes
+
+version 4.9.6.2
+* Bug fixes
+
+version 4.9.6.1
+* New "Abort compilation" button
+* Bug fixes
+* Now checks for vRoach existance when sending a crash report
+
+Version 4.9.5.5
+* New option in Editor Options: Show editor hints. User can disable the hints
+  displayed in the editor when the mouse moves over a word. Since this was the
+  cause of many errors (although it should be fixed by now), we are giving the
+  user the option to disable this feature.
+* New option in Editor Options (code-completion): Use code-completion cache.
+  Well, it adds caching to code-completion. Depending on the cache size,
+  the program may take a bit longer to start-up, but provides very fast
+  code-completion and the user has all the commands (belonging to the files
+  he added in the cache) at his fingertips. If, for example, the user adds
+  "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of
+  wxWindows! You get the picture...
+* Removed "Only show classes from current file" option in class browser settings.
+  It used to be a checkbox, allowing only two states (on or off), but there is
+  a third relevant option now: "Project classes" so it didn't fit the purpose...
+  The user can define this in the class browser's context menu under "View mode".
+* Fixed the dreaded "Clock skew detected" compiler warning!
+* Fixed many class browser bugs, including some that had to do with class folders.
+
+Version 4.9.5.4
+* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data)
+
+Version 4.9.5.3
+* Added ExceptionsAnalyzer. If the devcpp.map file is in the devcpp.exe directory
+  then we even get a stack trace in the bug report!
+* Added new WebUpdate module (inactive temporarily).
+* Added new code for code-completion caching of files (disabled - work in progress).
+
+Version 4.9.5.2
+* Added new option in class-browser: Use colors
+  (available when right-clicking the class-browser
+  and selecting "View mode").
+* Dev-C++ now traps access violation of your programs (and of itself too ;)
+
+Version 4.9.5.1
+* Implemented the "File/Export/Project to HTML" function.
+* Added "Tip of the day" system.
+* When running a source file in explorer, don't spawn new instance.
+  Instead open the file in an already launched Dev-C++.
+* Class-parser speed-up (50% to 85% improvement timed!!!)
+* Many code-completion updates. Now takes into account context,
+  class inheritance and visibility (shows items only from files
+  #included directly or indirectly)!
+* Caching of result set of code-completion for speed-up.
+* New option "Execution/Parameters" (and "Debug/Parameters").
+
+Version 4.9.5.0 (5.0 beta 5):
+* CPU Window (still in development)
+* ToDo  list
+* Backtrace in debugging
+* Run to cursor
+* Folders in Project and Class Browser
+* Send custom commands to GDB
+* Makefile can now be customized.
+* Modified the behaviour of the -c param : 20
+  -c <config file directory>
+* Saving of custom syntax parameter group
+* Possibility of changing compilers and tools filename.
+* Many bug fixes
+
+
+Version 4.9.4.1 (5.0 beta 4.1):
+
+* back to gcc 2.95.3
+* Profiling support
+* new update/packages checker (vUpdate)
+* Lots of bugfixes
+

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_entities.log

@@ -0,0 +1,4 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	filename	content_len	mime_type	md5	extraction_file	excerpt
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	79	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat	-
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	1918	FAKE_MIME	-	-	-
+1254722770.692804	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	NEWS.txt	10823	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat	-

testing/btest/Baseline/policy.protocols.smtp.mime/smtp_entities.log

@@ -0,0 +1,4 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	filename	content_len	mime_type	md5	extraction_file	excerpt
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	79	FAKE_MIME	92bca2e6cdcde73647125da7dccbdd07	-	-
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	1918	FAKE_MIME	-	-	-
+1254722770.692804	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	NEWS.txt	10823	FAKE_MIME	a968bb0f9f9d95835b2e74c845877e87	-	-

testing/btest/policy/protocols/smtp/basic.test

@@ -0,0 +1,4 @@
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp.log
+
+@load base/protocols/smtp

testing/btest/policy/protocols/smtp/mime-extract.test

@@ -0,0 +1,25 @@
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp_entities.log
+# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat
+# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test"
+# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_1.dat
+# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_2.dat
+
+@load base/protocols/smtp
+
+redef SMTP::extract_file_types=/text\/plain/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(SMTP::SMTP_ENTITIES);
+	Log::add_filter(SMTP::SMTP_ENTITIES, [$name="normalized-mime-types",
+									  	$pred=function(rec: SMTP::EntityInfo): bool
+		{
+		if ( rec?$mime_type )
+			rec$mime_type = "FAKE_MIME";
+		return T;
+		}
+	]);
+	}

testing/btest/policy/protocols/smtp/mime.test

@@ -0,0 +1,23 @@
+# Checks logging of mime types and md5 calculation.  Mime type in the log
+# is normalized to prevent sensitivity to libmagic version.
+
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp_entities.log
+
+@load base/protocols/smtp
+
+redef SMTP::generate_md5=/text\/plain/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(SMTP::SMTP_ENTITIES);
+	Log::add_filter(SMTP::SMTP_ENTITIES, [$name="normalized-mime-types",
+									  $pred=function(rec: SMTP::EntityInfo): bool
+		{
+		if ( rec?$mime_type )
+			rec$mime_type = "FAKE_MIME";
+		return T;
+		}
+	]);
+	}