Improve multipart HTTP/MIME entity file analysis.

Singular CR or LF characters in multipart body content are no longer
converted to a full CRLF (thus corrupting the file) and it also no
longer considers the CRLF before the multipart boundary as part of the
content.

Addresses BIT-1235.

src/analyzer/protocol/http/HTTP.cc

@@ -463,6 +463,20 @@ void HTTP_Entity::SubmitAllHeaders()
 	if ( DEBUG_http )
 		DEBUG_MSG("%.6f end of headers\n", network_time);
 
+	if ( Parent() &&
+	     Parent()->MIMEContentType() == mime::CONTENT_TYPE_MULTIPART )
+		{
+		// Don't treat single \r or \n characters in the multipart body content
+		// as lines because the MIME_Entity code will implicitly add back a
+		// \r\n for each line it receives.  We do this instead of setting
+		// plain delivery mode for the content line analyzer because
+		// the size of the content to deliver "plainly" may be unknown
+		// and just leaving it in that mode indefinitely screws up the
+		// detection of multipart boundaries.
+		http_message->content_line->SupressWeirds(true);
+		http_message->content_line->SetCRLFAsEOL(0);
+		}
+
 	// The presence of a message-body in a request is signaled by
 	// the inclusion of a Content-Length or Transfer-Encoding
 	// header field in the request's message-headers.
@@ -664,6 +678,13 @@ void HTTP_Message::EndEntity(mime::MIME_Entity* entity)
 
 	current_entity = (HTTP_Entity*) entity->Parent();
 
+	if ( entity->Parent() &&
+	     entity->Parent()->MIMEContentType() == mime::CONTENT_TYPE_MULTIPART )
+		{
+		content_line->SupressWeirds(false);
+		content_line->SetCRLFAsEOL();
+		}
+
 	// It is necessary to call Done when EndEntity is triggered by
 	// SubmitAllHeaders (through EndOfData).
 	if ( entity == top_level )

src/analyzer/protocol/http/HTTP.h

@@ -99,6 +99,8 @@ enum {
 // HTTP_MessageDone	-> {Request,Reply}Made
 
 class HTTP_Message : public mime::MIME_Message {
+friend class HTTP_Entity;
+
 public:
 	HTTP_Message(HTTP_Analyzer* analyzer, tcp::ContentLine_Analyzer* cl,
 			 bool is_orig, int expect_body, int64_t init_header_length);

src/analyzer/protocol/mime/MIME.cc

@@ -552,6 +552,7 @@ void MIME_Entity::init()
 	data_buf_offset = -1;
 
 	message = 0;
+	delay_adding_implicit_CRLF = false;
 	}
 
 MIME_Entity::~MIME_Entity()
@@ -1005,14 +1006,35 @@ void MIME_Entity::DecodeDataLine(int len, const char* data, int trailing_CRLF)
 
 void MIME_Entity::DecodeBinary(int len, const char* data, int trailing_CRLF)
 	{
+	if ( delay_adding_implicit_CRLF )
+		{
+		delay_adding_implicit_CRLF = false;
+		DataOctet(CR);
+		DataOctet(LF);
+		}
+
 	DataOctets(len, data);
 
 	if ( trailing_CRLF )
+		{
+		if ( Parent() &&
+		     Parent()->MIMEContentType() == mime::CONTENT_TYPE_MULTIPART )
+			{
+			// For multipart body content, we want to keep all implicit CRLFs
+			// except for the last because that one belongs to the multipart
+			// boundary delimiter, not the content.  Simply delaying the
+			// addition of implicit CRLFs until another chunk of content
+			// data comes in is a way to prevent the CRLF before the final
+			// message boundary from being accidentally added to the content.
+			delay_adding_implicit_CRLF = true;
+			}
+		else
 			{
 			DataOctet(CR);
 			DataOctet(LF);
 			}
 		}
+	}
 
 void MIME_Entity::DecodeQuotedPrintable(int len, const char* data)
 	{

src/analyzer/protocol/mime/MIME.h

@@ -172,6 +172,7 @@ protected:
 	int data_buf_offset;
 
 	MIME_Message* message;
+	bool delay_adding_implicit_CRLF;
 };
 
 // The reason I separate MIME_Message as an abstract class is to

src/analyzer/protocol/tcp/ContentLine.cc

@@ -32,6 +32,7 @@ void ContentLine_Analyzer::InitState()
 	seq_to_skip = 0;
 	plain_delivery_length = 0;
 	is_plain = 0;
+	suppress_weirds = false;
 
 	InitBuffer(0);
 	}
@@ -258,7 +259,7 @@ int ContentLine_Analyzer::DoDeliverOnce(int len, const u_char* data)
 
 			else
 				{
-				if ( Conn()->FlagEvent(SINGULAR_LF) )
+				if ( ! suppress_weirds && Conn()->FlagEvent(SINGULAR_LF) )
 					Conn()->Weird("line_terminated_with_single_LF");
 				buf[offset++] = c;
 				}
@@ -277,7 +278,7 @@ int ContentLine_Analyzer::DoDeliverOnce(int len, const u_char* data)
 		}
 
 		if ( last_char == '\r' )
-			if ( Conn()->FlagEvent(SINGULAR_CR) )
+			if ( ! suppress_weirds && Conn()->FlagEvent(SINGULAR_CR) )
 				Conn()->Weird("line_terminated_with_single_CR");
 
 		last_char = c;
@@ -307,7 +308,7 @@ void ContentLine_Analyzer::CheckNUL()
 			; // Ignore it.
 		else
 			{
-			if ( Conn()->FlagEvent(NUL_IN_LINE) )
+			if ( ! suppress_weirds && Conn()->FlagEvent(NUL_IN_LINE) )
 				Conn()->Weird("NUL_in_line");
 			flag_NULs = 0;
 			}

src/analyzer/protocol/tcp/ContentLine.h

@@ -15,6 +15,9 @@ public:
 	ContentLine_Analyzer(Connection* conn, bool orig);
 	~ContentLine_Analyzer();
 
+	void SupressWeirds(bool enable)
+		{ suppress_weirds = enable; }
+
 	// If enabled, flag (first) line with embedded NUL. Default off.
 	void SetIsNULSensitive(bool enable)
 		{ flag_NULs = enable; }
@@ -96,6 +99,8 @@ protected:
 	// Don't deliver further data.
 	int skip_deliveries;
 
+	bool suppress_weirds;
+
 	// If true, flag (first) line with embedded NUL.
 	unsigned int flag_NULs:1;
 

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out

@@ -1,39 +1,39 @@
 FILE_NEW
 file #0, 0, 0
 FILE_BOF_BUFFER
-test^M^J
+test
 FILE_OVER_NEW_CONNECTION
 FILE_STATE_REMOVE
-file #0, 6, 0
+file #0, 4, 0
 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp]
 source: HTTP
-MD5: 9f06243abcb89c70e0c331c61d871fa7
+MD5: 098f6bcd4621d373cade4e832627b4f6
-SHA1: fde773a18bb29f5ed65e6f0a7aa717fd1fa485d4
+SHA1: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3
-SHA256: 837ccb607e312b170fac7383d7ccfd61fa5072793f19a25e75fbacb56539b86b
+SHA256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
 FILE_NEW
 file #1, 0, 0
 FILE_BOF_BUFFER
-test2^M^J
+test2
 FILE_OVER_NEW_CONNECTION
 FILE_STATE_REMOVE
-file #1, 7, 0
+file #1, 5, 0
 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp]
 source: HTTP
-MD5: d68af81ef370b3873d50f09140068810
+MD5: ad0234829205b9033196ba818f7a872b
-SHA1: 51a7b6f2d91f6a87822dc04560f2972bc14fc97e
+SHA1: 109f4b3c50d7b0df729d299bc6f8e9ef9066971f
-SHA256: de0edd0ac4a705aff70f34734e90a1d0a1d8b76abe4bb53f3ea934bc105b3b17
+SHA256: 60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752
 FILE_NEW
 file #2, 0, 0
 FILE_BOF_BUFFER
-test3^M^J
+test3
 FILE_OVER_NEW_CONNECTION
 FILE_STATE_REMOVE
-file #2, 7, 0
+file #2, 5, 0
 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp]
 source: HTTP
-MD5: 1a3d75d44753ad246f0bd333cdaf08b0
+MD5: 8ad8757baa8564dc136c1e07507f4a98
-SHA1: 4f98809ab09272dfcc58266e3f23ae2393f70e76
+SHA1: 3ebfa301dc59196f18593c45e519287a23297589
-SHA256: 018c67a2c30ed9977e1dddfe98cac542165dac355cf9764c91a362613e752933
+SHA256: fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13
 FILE_NEW
 file #3, 0, 0
 FILE_BOF_BUFFER

testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out

@@ -6,12 +6,12 @@ MIME_TYPE
 text/plain
 FILE_OVER_NEW_CONNECTION
 FILE_STATE_REMOVE
-file #0, 79, 0
+file #0, 77, 0
 [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]
 source: SMTP
-MD5: 92bca2e6cdcde73647125da7dccbdd07
+MD5: 58aff3af22807bc5f4b6357c0038256c
-SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13
+SHA1: c39dc8cd0f8d8b1f7fc8b362c41e69fdf20f668a
-SHA256: 785a8a044d1454ec88837108f443bbb30cc4f529393ffd57118261036bfe59f5
+SHA256: 8d057f3af311c20675eea767a9df5fa31ff3597c6d5d50fd0cdc34766c40204d
 FILE_NEW
 file #1, 0, 0
 FILE_BOF_BUFFER

testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/thefile0

@@ -10,4 +10,3 @@ Find the attachment
 
 GPS
 
-

testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log

@@ -305,15 +305,15 @@
                   [2] is_orig: bool      = T
 
 1254722770.692743 file_new
-                  [0] f: fa_file         = [id=Fel9gs4OtNEV6gUJZ5, parent_id=<uninitialized>, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, radius=<uninitialized>, snmp=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=3], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=<uninitialized>, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=<uninitialized>, u2_events=<uninitialized>]
+                  [0] f: fa_file         = [id=Fel9gs4OtNEV6gUJZ5, parent_id=<uninitialized>, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, radius=<uninitialized>, snmp=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=3], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=<uninitialized>, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=<uninitialized>, u2_events=<uninitialized>]
 
 1254722770.692743 file_over_new_connection
-                  [0] f: fa_file         = [id=Fel9gs4OtNEV6gUJZ5, parent_id=<uninitialized>, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, radius=<uninitialized>, snmp=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=3], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=<uninitialized>, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^J}, rx_hosts={^J^J}, conn_uids={^J^J}, source=SMTP, depth=0, analyzers={^J^J}, mime_type=text/plain, filename=<uninitialized>, duration=0 secs, local_orig=<uninitialized>, is_orig=F, seen_bytes=0, total_bytes=<uninitialized>, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=<uninitialized>, md5=<uninitialized>, sha1=<uninitialized>, sha256=<uninitialized>, x509=<uninitialized>, extracted=<uninitialized>], u2_events=<uninitialized>]
+                  [0] f: fa_file         = [id=Fel9gs4OtNEV6gUJZ5, parent_id=<uninitialized>, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, radius=<uninitialized>, snmp=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=3], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=<uninitialized>, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^J}, rx_hosts={^J^J}, conn_uids={^J^J}, source=SMTP, depth=0, analyzers={^J^J}, mime_type=text/plain, filename=<uninitialized>, duration=0 secs, local_orig=<uninitialized>, is_orig=F, seen_bytes=0, total_bytes=<uninitialized>, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=<uninitialized>, md5=<uninitialized>, sha1=<uninitialized>, sha256=<uninitialized>, x509=<uninitialized>, extracted=<uninitialized>], u2_events=<uninitialized>]
                   [1] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, radius=<uninitialized>, snmp=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I<raj_deol2002in@yahoo.co.in>^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={^J^I<raj_deol2002in@yahoo.co.in>^J}, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=3], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
                   [2] is_orig: bool      = F
 
 1254722770.692743 file_state_remove
-                  [0] f: fa_file         = [id=Fel9gs4OtNEV6gUJZ5, parent_id=<uninitialized>, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, radius=<uninitialized>, snmp=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=3], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]^J}, last_active=1254722770.692743, seen_bytes=79, total_bytes=<uninitialized>, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=3, analyzers={^J^J}, mime_type=text/plain, filename=<uninitialized>, duration=0 secs, local_orig=<uninitialized>, is_orig=F, seen_bytes=0, total_bytes=<uninitialized>, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=<uninitialized>, md5=<uninitialized>, sha1=<uninitialized>, sha256=<uninitialized>, x509=<uninitialized>, extracted=<uninitialized>], u2_events=<uninitialized>]
+                  [0] f: fa_file         = [id=Fel9gs4OtNEV6gUJZ5, parent_id=<uninitialized>, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, radius=<uninitialized>, snmp=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={^J^I^I<raj_deol2002in@yahoo.co.in>^J^I}, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=3], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]^J}, last_active=1254722770.692743, seen_bytes=77, total_bytes=<uninitialized>, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=3, analyzers={^J^J}, mime_type=text/plain, filename=<uninitialized>, duration=0 secs, local_orig=<uninitialized>, is_orig=F, seen_bytes=0, total_bytes=<uninitialized>, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=<uninitialized>, md5=<uninitialized>, sha1=<uninitialized>, sha256=<uninitialized>, x509=<uninitialized>, extracted=<uninitialized>], u2_events=<uninitialized>]
 
 1254722770.692743 get_file_handle
                   [0] tag: enum          = Analyzer::ANALYZER_SMTP