More file analysis updates.

- Recorrected the module name to Files.

  - Added Files::analyzer_name to get a more readable name for a
    file analyzer.

  - Improved and just overall better handled multipart mime
    transfers in HTTP and SMTP.  HTTP now has orig_fuids and resp_fuids
    log fields since multiple "files" can be transferred with
    multipart mime in a single request/response pair.  SMTP has
    an fuids field which has file unique IDs for all parts
    transferred. FTP and IRC have a log field named fuid added
    because only a single file can be transferred per irc and ftp
    log line.

scripts/base/protocols/ftp/__load__.bro

@@ -1,4 +1,4 @@
 @load ./utils-commands
 @load ./main
-@load ./file-analysis
+@load ./files
 @load ./gridftp

scripts/base/protocols/ftp/file-analysis.bro

@@ -1,23 +0,0 @@
-@load ./main
-@load base/utils/conn-ids
-@load base/frameworks/files
-
-module FTP;
-
-export {
-	## Default file handle provider for FTP.
-	global get_file_handle: function(c: connection, is_orig: bool): string;
-}
-
-function get_file_handle(c: connection, is_orig: bool): string
-	{
-	if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) 
-		return "";
-
-	return cat(ANALYZER_FTP_DATA, c$start_time, c$id, is_orig);
-	}
-
-event bro_init() &priority=5
-	{
-	Files::register_protocol(ANALYZER_FTP_DATA, FTP::get_file_handle);
-	}

scripts/base/protocols/ftp/files.bro

@@ -0,0 +1,40 @@
+@load ./main
+@load base/utils/conn-ids
+@load base/frameworks/files
+
+module FTP;
+
+export {
+	redef record Info += {
+		## File unique ID.
+		fuid: string &optional &log;
+	};
+
+	## Default file handle provider for FTP.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) 
+		return "";
+
+	return cat(Analyzer::ANALYZER_FTP_DATA, c$start_time, c$id, is_orig);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_FTP_DATA, FTP::get_file_handle);
+	}
+
+
+event file_over_new_connection(f: fa_file, c: connection) &priority=5
+	{
+	if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) 
+		return;
+
+	local ftp = ftp_data_expected[c$id$resp_h, c$id$resp_p];
+	ftp$fuid = f$id;
+	if ( f?$mime_type )
+		ftp$mime_type = f$mime_type;
+	}

scripts/base/protocols/http/__load__.bro

@@ -1,6 +1,4 @@
 @load ./main
+@load ./entities
 @load ./utils
-@load ./file-analysis
-#@load ./file-ident
-#@load ./file-hash
-#@load ./file-extract
+@load ./files

scripts/base/protocols/http/entities.bro

@@ -0,0 +1,70 @@
+##! Analysis and logging for MIME entities found in HTTP sessions.
+
+@load base/frameworks/files
+@load base/utils/strings
+@load base/utils/files
+@load ./main
+
+module HTTP;
+
+export {
+	type Entity: record {
+		## Depth of the entity if multiple entities are sent in a single transaction.
+		depth: count &default=0;
+
+		## Filename for the entity if discovered from a header.
+		filename: string &optional;
+	};
+
+	redef record Info += {
+		## The current entity being seen.
+		entity:          Entity    &optional;
+
+		## Current number of MIME entities in the HTTP request message body.
+		orig_mime_depth: count     &default=0;
+		## Current number of MIME entities in the HTTP response message body.
+		resp_mime_depth: count     &default=0;
+	};
+}
+
+event http_begin_entity(c: connection, is_orig: bool) &priority=10
+	{
+	set_state(c, F, is_orig);
+
+	if ( is_orig )
+		++c$http$orig_mime_depth;
+	else
+		++c$http$resp_mime_depth;
+
+	c$http$entity = Entity($depth = is_orig ? c$http$orig_mime_depth : c$http$resp_mime_depth);
+	}
+
+event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3
+	{
+	if ( name == "CONTENT-DISPOSITION" &&
+	     /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value )
+		{
+		c$http$entity$filename = extract_filename_from_content_disposition(value);
+		}
+	else if ( name == "CONTENT-TYPE" &&
+	          /[nN][aA][mM][eE][:blank:]*=/ in value )
+		{
+		c$http$entity$filename = extract_filename_from_content_disposition(value);
+		}
+	}
+
+event file_over_new_connection(f: fa_file, c: connection) &priority=5
+	{
+	if ( f$source == "HTTP" && c$http?$entity ) 
+		{
+		f$info$depth = c$http$entity$depth;
+		if ( c$http$entity?$filename )
+			f$info$filename = c$http$entity$filename;
+		}
+	}
+
+event http_end_entity(c: connection, is_orig: bool) &priority=5
+	{
+	if ( c?$http && c$http?$entity ) 
+		delete c$http$entity;
+	}

scripts/base/protocols/http/files.bro

@@ -1,17 +1,17 @@
 @load ./main
+@load ./entities
 @load ./utils
-@load base/utils/conn-ids
 @load base/frameworks/files
 
 module HTTP;
 
 export {
 	redef record Info += {
-		## The sniffed mime type of the data being sent by the client.
-		client_mime_type: string &log &optional;
+		## An ordered vector of file unique IDs seen sent by the originator (client).
+		orig_fuids:    vector of string &log &default=string_vec();
 
-		## The sniffed mime type of the data being returned by the server.
-		mime_type:        string &log &optional;
+		## An ordered vector of file unique IDs seen sent by the responder (server).
+		resp_fuids:    vector of string &log &default=string_vec();
 	};
 
 	## Default file handle provider for HTTP.
@@ -26,33 +26,27 @@ function get_file_handle(c: connection, is_orig: bool): string
 	local mime_depth = is_orig ? c$http$orig_mime_depth : c$http$resp_mime_depth;
 	if ( c$http$range_request )
 		{
-		return cat(ANALYZER_HTTP, is_orig, c$id$orig_h, mime_depth, build_url(c$http));
+		return cat(Analyzer::ANALYZER_HTTP, is_orig, c$id$orig_h, mime_depth, build_url(c$http));
 		}
 	else
 		{
-		return cat(ANALYZER_HTTP, c$start_time, is_orig, 
+		return cat(Analyzer::ANALYZER_HTTP, c$start_time, is_orig, 
 		           c$http$trans_depth, mime_depth, id_string(c$id));
 		}
 	}
 
 event bro_init() &priority=5
 	{
-	Files::register_protocol(ANALYZER_HTTP, HTTP::get_file_handle);
+	Files::register_protocol(Analyzer::ANALYZER_HTTP, HTTP::get_file_handle);
 	}
 
 event file_over_new_connection(f: fa_file, c: connection) &priority=5
 	{
 	if ( c?$http )
 		{
-		#if (!f?$mime_type)
-		#	print f;
-#
-		#if ( f$is_orig )
-		#	c$http$client_mime_type = f$mime_type;
-		#else
-		#	c$http$mime_type = f$mime_type;
-
-		if ( c$http?$filename )
-			f$info$filename = c$http$filename;
+		if ( f$is_orig )
+			c$http$orig_fuids[|c$http$orig_fuids|] = f$id;
+		else
+			c$http$resp_fuids[|c$http$resp_fuids|] = f$id;
 		}
 	}

scripts/base/protocols/http/main.bro

@@ -75,10 +75,6 @@ export {
 		## Indicates if this request can assume 206 partial content in
 		## response.
 		range_request:           bool      &default=F;
-		## Number of MIME entities in the HTTP request message body so far.
-		orig_mime_depth:         count     &default=0;
-		## Number of MIME entities in the HTTP response message body so far.
-		resp_mime_depth:         count     &default=0;
 	};
 	
 	## Structure to maintain state for an HTTP connection with multiple 
@@ -104,8 +100,8 @@ export {
 	} &redef;
 
 	## A list of HTTP methods. Other methods will generate a weird. Note
-        ## that the HTTP analyzer will only accept methods consisting solely
-        ## of letters ``[A-Za-z]``.
+	## that the HTTP analyzer will only accept methods consisting solely
+	## of letters ``[A-Za-z]``.
 	const http_methods: set[string] = {
 		"GET", "POST", "HEAD", "OPTIONS",
 		"PUT", "DELETE", "TRACE", "CONNECT",
@@ -275,25 +271,9 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 				}
 			}
 		}
-	
-	else # server headers
-		{
-		if ( name == "CONTENT-DISPOSITION" &&
-		     /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value )
-			c$http$filename = extract_filename_from_content_disposition(value);
-		}
+
 	}
 	
-event http_begin_entity(c: connection, is_orig: bool) &priority=5
-	{
-	set_state(c, F, is_orig);
-
-	if ( is_orig )
-		++c$http$orig_mime_depth;
-	else
-		++c$http$resp_mime_depth;
-	}
-
 event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = 5
 	{
 	set_state(c, F, is_orig);

scripts/base/protocols/irc/__load__.bro

@@ -1,3 +1,3 @@
 @load ./main
-#@load ./dcc-send
-@load ./file-analysis
+@load ./dcc-send
+@load ./files

scripts/base/protocols/irc/dcc-send.bro

@@ -49,13 +49,15 @@ function log_dcc(f: fa_file)
 		delete irc$dcc_file_name;
 		delete irc$dcc_file_size;
 		delete irc$dcc_mime_type;
+
+		delete dcc_expected_transfers[cid$resp_h, cid$resp_p];
 		return;
 		}
 	}
 
 event file_new(f: fa_file) &priority=-5
 	{
-	if ( f?$source && f$source == "IRC_DATA" ) 
+	if ( f$source == "IRC_DATA" ) 
 		log_dcc(f);
 	}
 

scripts/base/protocols/irc/files.bro

@@ -0,0 +1,41 @@
+@load ./dcc-send
+@load base/utils/conn-ids
+@load base/frameworks/files
+
+module IRC;
+
+export {
+	redef record Info += {
+		## File unique ID.
+		fuid: string &log &optional;
+	};
+
+	## Default file handle provider for IRC.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers ) 
+		return "";
+
+	return cat(Analyzer::ANALYZER_IRC_DATA, c$start_time, c$id, is_orig);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_IRC_DATA, IRC::get_file_handle);
+	}
+
+event file_over_new_connection(f: fa_file, c: connection) &priority=5
+	{
+	if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers ) 
+		return;
+
+	local irc = dcc_expected_transfers[c$id$resp_h, c$id$resp_p];
+	irc$fuid = f$id;
+	if ( irc?$dcc_file_name )
+		f$info$filename = irc$dcc_file_name;
+	if ( f?$mime_type )
+		irc$dcc_mime_type = f$mime_type;
+	}

scripts/base/protocols/smtp/__load__.bro

@@ -1,4 +1,3 @@
 @load ./main
 @load ./entities
-#@load ./entities-excerpt
-@load ./file-analysis
+@load ./files

scripts/base/protocols/smtp/entities.bro

@@ -9,6 +9,7 @@ module SMTP;
 
 export {
 	type Entity: record {
+		## Filename for the entity if discovered from a header.
 		filename: string &optional;
 	};
 
@@ -26,8 +27,6 @@ export {
 
 event mime_begin_entity(c: connection) &priority=10
 	{
-	#print fmt("%s : begin entity", c$uid);
-
 	c$smtp$entity = Entity();
 	++c$smtp_state$mime_depth;
 	}

scripts/base/protocols/smtp/file-analysis.bro

@@ -1,22 +0,0 @@
-@load ./main
-@load ./entities
-@load base/utils/conn-ids
-@load base/frameworks/files
-
-module SMTP;
-
-export {
-	## Default file handle provider for SMTP.
-	global get_file_handle: function(c: connection, is_orig: bool): string;
-}
-
-function get_file_handle(c: connection, is_orig: bool): string
-	{
-	return cat(ANALYZER_SMTP, c$start_time, c$smtp$trans_depth,
-	           c$smtp_state$mime_depth);
-	}
-
-event bro_init() &priority=5
-	{
-	Files::register_protocol(ANALYZER_SMTP, SMTP::get_file_handle);
-	}

scripts/base/protocols/smtp/files.bro

@@ -0,0 +1,34 @@
+@load ./main
+@load ./entities
+@load base/utils/conn-ids
+@load base/frameworks/files
+
+module SMTP;
+
+export {
+	redef record Info += {
+		## An ordered vector of file unique IDs seen attached to
+		## the message.
+		fuids: vector of string &log &default=string_vec();
+	};
+
+	## Default file handle provider for SMTP.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	return cat(Analyzer::ANALYZER_SMTP, c$start_time, c$smtp$trans_depth,
+	           c$smtp_state$mime_depth);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_SMTP, SMTP::get_file_handle);
+	}
+
+event file_over_new_connection(f: fa_file, c: connection) &priority=5
+	{
+	if ( c?$smtp )
+		c$smtp$fuids[|c$smtp$fuids|] = f$id;
+	}