More file analysis updates.

- Recorrected the module name to Files.

  - Added Files::analyzer_name to get a more readable name for a
    file analyzer.

  - Improved and just overall better handled multipart mime
    transfers in HTTP and SMTP.  HTTP now has orig_fuids and resp_fuids
    log fields since multiple "files" can be transferred with
    multipart mime in a single request/response pair.  SMTP has
    an fuids field which has file unique IDs for all parts
    transferred. FTP and IRC have a log field named fuid added
    because only a single file can be transferred per irc and ftp
    log line.

scripts/base/frameworks/files/main.bro

@@ -61,7 +61,7 @@ export {
 		depth: count &default=0 &log;
 
 		## A set of analysis types done during the file analysis.
-		analyzers: set[Analyzer] &log;
+		analyzers: set[string] &log;
 
 		## A mime type provided by libmagic against the *bof_buffer*, or
 		## in the cases where no buffering of the beginning of file occurs,
@@ -76,11 +76,16 @@ export {
 		## The duration the file was analyzed for.
 		duration: interval &log &default=0secs;
 
-		## If the source of this file is is a network connection, this field
+		## If the source of this file is a network connection, this field
 		## indicates if the data originated from the local network or not as
 		## determined by the configured bro:see:`Site::local_nets`.
 		local_orig: bool &log &optional;
 
+		## If the source of this file is a network connection, this field
+		## indicates if the file is being sent by the originator of the connection
+		## or the responder.
+		is_orig: bool &log &optional;
+
 		## Number of bytes provided to the file analysis engine for the file.
 		seen_bytes: count &log &default=0;
 
@@ -105,7 +110,7 @@ export {
 
 	## A table that can be used to disable file analysis completely for
 	## any files transferred over given network protocol analyzers.
-	const disable: table[Analyzer::Tag] of bool = table() &redef;
+	const disable: table[Files::Tag] of bool = table() &redef;
 
 	## The salt concatenated to unique file handle strings generated by
 	## :bro:see:`get_file_handle` before hashing them in to a file id
@@ -139,7 +144,7 @@ export {
 	##          for the *id* isn't currently active or the *args*
 	##          were invalid for the analyzer type.
 	global add_analyzer: function(f: fa_file, 
-	                              tag: Files::Analyzer, 
+	                              tag: Files::Tag, 
 	                              args: AnalyzerArgs &default=AnalyzerArgs()): bool;
 
 	## Removes an analyzer from the analysis of a given file.
@@ -150,7 +155,7 @@ export {
 	##
 	## Returns: true if the analyzer will be removed, or false if analysis
 	##          for the *id* isn't currently active.
-	global remove_analyzer: function(f: fa_file, tag: Files::Analyzer, args: AnalyzerArgs): bool;
+	global remove_analyzer: function(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool;
 
 	## Stops/ignores any further analysis of a given file.
 	##
@@ -161,6 +166,13 @@ export {
 	##          isn't currently active.
 	global stop: function(f: fa_file): bool;
 
+	## Translates an file analyzer enum value to a string with the analyzer's name.
+	##
+	## tag: The analyzer tag.
+	##
+	## Returns: The analyzer name corresponding to the tag.
+	global analyzer_name: function(tag: Files::Tag): string;
+
 	## Register callbacks for protocols that work with the Files framework.  
 	## The callbacks must uniquely identify a file and each protocol can 
 	## only have a single callback registered for it.
@@ -171,7 +183,7 @@ export {
 	##           defined previously.
 	##
 	## Returns: true if the protocol being registered was not previously registered.
-	global register_protocol: function(tag: AnalyzerTag, callback: function(c: connection, is_orig: bool): string): bool;
+	global register_protocol: function(tag: Files::Tag, callback: function(c: connection, is_orig: bool): string): bool;
 
 	## Register a callback for file analyzers to use if they need to do some manipulation
 	## when they are being added to a file before the core code takes over.  This is 
@@ -181,7 +193,7 @@ export {
 	## tag: Tag for the file analyzer.
 	##
 	## callback: Function to execute when the given file analyzer is being added.
-	global register_analyzer_add_callback: function(tag: Files::Analyzer, callback: function(f: fa_file, args: AnalyzerArgs));
+	global register_analyzer_add_callback: function(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs));
 
 	## Event that can be handled to access the Info record as it is sent on
 	## to the logging framework.
@@ -194,14 +206,14 @@ redef record fa_file += {
 
 redef record AnalyzerArgs += {
 	# This is used interally for the core file analyzer api.
-	tag: Files::Analyzer &optional;
+	tag: Files::Tag &optional;
 };
 
 # Store the callbacks for protocol analyzers that have files.
-global registered_protocols: table[AnalyzerTag] of function(c: connection, is_orig: bool): string = table()
+global registered_protocols: table[Files::Tag] of function(c: connection, is_orig: bool): string = table()
 		&default=function(c: connection, is_orig: bool): string { return cat(c$uid, is_orig); };
 
-global analyzer_add_callbacks: table[Files::Analyzer] of function(f: fa_file, args: AnalyzerArgs) = table();
+global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
 
 event bro_init() &priority=5
 	{
@@ -227,6 +239,8 @@ function set_info(f: fa_file)
 		f$info$total_bytes = f$total_bytes;
 	f$info$missing_bytes = f$missing_bytes;
 	f$info$overflow_bytes = f$overflow_bytes;
+	if ( f?$is_orig )
+		f$info$is_orig = f$is_orig;
 	if ( f?$mime_type ) 
 		f$info$mime_type = f$mime_type;
 	}
@@ -236,11 +250,11 @@ function set_timeout_interval(f: fa_file, t: interval): bool
 	return __set_timeout_interval(f$id, t);
 	}
 
-function add_analyzer(f: fa_file, tag: Analyzer, args: AnalyzerArgs): bool
+function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
 	{
 	# This is to construct the correct args for the core API.
 	args$tag = tag;
-	add f$info$analyzers[tag];
+	add f$info$analyzers[Files::analyzer_name(tag)];
 
 	if ( tag in analyzer_add_callbacks )
 		analyzer_add_callbacks[tag](f, args);
@@ -253,12 +267,12 @@ function add_analyzer(f: fa_file, tag: Analyzer, args: AnalyzerArgs): bool
 	return T;
 	}
 
-function register_analyzer_add_callback(tag: Files::Analyzer, callback: function(f: fa_file, args: AnalyzerArgs))
+function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs))
 	{
 	analyzer_add_callbacks[tag] = callback;
 	}
 
-function remove_analyzer(f: fa_file, tag: Files::Analyzer, args: AnalyzerArgs): bool
+function remove_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
 	{
 	args$tag = tag;
 	return __remove_analyzer(f$id, args);
@@ -269,6 +283,11 @@ function stop(f: fa_file): bool
 	return __stop(f$id);
 	}
 
+function analyzer_name(tag: Files::Tag): string
+	{
+	return __analyzer_name(tag);
+	}
+
 event file_new(f: fa_file) &priority=10
 	{
 	set_info(f);
@@ -302,14 +321,14 @@ event file_state_remove(f: fa_file) &priority=-10
 	Log::write(Files::LOG, f$info);
 	}
 
-function register_protocol(tag: AnalyzerTag, callback: function(c: connection, is_orig: bool): string): bool
+function register_protocol(tag: Files::Tag, callback: function(c: connection, is_orig: bool): string): bool
 	{
 	local result = (tag !in registered_protocols);
 	registered_protocols[tag] = callback;
 	return result;
 	}
 
-event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool) &priority=5
+event get_file_handle(tag: Files::Tag, c: connection, is_orig: bool) &priority=5
 	{
 	local handler = registered_protocols[tag];
 	set_file_handle(handler(c, is_orig));

scripts/base/protocols/ftp/__load__.bro

@@ -1,4 +1,4 @@
 @load ./utils-commands
 @load ./main
-@load ./file-analysis
+@load ./files
 @load ./gridftp

scripts/base/protocols/ftp/file-analysis.bro

@@ -1,23 +0,0 @@
-@load ./main
-@load base/utils/conn-ids
-@load base/frameworks/files
-
-module FTP;
-
-export {
-	## Default file handle provider for FTP.
-	global get_file_handle: function(c: connection, is_orig: bool): string;
-}
-
-function get_file_handle(c: connection, is_orig: bool): string
-	{
-	if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) 
-		return "";
-
-	return cat(ANALYZER_FTP_DATA, c$start_time, c$id, is_orig);
-	}
-
-event bro_init() &priority=5
-	{
-	Files::register_protocol(ANALYZER_FTP_DATA, FTP::get_file_handle);
-	}

scripts/base/protocols/ftp/files.bro

@@ -0,0 +1,40 @@
+@load ./main
+@load base/utils/conn-ids
+@load base/frameworks/files
+
+module FTP;
+
+export {
+	redef record Info += {
+		## File unique ID.
+		fuid: string &optional &log;
+	};
+
+	## Default file handle provider for FTP.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) 
+		return "";
+
+	return cat(Analyzer::ANALYZER_FTP_DATA, c$start_time, c$id, is_orig);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_FTP_DATA, FTP::get_file_handle);
+	}
+
+
+event file_over_new_connection(f: fa_file, c: connection) &priority=5
+	{
+	if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) 
+		return;
+
+	local ftp = ftp_data_expected[c$id$resp_h, c$id$resp_p];
+	ftp$fuid = f$id;
+	if ( f?$mime_type )
+		ftp$mime_type = f$mime_type;
+	}

scripts/base/protocols/http/__load__.bro

@@ -1,6 +1,4 @@
 @load ./main
+@load ./entities
 @load ./utils
-@load ./file-analysis
+@load ./files
-#@load ./file-ident
-#@load ./file-hash
-#@load ./file-extract

scripts/base/protocols/http/entities.bro

@@ -0,0 +1,70 @@
+##! Analysis and logging for MIME entities found in HTTP sessions.
+
+@load base/frameworks/files
+@load base/utils/strings
+@load base/utils/files
+@load ./main
+
+module HTTP;
+
+export {
+	type Entity: record {
+		## Depth of the entity if multiple entities are sent in a single transaction.
+		depth: count &default=0;
+
+		## Filename for the entity if discovered from a header.
+		filename: string &optional;
+	};
+
+	redef record Info += {
+		## The current entity being seen.
+		entity:          Entity    &optional;
+
+		## Current number of MIME entities in the HTTP request message body.
+		orig_mime_depth: count     &default=0;
+		## Current number of MIME entities in the HTTP response message body.
+		resp_mime_depth: count     &default=0;
+	};
+}
+
+event http_begin_entity(c: connection, is_orig: bool) &priority=10
+	{
+	set_state(c, F, is_orig);
+
+	if ( is_orig )
+		++c$http$orig_mime_depth;
+	else
+		++c$http$resp_mime_depth;
+
+	c$http$entity = Entity($depth = is_orig ? c$http$orig_mime_depth : c$http$resp_mime_depth);
+	}
+
+event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3
+	{
+	if ( name == "CONTENT-DISPOSITION" &&
+	     /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value )
+		{
+		c$http$entity$filename = extract_filename_from_content_disposition(value);
+		}
+	else if ( name == "CONTENT-TYPE" &&
+	          /[nN][aA][mM][eE][:blank:]*=/ in value )
+		{
+		c$http$entity$filename = extract_filename_from_content_disposition(value);
+		}
+	}
+
+event file_over_new_connection(f: fa_file, c: connection) &priority=5
+	{
+	if ( f$source == "HTTP" && c$http?$entity ) 
+		{
+		f$info$depth = c$http$entity$depth;
+		if ( c$http$entity?$filename )
+			f$info$filename = c$http$entity$filename;
+		}
+	}
+
+event http_end_entity(c: connection, is_orig: bool) &priority=5
+	{
+	if ( c?$http && c$http?$entity ) 
+		delete c$http$entity;
+	}

scripts/base/protocols/http/files.bro

@@ -1,17 +1,17 @@
 @load ./main
+@load ./entities
 @load ./utils
-@load base/utils/conn-ids
 @load base/frameworks/files
 
 module HTTP;
 
 export {
 	redef record Info += {
-		## The sniffed mime type of the data being sent by the client.
+		## An ordered vector of file unique IDs seen sent by the originator (client).
-		client_mime_type: string &log &optional;
+		orig_fuids:    vector of string &log &default=string_vec();
 
-		## The sniffed mime type of the data being returned by the server.
+		## An ordered vector of file unique IDs seen sent by the responder (server).
-		mime_type:        string &log &optional;
+		resp_fuids:    vector of string &log &default=string_vec();
 	};
 
 	## Default file handle provider for HTTP.
@@ -26,33 +26,27 @@ function get_file_handle(c: connection, is_orig: bool): string
 	local mime_depth = is_orig ? c$http$orig_mime_depth : c$http$resp_mime_depth;
 	if ( c$http$range_request )
 		{
-		return cat(ANALYZER_HTTP, is_orig, c$id$orig_h, mime_depth, build_url(c$http));
+		return cat(Analyzer::ANALYZER_HTTP, is_orig, c$id$orig_h, mime_depth, build_url(c$http));
 		}
 	else
 		{
-		return cat(ANALYZER_HTTP, c$start_time, is_orig, 
+		return cat(Analyzer::ANALYZER_HTTP, c$start_time, is_orig, 
 		           c$http$trans_depth, mime_depth, id_string(c$id));
 		}
 	}
 
 event bro_init() &priority=5
 	{
-	Files::register_protocol(ANALYZER_HTTP, HTTP::get_file_handle);
+	Files::register_protocol(Analyzer::ANALYZER_HTTP, HTTP::get_file_handle);
 	}
 
 event file_over_new_connection(f: fa_file, c: connection) &priority=5
 	{
 	if ( c?$http )
 		{
-		#if (!f?$mime_type)
+		if ( f$is_orig )
-		#	print f;
+			c$http$orig_fuids[|c$http$orig_fuids|] = f$id;
-#
+		else
-		#if ( f$is_orig )
+			c$http$resp_fuids[|c$http$resp_fuids|] = f$id;
-		#	c$http$client_mime_type = f$mime_type;
-		#else
-		#	c$http$mime_type = f$mime_type;
-
-		if ( c$http?$filename )
-			f$info$filename = c$http$filename;
 		}
 	}

scripts/base/protocols/http/main.bro

@@ -75,10 +75,6 @@ export {
 		## Indicates if this request can assume 206 partial content in
 		## response.
 		range_request:           bool      &default=F;
-		## Number of MIME entities in the HTTP request message body so far.
-		orig_mime_depth:         count     &default=0;
-		## Number of MIME entities in the HTTP response message body so far.
-		resp_mime_depth:         count     &default=0;
 	};
 	
 	## Structure to maintain state for an HTTP connection with multiple 
@@ -276,22 +272,6 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 			}
 		}
 
-	else # server headers
-		{
-		if ( name == "CONTENT-DISPOSITION" &&
-		     /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value )
-			c$http$filename = extract_filename_from_content_disposition(value);
-		}
-	}
-	
-event http_begin_entity(c: connection, is_orig: bool) &priority=5
-	{
-	set_state(c, F, is_orig);
-
-	if ( is_orig )
-		++c$http$orig_mime_depth;
-	else
-		++c$http$resp_mime_depth;
 	}
 	
 event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = 5

scripts/base/protocols/irc/__load__.bro

@@ -1,3 +1,3 @@
 @load ./main
-#@load ./dcc-send
+@load ./dcc-send
-@load ./file-analysis
+@load ./files

scripts/base/protocols/irc/dcc-send.bro

@@ -49,13 +49,15 @@ function log_dcc(f: fa_file)
 		delete irc$dcc_file_name;
 		delete irc$dcc_file_size;
 		delete irc$dcc_mime_type;
+
+		delete dcc_expected_transfers[cid$resp_h, cid$resp_p];
 		return;
 		}
 	}
 
 event file_new(f: fa_file) &priority=-5
 	{
-	if ( f?$source && f$source == "IRC_DATA" ) 
+	if ( f$source == "IRC_DATA" ) 
 		log_dcc(f);
 	}
 

scripts/base/protocols/irc/files.bro

@@ -0,0 +1,41 @@
+@load ./dcc-send
+@load base/utils/conn-ids
+@load base/frameworks/files
+
+module IRC;
+
+export {
+	redef record Info += {
+		## File unique ID.
+		fuid: string &log &optional;
+	};
+
+	## Default file handle provider for IRC.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers ) 
+		return "";
+
+	return cat(Analyzer::ANALYZER_IRC_DATA, c$start_time, c$id, is_orig);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_IRC_DATA, IRC::get_file_handle);
+	}
+
+event file_over_new_connection(f: fa_file, c: connection) &priority=5
+	{
+	if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers ) 
+		return;
+
+	local irc = dcc_expected_transfers[c$id$resp_h, c$id$resp_p];
+	irc$fuid = f$id;
+	if ( irc?$dcc_file_name )
+		f$info$filename = irc$dcc_file_name;
+	if ( f?$mime_type )
+		irc$dcc_mime_type = f$mime_type;
+	}

scripts/base/protocols/smtp/__load__.bro

@@ -1,4 +1,3 @@
 @load ./main
 @load ./entities
-#@load ./entities-excerpt
+@load ./files
-@load ./file-analysis

scripts/base/protocols/smtp/entities.bro

@@ -9,6 +9,7 @@ module SMTP;
 
 export {
 	type Entity: record {
+		## Filename for the entity if discovered from a header.
 		filename: string &optional;
 	};
 
@@ -26,8 +27,6 @@ export {
 
 event mime_begin_entity(c: connection) &priority=10
 	{
-	#print fmt("%s : begin entity", c$uid);
-
 	c$smtp$entity = Entity();
 	++c$smtp_state$mime_depth;
 	}

scripts/base/protocols/smtp/file-analysis.bro

@@ -1,22 +0,0 @@
-@load ./main
-@load ./entities
-@load base/utils/conn-ids
-@load base/frameworks/files
-
-module SMTP;
-
-export {
-	## Default file handle provider for SMTP.
-	global get_file_handle: function(c: connection, is_orig: bool): string;
-}
-
-function get_file_handle(c: connection, is_orig: bool): string
-	{
-	return cat(ANALYZER_SMTP, c$start_time, c$smtp$trans_depth,
-	           c$smtp_state$mime_depth);
-	}
-
-event bro_init() &priority=5
-	{
-	Files::register_protocol(ANALYZER_SMTP, SMTP::get_file_handle);
-	}

scripts/base/protocols/smtp/files.bro

@@ -0,0 +1,34 @@
+@load ./main
+@load ./entities
+@load base/utils/conn-ids
+@load base/frameworks/files
+
+module SMTP;
+
+export {
+	redef record Info += {
+		## An ordered vector of file unique IDs seen attached to
+		## the message.
+		fuids: vector of string &log &default=string_vec();
+	};
+
+	## Default file handle provider for SMTP.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	return cat(Analyzer::ANALYZER_SMTP, c$start_time, c$smtp$trans_depth,
+	           c$smtp_state$mime_depth);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_SMTP, SMTP::get_file_handle);
+	}
+
+event file_over_new_connection(f: fa_file, c: connection) &priority=5
+	{
+	if ( c?$smtp )
+		c$smtp$fuids[|c$smtp$fuids|] = f$id;
+	}

src/file_analysis/Manager.cc

@@ -19,8 +19,8 @@ string Manager::salt;
 
 Manager::Manager()
 	{
-	tag_enum_type = new EnumType("FileAnalysis::Tag");
+	tag_enum_type = new EnumType("Files::Tag");
-	::ID* id = install_ID("Tag", "FileAnalysis", true, true);
+	::ID* id = install_ID("Tag", "Files", true, true);
 	add_type(id, tag_enum_type, 0, 0);
 	}
 
@@ -42,7 +42,7 @@ void Manager::RegisterAnalyzerComponent(Component* component)
 	{
 	const char* cname = component->CanonicalName();
 
-	if ( tag_enum_type->Lookup("FileAnalysis", cname) != -1 )
+	if ( tag_enum_type->Lookup("Files", cname) != -1 )
 		reporter->FatalError("File Analyzer %s defined more than once", cname);
 
 	DBG_LOG(DBG_FILE_ANALYSIS, "Registering analyzer %s (tag %s)",
@@ -54,7 +54,7 @@ void Manager::RegisterAnalyzerComponent(Component* component)
 	        component->Tag().AsEnumVal()->InternalInt(), component));
 
 	string id = fmt("ANALYZER_%s", cname);
-	tag_enum_type->AddName("FileAnalysis", id.c_str(),
+	tag_enum_type->AddName("Files", id.c_str(),
 						   component->Tag().AsEnumVal()->InternalInt(), true);
 	}
 

src/file_analysis/analyzer/hash/events.bif

@@ -7,6 +7,6 @@
 ##
 ## hash: The result of the hashing.
 ##
-## .. bro:see:: FileAnalysis::add_analyzer FileAnalysis::ANALYZER_MD5
+## .. bro:see:: Files::add_analyzer Files::ANALYZER_MD5
-##    FileAnalysis::ANALYZER_SHA1 FileAnalysis::ANALYZER_SHA256
+##    Files::ANALYZER_SHA1 Files::ANALYZER_SHA256
 event file_hash%(f: fa_file, kind: string, hash: string%);

src/file_analysis/file_analysis.bif

@@ -42,6 +42,12 @@ function Files::__stop%(file_id: string%): bool
 	return new Val(result, TYPE_BOOL);
 	%}
 
+## :bro:see:`Files::analyzer_name`.
+function Files::__analyzer_name%(tag: Files::Tag%) : string
+	%{
+	return new StringVal(file_mgr->GetAnalyzerName(tag->InternalInt()));
+	%}
+
 module GLOBAL;
 
 ## For use within a :bro:see:`get_file_handle` handler to set a unique