Merge remote-tracking branch 'origin/topic/jsiwek/faf-cleanup'

Closes #1002.

* origin/topic/jsiwek/faf-cleanup:
  Move file analyzers to new plugin infrastructure.
  Add a general file analysis overview/how-to document.
  Improve file analysis doxygen comments.
  Improve tracking of HTTP file extraction (addresses #988).
  Fix HTTP multipart body file analysis.
  Remove logging of analyzers field of FileAnalysis::Info.
  Remove extraction counter in default file extraction scripts.
  Remove FileAnalysis::postpone_timeout.
  Make default get_file_handle handlers &priority=5.
  Add input interface to forward data for file analysis.
  File analysis framework interface simplifications.

scripts/base/protocols/ftp/file-analysis.bro

@@ -41,6 +41,7 @@ function get_file_handle(c: connection, is_orig: bool): string
 module GLOBAL;
 
 event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool)
+	&priority=5
 	{
 	if ( tag != Analyzer::ANALYZER_FTP_DATA ) return;
 	set_file_handle(FTP::get_file_handle(c, is_orig));

scripts/base/protocols/ftp/file-extract.bro

@@ -13,8 +13,6 @@ export {
 	const extraction_prefix = "ftp-item" &redef;
 }
 
-global extract_count: count = 0;
-
 redef record Info += {
 	## On disk file where it was extracted to.
 	extraction_file:       string &log &optional;
@@ -26,8 +24,7 @@ redef record Info += {
 
 function get_extraction_name(f: fa_file): string
 	{
-	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
-	++extract_count;
+	local r = fmt("%s-%s.dat", extraction_prefix, f$id);
 	return r;
 	}
 

scripts/base/protocols/http/file-analysis.bro

@@ -6,25 +6,48 @@
 module HTTP;
 
 export {
+	redef record HTTP::Info += {
+		## Number of MIME entities in the HTTP request message body so far.
+		request_mime_level: count &default=0;
+		## Number of MIME entities in the HTTP response message body so far.
+		response_mime_level: count &default=0;
+	};
+
 	## Default file handle provider for HTTP.
 	global get_file_handle: function(c: connection, is_orig: bool): string;
 }
 
+event http_begin_entity(c: connection, is_orig: bool) &priority=5
+	{
+	if ( ! c?$http )
+		return;
+
+	if ( is_orig )
+		++c$http$request_mime_level;
+	else
+		++c$http$response_mime_level;
+	}
+
 function get_file_handle(c: connection, is_orig: bool): string
 	{
 	if ( ! c?$http ) return "";
 
+	local mime_level: count =
+	        is_orig ? c$http$request_mime_level : c$http$response_mime_level;
+	local mime_level_str: string = mime_level > 1 ? cat(mime_level) : "";
+
 	if ( c$http$range_request )
 		return cat(Analyzer::ANALYZER_HTTP, " ", is_orig, " ", c$id$orig_h, " ",
 		           build_url(c$http));
 
 	return cat(Analyzer::ANALYZER_HTTP, " ", c$start_time, " ", is_orig, " ",
-	           c$http$trans_depth, " ", id_string(c$id));
+	           c$http$trans_depth, mime_level_str, " ", id_string(c$id));
 	}
 
 module GLOBAL;
 
 event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool)
+	&priority=5
 	{
 	if ( tag != Analyzer::ANALYZER_HTTP ) return;
 	set_file_handle(HTTP::get_file_handle(c, is_orig));

scripts/base/protocols/http/file-extract.bro

@@ -14,8 +14,11 @@ export {
 	const extraction_prefix = "http-item" &redef;
 
 	redef record Info += {
-		## On-disk file where the response body was extracted to.
-		extraction_file:  string &log &optional;
+		## On-disk location where files in request body were extracted.
+		extracted_request_files: vector of string &log &optional;
+
+		## On-disk location where files in response body were extracted.
+		extracted_response_files: vector of string &log &optional;
 		
 		## Indicates if the response body is to be extracted or not.  Must be 
 		## set before or by the first :bro:see:`file_new` for the file content.
@@ -23,15 +26,28 @@ export {
 	};
 }
 
-global extract_count: count = 0;
-
 function get_extraction_name(f: fa_file): string
 	{
-	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
-	++extract_count;
+	local r = fmt("%s-%s.dat", extraction_prefix, f$id);
 	return r;
 	}
 
+function add_extraction_file(c: connection, is_orig: bool, fn: string)
+	{
+	if ( is_orig )
+		{
+		if ( ! c$http?$extracted_request_files )
+			c$http$extracted_request_files = vector();
+		c$http$extracted_request_files[|c$http$extracted_request_files|] = fn;
+		}
+	else
+		{
+		if ( ! c$http?$extracted_response_files )
+			c$http$extracted_response_files = vector();
+		c$http$extracted_response_files[|c$http$extracted_response_files|] = fn;
+		}
+	}
+
 event file_new(f: fa_file) &priority=5
 	{
 	if ( ! f?$source ) return;
@@ -51,7 +67,7 @@ event file_new(f: fa_file) &priority=5
 			{
 			c = f$conns[cid];
 			if ( ! c?$http ) next;
-			c$http$extraction_file = fname;
+			add_extraction_file(c, f$is_orig, fname);
 			}
 
 		return;
@@ -79,6 +95,6 @@ event file_new(f: fa_file) &priority=5
 			{
 			c = f$conns[cid];
 			if ( ! c?$http ) next;
-			c$http$extraction_file = fname;
+			add_extraction_file(c, f$is_orig, fname);
 			}
 	}

scripts/base/protocols/irc/dcc-send.bro

@@ -39,8 +39,6 @@ export {
 
 global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins;
 
-global extract_count: count = 0;
-
 function set_dcc_mime(f: fa_file)
 	{
 	if ( ! f?$conns ) return;
@@ -75,8 +73,7 @@ function set_dcc_extraction_file(f: fa_file, filename: string)
 
 function get_extraction_name(f: fa_file): string
 	{
-	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
-	++extract_count;
+	local r = fmt("%s-%s.dat", extraction_prefix, f$id);
 	return r;
 	}
 

scripts/base/protocols/irc/file-analysis.bro

@@ -18,6 +18,7 @@ function get_file_handle(c: connection, is_orig: bool): string
 module GLOBAL;
 
 event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool)
+	&priority=5
 	{
 	if ( tag != Analyzer::ANALYZER_IRC_DATA ) return;
 	set_file_handle(IRC::get_file_handle(c, is_orig));

scripts/base/protocols/smtp/entities.bro

@@ -66,8 +66,6 @@ export {
 	global log_mime: event(rec: EntityInfo);
 }
 
-global extract_count: count = 0;
-
 event bro_init() &priority=5
 	{
 	Log::create_stream(SMTP::ENTITIES_LOG, [$columns=EntityInfo, $ev=log_mime]);
@@ -90,8 +88,7 @@ function set_session(c: connection, new_entity: bool)
 
 function get_extraction_name(f: fa_file): string
 	{
-	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
-	++extract_count;
+	local r = fmt("%s-%s.dat", extraction_prefix, f$id);
 	return r;
 	}
 
@@ -127,7 +124,6 @@ event file_new(f: fa_file) &priority=5
 				                           [$tag=FileAnalysis::ANALYZER_EXTRACT,
 				                            $extract_filename=fname]);
 				extracting = T;
-				++extract_count;
 				}
 
 			c$smtp$current_entity$extraction_file = fname;

scripts/base/protocols/smtp/file-analysis.bro

@@ -20,6 +20,7 @@ function get_file_handle(c: connection, is_orig: bool): string
 module GLOBAL;
 
 event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool)
+	&priority=5
 	{
 	if ( tag != Analyzer::ANALYZER_SMTP ) return;
 	set_file_handle(SMTP::get_file_handle(c, is_orig));