Merge remote-tracking branch 'origin/topic/jsiwek/faf-cleanup'

Closes #1002.

* origin/topic/jsiwek/faf-cleanup:
  Move file analyzers to new plugin infrastructure.
  Add a general file analysis overview/how-to document.
  Improve file analysis doxygen comments.
  Improve tracking of HTTP file extraction (addresses #988).
  Fix HTTP multipart body file analysis.
  Remove logging of analyzers field of FileAnalysis::Info.
  Remove extraction counter in default file extraction scripts.
  Remove FileAnalysis::postpone_timeout.
  Make default get_file_handle handlers &priority=5.
  Add input interface to forward data for file analysis.
  File analysis framework interface simplifications.

scripts/base/protocols/http/file-analysis.bro

@@ -6,25 +6,48 @@
 module HTTP;
 
 export {
+	redef record HTTP::Info += {
+		## Number of MIME entities in the HTTP request message body so far.
+		request_mime_level: count &default=0;
+		## Number of MIME entities in the HTTP response message body so far.
+		response_mime_level: count &default=0;
+	};
+
 	## Default file handle provider for HTTP.
 	global get_file_handle: function(c: connection, is_orig: bool): string;
 }
 
+event http_begin_entity(c: connection, is_orig: bool) &priority=5
+	{
+	if ( ! c?$http )
+		return;
+
+	if ( is_orig )
+		++c$http$request_mime_level;
+	else
+		++c$http$response_mime_level;
+	}
+
 function get_file_handle(c: connection, is_orig: bool): string
 	{
 	if ( ! c?$http ) return "";
 
+	local mime_level: count =
+	        is_orig ? c$http$request_mime_level : c$http$response_mime_level;
+	local mime_level_str: string = mime_level > 1 ? cat(mime_level) : "";
+
 	if ( c$http$range_request )
 		return cat(Analyzer::ANALYZER_HTTP, " ", is_orig, " ", c$id$orig_h, " ",
 		           build_url(c$http));
 
 	return cat(Analyzer::ANALYZER_HTTP, " ", c$start_time, " ", is_orig, " ",
-	           c$http$trans_depth, " ", id_string(c$id));
+	           c$http$trans_depth, mime_level_str, " ", id_string(c$id));
 	}
 
 module GLOBAL;
 
 event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool)
+	&priority=5
 	{
 	if ( tag != Analyzer::ANALYZER_HTTP ) return;
 	set_file_handle(HTTP::get_file_handle(c, is_orig));

scripts/base/protocols/http/file-extract.bro

@@ -14,8 +14,11 @@ export {
 	const extraction_prefix = "http-item" &redef;
 
 	redef record Info += {
-		## On-disk file where the response body was extracted to.
-		extraction_file:  string &log &optional;
+		## On-disk location where files in request body were extracted.
+		extracted_request_files: vector of string &log &optional;
+
+		## On-disk location where files in response body were extracted.
+		extracted_response_files: vector of string &log &optional;
 		
 		## Indicates if the response body is to be extracted or not.  Must be 
 		## set before or by the first :bro:see:`file_new` for the file content.
@@ -23,15 +26,28 @@ export {
 	};
 }
 
-global extract_count: count = 0;
-
 function get_extraction_name(f: fa_file): string
 	{
-	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
-	++extract_count;
+	local r = fmt("%s-%s.dat", extraction_prefix, f$id);
 	return r;
 	}
 
+function add_extraction_file(c: connection, is_orig: bool, fn: string)
+	{
+	if ( is_orig )
+		{
+		if ( ! c$http?$extracted_request_files )
+			c$http$extracted_request_files = vector();
+		c$http$extracted_request_files[|c$http$extracted_request_files|] = fn;
+		}
+	else
+		{
+		if ( ! c$http?$extracted_response_files )
+			c$http$extracted_response_files = vector();
+		c$http$extracted_response_files[|c$http$extracted_response_files|] = fn;
+		}
+	}
+
 event file_new(f: fa_file) &priority=5
 	{
 	if ( ! f?$source ) return;
@@ -51,7 +67,7 @@ event file_new(f: fa_file) &priority=5
 			{
 			c = f$conns[cid];
 			if ( ! c?$http ) next;
-			c$http$extraction_file = fname;
+			add_extraction_file(c, f$is_orig, fname);
 			}
 
 		return;
@@ -79,6 +95,6 @@ event file_new(f: fa_file) &priority=5
 			{
 			c = f$conns[cid];
 			if ( ! c?$http ) next;
-			c$http$extraction_file = fname;
+			add_extraction_file(c, f$is_orig, fname);
 			}
 	}