Merge remote-tracking branch 'origin/topic/jsiwek/faf-cleanup' into topic/seth/faf-updates

doc/file-analysis.rst

@@ -0,0 +1,184 @@
+=============
+File Analysis
+=============
+
+.. rst-class:: opening
+
+    In the past, writing Bro scripts with the intent of analyzing file
+    content could be cumbersome because of the fact that the content
+    would be presented in different ways, via events, at the
+    script-layer depending on which network protocol was involved in the
+    file transfer.  Scripts written to analyze files over one protocol
+    would have to be copied and modified to fit other protocols.  The
+    file analysis framework (FAF) is an attempt to provide a generalized
+    presentation of file-related information.  The information regarding
+    the protocol involved in transporting a file over the network is
+    still available, but it no longer has to dictate how one organizes
+    their scripting logic to handle it.  A goal of the FAF is to
+    provide analysis specifically for files that is analogous to the
+    analysis Bro provides for network connections.
+
+.. contents::
+
+File Lifecycle Events
+=====================
+
+The key events that may occur during the lifetime of a file are:
+:bro:see:`file_new`, :bro:see:`file_over_new_connection`,
+:bro:see:`file_timeout`, :bro:see:`file_gap`, and
+:bro:see:`file_state_remove`.  Handling any of these events provides
+some information about the file such as which network
+:bro:see:`connection` and protocol are transporting the file, how many
+bytes have been transferred so far, and its MIME type.
+
+.. code:: bro
+
+    event connection_state_remove(c: connection)
+        {
+        print "connection_state_remove";
+        print c$uid;
+        print c$id;
+        for ( s in c$service )
+            print s;
+        }
+
+    event file_state_remove(f: fa_file)
+        {
+        print "file_state_remove";
+        print f$id;
+        for ( cid in f$conns )
+            {
+            print f$conns[cid]$uid;
+            print cid;
+            }
+        print f$source;
+        }
+
+might give output like::
+
+    file_state_remove
+    Cx92a0ym5R8
+    REs2LQfVW2j
+    [orig_h=10.0.0.7, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
+    HTTP
+    connection_state_remove
+    REs2LQfVW2j
+    [orig_h=10.0.0.7, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
+    HTTP
+
+This doesn't perform any interesting analysis yet, but does highlight
+the similarity between analysis of connections and files.  Connections
+are identified by the usual 5-tuple or a convenient UID string while
+files are identified just by a string of the same format as the
+connection UID.  So there's unique ways to identify both files and
+connections and files hold references to a connection (or connections)
+that transported it.
+
+Adding Analysis
+===============
+
+There are builtin file analyzers which can be attached to files.  Once
+attached, they start receiving the contents of the file as Bro extracts
+it from an ongoing network connection.  What they do with the file
+contents is up to the particular file analyzer implementation, but
+they'll typically either report further information about the file via
+events (e.g. :bro:see:`FileAnalysis::ANALYZER_MD5` will report the
+file's MD5 checksum via :bro:see:`file_hash` once calculated) or they'll
+have some side effect (e.g. :bro:see:`FileAnalysis::ANALYZER_EXTRACT`
+will write the contents of the file out to the local file system).
+
+In the future there may be file analyzers that automatically attach to
+files based on heuristics, similar to the Dynamic Protocol Detection
+(DPD) framework for connections, but many will always require an
+explicit attachment decision:
+
+.. code:: bro
+
+    event file_new(f: fa_file)
+        {
+        print "new file", f$id;
+        if ( f?$mime_type && f$mime_type == "text/plain" )
+            FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
+        }
+
+    event file_hash(f: fa_file, kind: string, hash: string)
+        {
+        print "file_hash", f$id, kind, hash;
+        }
+
+this script calculates MD5s for all plain text files and might give
+output::
+
+    new file, Cx92a0ym5R8
+    file_hash, Cx92a0ym5R8, md5, 397168fd09991a0e712254df7bc639ac
+
+Some file analyzers might have tunable parameters that need to be
+specified in the call to :bro:see:`FileAnalysis::add_analyzer`:
+
+.. code:: bro
+
+    event file_new(f: fa_file)
+        {
+        FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
+                                       $extract_filename="./myfile"]);
+        }
+
+In this case, the file extraction analyzer doesn't generate any further
+events, but does have the side effect of writing out the file contents
+to the local file system at the specified location of ``./myfile``.  Of
+course, for a network with more than a single file being transferred,
+it's probably preferable to specify a different extraction path for each
+file, unlike this example.
+
+Regardless of which file analyzers end up acting on a file, general
+information about the file (e.g. size, time of last data transferred,
+MIME type, etc.) are logged in ``file_analysis.log``.
+
+Input Framework Integration
+===========================
+
+The FAF comes with a simple way to integrate with the :doc:`Input
+Framework <input>`, so that Bro can analyze files from external sources
+in the same way it analyzes files that it sees coming over traffic from
+a network interface it's monitoring.  It only requires a call to
+:bro:see:`Input::add_analysis`:
+
+.. code:: bro
+
+    redef exit_only_after_terminate = T;
+
+    event file_new(f: fa_file)
+        {
+        print "new file", f$id;
+        FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
+        }
+
+    event file_state_remove(f: fa_file)
+        {
+        Input::remove(f$source);
+        terminate();
+        }
+
+    event file_hash(f: fa_file, kind: string, hash: string)
+        {
+        print "file_hash", f$id, kind, hash;
+        }
+
+    event bro_init()
+        {
+        local source: string = "./myfile";
+        Input::add_analysis([$source=source, $name=source]);
+        }
+
+Note that the "source" field of :bro:see:`fa_file` corresponds to the
+"name" field of :bro:see:`Input::AnalysisDescription` since that is what
+the input framework uses to uniquely identify an input stream.
+
+The output of the above script may be::
+
+    new file, G1fS2xthS4l
+    file_hash, G1fS2xthS4l, md5, 54098b367d2e87b078671fad4afb9dbb
+
+Nothing that special, but it at least verifies the MD5 file analyzer
+saw all the bytes of the input file and calculated the checksum
+correctly!

doc/index.rst

@@ -25,6 +25,7 @@ Frameworks
    notice
    logging
    input
+   file-analysis
    cluster
    signatures
 

scripts/base/frameworks/file-analysis/main.bro

@@ -22,11 +22,13 @@ export {
 		extract_filename: string &optional;
 
 		## An event which will be generated for all new file contents,
-		## chunk-wise.
+		## chunk-wise.  Used when *tag* is
+		## :bro:see:`FileAnalysis::ANALYZER_DATA_EVENT`.
 		chunk_event: event(f: fa_file, data: string, off: count) &optional;
 
 		## An event which will be generated for all new file contents,
-		## stream-wise.
+		## stream-wise.  Used when *tag* is
+		## :bro:see:`FileAnalysis::ANALYZER_DATA_EVENT`.
 		stream_event: event(f: fa_file, data: string) &optional;
 	} &redef;
 
@@ -90,7 +92,7 @@ export {
 		conn_uids: set[string] &log;
 
 		## A set of analysis types done during the file analysis.
-		analyzers: set[Analyzer] &log;
+		analyzers: set[Analyzer];
 
 		## Local filenames of extracted files.
 		extracted_files: set[string] &log;
@@ -123,7 +125,9 @@ export {
 
 	## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is
 	## used to determine the length of inactivity that is allowed for a file
-	## before internal state related to it is cleaned up.
+	## before internal state related to it is cleaned up.  When used within a
+	## :bro:see:`file_timeout` handler, the analysis will delay timing out
+	## again for the period specified by *t*.
 	##
 	## f: the file.
 	##
@@ -133,18 +137,6 @@ export {
 	##          for the *id* isn't currently active.
 	global set_timeout_interval: function(f: fa_file, t: interval): bool;
 
-	## Postpones the timeout of file analysis for a given file.
-	## When used within a :bro:see:`file_timeout` handler for, the analysis
-	## the analysis will delay timing out for the period of time indicated by
-	## the *timeout_interval* field of :bro:see:`fa_file`, which can be set
-	## with :bro:see:`FileAnalysis::set_timeout_interval`.
-	##
-	## f: the file.
-	##
-	## Returns: true if the timeout will be postponed, or false if analysis
-	##          for the *id* isn't currently active.
-	global postpone_timeout: function(f: fa_file): bool;
-
 	## Adds an analyzer to the analysis of a given file.
 	##
 	## f: the file.
@@ -174,58 +166,6 @@ export {
 	##          rest of it's contents, or false if analysis for the *id*
 	##          isn't currently active.
 	global stop: function(f: fa_file): bool;
-
-	## Sends a sequential stream of data in for file analysis.
-	## Meant for use when providing external file analysis input (e.g.
-	## from the input framework).
-	##
-	## source: a string that uniquely identifies the logical file that the
-	##         data is a part of and describes its source.
-	##
-	## data: bytestring contents of the file to analyze.
-	global data_stream: function(source: string, data: string);
-
-	## Sends a non-sequential chunk of data in for file analysis.
-	## Meant for use when providing external file analysis input (e.g.
-	## from the input framework).
-	##
-	## source: a string that uniquely identifies the logical file that the
-	##         data is a part of and describes its source.
-	##
-	## data: bytestring contents of the file to analyze.
-	##
-	## offset: the offset within the file that this chunk starts.
-	global data_chunk: function(source: string, data: string, offset: count);
-
-	## Signals a content gap in the file bytestream.
-	## Meant for use when providing external file analysis input (e.g.
-	## from the input framework).
-	##
-	## source: a string that uniquely identifies the logical file that the
-	##         data is a part of and describes its source.
-	##
-	## offset: the offset within the file that this gap starts.
-	##
-	## len: the number of bytes that are missing.
-	global gap: function(source: string, offset: count, len: count);
-
-	## Signals the total size of a file.
-	## Meant for use when providing external file analysis input (e.g.
-	## from the input framework).
-	##
-	## source: a string that uniquely identifies the logical file that the
-	##         data is a part of and describes its source.
-	##
-	## size: the number of bytes that comprise the full file.
-	global set_size: function(source: string, size: count);
-
-	## Signals the end of a file.
-	## Meant for use when providing external file analysis input (e.g.
-	## from the input framework).
-	##
-	## source: a string that uniquely identifies the logical file that the
-	##         data is a part of and describes its source.
-	global eof: function(source: string);
 }
 
 redef record fa_file += {
@@ -272,11 +212,6 @@ function set_timeout_interval(f: fa_file, t: interval): bool
 	return __set_timeout_interval(f$id, t);
 	}
 
-function postpone_timeout(f: fa_file): bool
-	{
-	return __postpone_timeout(f$id);
-	}
-
 function add_analyzer(f: fa_file, args: AnalyzerArgs): bool
 	{
 	if ( ! __add_analyzer(f$id, args) ) return F;
@@ -300,31 +235,6 @@ function stop(f: fa_file): bool
 	return __stop(f$id);
 	}
 
-function data_stream(source: string, data: string)
-	{
-	__data_stream(source, data);
-	}
-
-function data_chunk(source: string, data: string, offset: count)
-	{
-	__data_chunk(source, data, offset);
-	}
-
-function gap(source: string, offset: count, len: count)
-	{
-	__gap(source, offset, len);
-	}
-
-function set_size(source: string, size: count)
-	{
-	__set_size(source, size);
-	}
-
-function eof(source: string)
-	{
-	__eof(source);
-	}
-
 event bro_init() &priority=5
 	{
 	Log::create_stream(FileAnalysis::LOG,

scripts/base/frameworks/input/main.bro

@@ -122,6 +122,35 @@ export {
 		config: table[string] of string &default=table();
 	};
 
+	## A file analyis input stream type used to forward input data to the
+	## file analysis framework.
+	type AnalysisDescription: record {
+
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+
+		## Reader to use for this steam.  Compatible readers must be
+		## able to accept a filter of a single string type (i.e.
+		## they read a byte stream).
+		reader: Reader &default=Input::READER_BINARY;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Descriptive name that uniquely identifies the input source.
+		## Can be used used to remove a stream at a later time.
+		## This will also be used for the unique *source* field of
+		## :bro:see:`fa_file`.  Most of the time, the best choice for this
+		## field will be the same value as the *source* field.
+		name: string;
+
+		## A key/value table that will be passed on the reader.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+		config: table[string] of string &default=table();
+	};
+
 	## Create a new table input from a given source. Returns true on success.
 	##
 	## description: `TableDescription` record describing the source.
@@ -132,6 +161,14 @@ export {
 	## description: `TableDescription` record describing the source.
 	global add_event: function(description: Input::EventDescription) : bool;
 
+	## Create a new file analysis input from a given source.  Data read from
+	## the source is automatically forwarded to the file analysis framework.
+	##
+	## description: A record describing the source
+	##
+	## Returns: true on sucess.
+	global add_analysis: function(description: Input::AnalysisDescription) : bool;
+
 	## Remove a input stream. Returns true on success and false if the named stream was
 	## not found.
 	##
@@ -164,6 +201,11 @@ function add_event(description: Input::EventDescription) : bool
 	return __create_event_stream(description);
 	}
 
+function add_analysis(description: Input::AnalysisDescription) : bool
+	{
+	return __create_analysis_stream(description);
+	}
+
 function remove(id: string) : bool
 	{
 	return __remove_stream(id);

scripts/base/protocols/ftp/file-analysis.bro

@@ -41,6 +41,7 @@ function get_file_handle(c: connection, is_orig: bool): string
 module GLOBAL;
 
 event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
+	&priority=5
 	{
 	if ( tag != ANALYZER_FTP_DATA ) return;
 	set_file_handle(FTP::get_file_handle(c, is_orig));

scripts/base/protocols/ftp/file-extract.bro

@@ -13,8 +13,6 @@ export {
 	const extraction_prefix = "ftp-item" &redef;
 }
 
-global extract_count: count = 0;
-
 redef record Info += {
 	## On disk file where it was extracted to.
 	extraction_file:       string &log &optional;
@@ -26,8 +24,7 @@ redef record Info += {
 
 function get_extraction_name(f: fa_file): string
 	{
-	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
+	local r = fmt("%s-%s.dat", extraction_prefix, f$id);
-	++extract_count;
 	return r;
 	}
 

scripts/base/protocols/http/file-analysis.bro

@@ -6,25 +6,47 @@
 module HTTP;
 
 export {
+	redef record HTTP::Info += {
+		## Number of MIME entities in the HTTP request message body so far.
+		request_mime_level: count &default=0;
+		## Number of MIME entities in the HTTP response message body so far.
+		response_mime_level: count &default=0;
+	};
+
 	## Default file handle provider for HTTP.
 	global get_file_handle: function(c: connection, is_orig: bool): string;
 }
 
+event http_begin_entity(c: connection, is_orig: bool) &priority=5
+	{
+	if ( ! c?$http ) return;
+
+	if ( is_orig )
+		++c$http$request_mime_level;
+	else
+		++c$http$response_mime_level;
+	}
+
 function get_file_handle(c: connection, is_orig: bool): string
 	{
 	if ( ! c?$http ) return "";
 
+	local mime_level: count =
+	        is_orig ? c$http$request_mime_level : c$http$response_mime_level;
+	local mime_level_str: string = mime_level > 1 ? cat(mime_level) : "";
+
 	if ( c$http$range_request )
 		return cat(ANALYZER_HTTP, " ", is_orig, " ", c$id$orig_h, " ",
 		           build_url(c$http));
 
 	return cat(ANALYZER_HTTP, " ", c$start_time, " ", is_orig, " ",
-	           c$http$trans_depth, " ", id_string(c$id));
+	           c$http$trans_depth, mime_level_str, " ", id_string(c$id));
 	}
 
 module GLOBAL;
 
 event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
+	&priority=5
 	{
 	if ( tag != ANALYZER_HTTP ) return;
 	set_file_handle(HTTP::get_file_handle(c, is_orig));

scripts/base/protocols/http/file-extract.bro

@@ -14,8 +14,11 @@ export {
 	const extraction_prefix = "http-item" &redef;
 
 	redef record Info += {
-		## On-disk file where the response body was extracted to.
+		## On-disk location where files in request body were extracted.
-		extraction_file:  string &log &optional;
+		extracted_request_files: vector of string &log &optional;
+
+		## On-disk location where files in response body were extracted.
+		extracted_response_files: vector of string &log &optional;
 		
 		## Indicates if the response body is to be extracted or not.  Must be 
 		## set before or by the first :bro:see:`file_new` for the file content.
@@ -23,15 +26,28 @@ export {
 	};
 }
 
-global extract_count: count = 0;
-
 function get_extraction_name(f: fa_file): string
 	{
-	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
+	local r = fmt("%s-%s.dat", extraction_prefix, f$id);
-	++extract_count;
 	return r;
 	}
 
+function add_extraction_file(c: connection, is_orig: bool, fn: string)
+	{
+	if ( is_orig )
+		{
+		if ( ! c$http?$extracted_request_files )
+			c$http$extracted_request_files = vector();
+		c$http$extracted_request_files[|c$http$extracted_request_files|] = fn;
+		}
+	else
+		{
+		if ( ! c$http?$extracted_response_files )
+			c$http$extracted_response_files = vector();
+		c$http$extracted_response_files[|c$http$extracted_response_files|] = fn;
+		}
+	}
+
 event file_new(f: fa_file) &priority=5
 	{
 	if ( ! f?$source ) return;
@@ -51,7 +67,7 @@ event file_new(f: fa_file) &priority=5
 			{
 			c = f$conns[cid];
 			if ( ! c?$http ) next;
-			c$http$extraction_file = fname;
+			add_extraction_file(c, f$is_orig, fname);
 			}
 
 		return;
@@ -79,6 +95,6 @@ event file_new(f: fa_file) &priority=5
 			{
 			c = f$conns[cid];
 			if ( ! c?$http ) next;
-			c$http$extraction_file = fname;
+			add_extraction_file(c, f$is_orig, fname);
 			}
 	}

scripts/base/protocols/irc/dcc-send.bro

@@ -39,8 +39,6 @@ export {
 
 global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins;
 
-global extract_count: count = 0;
-
 function set_dcc_mime(f: fa_file)
 	{
 	if ( ! f?$conns ) return;
@@ -75,8 +73,7 @@ function set_dcc_extraction_file(f: fa_file, filename: string)
 
 function get_extraction_name(f: fa_file): string
 	{
-	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
+	local r = fmt("%s-%s.dat", extraction_prefix, f$id);
-	++extract_count;
 	return r;
 	}
 

scripts/base/protocols/irc/file-analysis.bro

@@ -18,6 +18,7 @@ function get_file_handle(c: connection, is_orig: bool): string
 module GLOBAL;
 
 event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
+	&priority=5
 	{
 	if ( tag != ANALYZER_IRC_DATA ) return;
 	set_file_handle(IRC::get_file_handle(c, is_orig));

scripts/base/protocols/smtp/entities.bro

@@ -66,8 +66,6 @@ export {
 	global log_mime: event(rec: EntityInfo);
 }
 
-global extract_count: count = 0;
-
 event bro_init() &priority=5
 	{
 	Log::create_stream(SMTP::ENTITIES_LOG, [$columns=EntityInfo, $ev=log_mime]);
@@ -90,8 +88,7 @@ function set_session(c: connection, new_entity: bool)
 
 function get_extraction_name(f: fa_file): string
 	{
-	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
+	local r = fmt("%s-%s.dat", extraction_prefix, f$id);
-	++extract_count;
 	return r;
 	}
 
@@ -127,7 +124,6 @@ event file_new(f: fa_file) &priority=5
 				                           [$tag=FileAnalysis::ANALYZER_EXTRACT,
 				                            $extract_filename=fname]);
 				extracting = T;
-				++extract_count;
 				}
 
 			c$smtp$current_entity$extraction_file = fname;

scripts/base/protocols/smtp/file-analysis.bro

@@ -20,6 +20,7 @@ function get_file_handle(c: connection, is_orig: bool): string
 module GLOBAL;
 
 event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
+	&priority=5
 	{
 	if ( tag != ANALYZER_SMTP ) return;
 	set_file_handle(SMTP::get_file_handle(c, is_orig));

src/CMakeLists.txt

@@ -464,7 +464,6 @@ set(bro_SRCS
     file_analysis/Manager.cc
     file_analysis/File.cc
     file_analysis/FileTimer.cc
-    file_analysis/FileID.h
     file_analysis/Analyzer.h
     file_analysis/AnalyzerSet.cc
     file_analysis/Extract.cc

src/event.bif

@@ -7024,7 +7024,7 @@ event file_over_new_connection%(f: fa_file, c: connection%);
 ## f: The file.
 ##
 ## .. bro:see:: file_new file_over_new_connection file_gap file_state_remove
-##    default_file_timeout_interval FileAnalysis::postpone_timeout
+##    default_file_timeout_interval FileAnalysis::set_timeout_interval
 ##    FileAnalysis::set_timeout_interval
 event file_timeout%(f: fa_file%);
 

src/file_analysis.bif

@@ -27,30 +27,19 @@ enum Analyzer %{
 	ANALYZER_DATA_EVENT,
 %}
 
-## :bro:see:`FileAnalysis::postpone_timeout`.
-function FileAnalysis::__postpone_timeout%(file_id: string%): bool
-	%{
-	using file_analysis::FileID;
-	bool result = file_mgr->PostponeTimeout(FileID(file_id->CheckString()));
-	return new Val(result, TYPE_BOOL);
-	%}
-
 ## :bro:see:`FileAnalysis::set_timeout_interval`.
 function FileAnalysis::__set_timeout_interval%(file_id: string, t: interval%): bool
 	%{
-	using file_analysis::FileID;
+	bool result = file_mgr->SetTimeoutInterval(file_id->CheckString(), t);
-	bool result = file_mgr->SetTimeoutInterval(FileID(file_id->CheckString()),
-	                                           t);
 	return new Val(result, TYPE_BOOL);
 	%}
 
 ## :bro:see:`FileAnalysis::add_analyzer`.
 function FileAnalysis::__add_analyzer%(file_id: string, args: any%): bool
 	%{
-	using file_analysis::FileID;
 	using BifType::Record::FileAnalysis::AnalyzerArgs;
 	RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
-	bool result = file_mgr->AddAnalyzer(FileID(file_id->CheckString()), rv);
+	bool result = file_mgr->AddAnalyzer(file_id->CheckString(), rv);
 	Unref(rv);
 	return new Val(result, TYPE_BOOL);
 	%}
@@ -58,10 +47,9 @@ function FileAnalysis::__add_analyzer%(file_id: string, args: any%): bool
 ## :bro:see:`FileAnalysis::remove_analyzer`.
 function FileAnalysis::__remove_analyzer%(file_id: string, args: any%): bool
 	%{
-	using file_analysis::FileID;
 	using BifType::Record::FileAnalysis::AnalyzerArgs;
 	RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
-	bool result = file_mgr->RemoveAnalyzer(FileID(file_id->CheckString()), rv);
+	bool result = file_mgr->RemoveAnalyzer(file_id->CheckString(), rv);
 	Unref(rv);
 	return new Val(result, TYPE_BOOL);
 	%}
@@ -69,47 +57,10 @@ function FileAnalysis::__remove_analyzer%(file_id: string, args: any%): bool
 ## :bro:see:`FileAnalysis::stop`.
 function FileAnalysis::__stop%(file_id: string%): bool
 	%{
-	using file_analysis::FileID;
+	bool result = file_mgr->IgnoreFile(file_id->CheckString());
-	bool result = file_mgr->IgnoreFile(FileID(file_id->CheckString()));
 	return new Val(result, TYPE_BOOL);
 	%}
 
-## :bro:see:`FileAnalysis::data_stream`.
-function FileAnalysis::__data_stream%(source: string, data: string%): any
-	%{
-	file_mgr->DataIn(data->Bytes(), data->Len(), source->CheckString());
-	return 0;
-	%}
-
-## :bro:see:`FileAnalysis::data_chunk`.
-function FileAnalysis::__data_chunk%(source: string, data: string,
-                                     offset: count%): any
-	%{
-	file_mgr->DataIn(data->Bytes(), data->Len(), offset, source->CheckString());
-	return 0;
-	%}
-
-## :bro:see:`FileAnalysis::gap`.
-function FileAnalysis::__gap%(source: string, offset: count, len: count%): any
-	%{
-	file_mgr->Gap(offset, len, source->CheckString());
-	return 0;
-	%}
-
-## :bro:see:`FileAnalysis::set_size`.
-function FileAnalysis::__set_size%(source: string, size: count%): any
-	%{
-	file_mgr->SetSize(size, source->CheckString());
-	return 0;
-	%}
-
-## :bro:see:`FileAnalysis::eof`.
-function FileAnalysis::__eof%(source: string%): any
-	%{
-	file_mgr->EndOfFile(source->CheckString());
-	return 0;
-	%}
-
 module GLOBAL;
 
 ## For use within a :bro:see:`get_file_handle` handler to set a unique

src/file_analysis/Analyzer.h

@@ -17,6 +17,11 @@ class File;
  */
 class Analyzer {
 public:
+
+	/**
+	 * Destructor.  Nothing special about it. Virtual since we definitely expect
+	 * to delete instances of derived classes via pointers to this class.
+	 */
 	virtual ~Analyzer()
 		{
 		DBG_LOG(DBG_FILE_ANALYSIS, "Destroy file analyzer %d", tag);
@@ -24,7 +29,10 @@ public:
 		}
 
 	/**
-	 * Subclasses may override this to receive file data non-sequentially.
+	 * Subclasses may override this metod to receive file data non-sequentially.
+	 * @param data points to start of a chunk of file data.
+	 * @param len length in bytes of the chunk of data pointed to by \a data.
+	 * @param offset the byte offset within full file that data chunk starts.
 	 * @return true if the analyzer is still in a valid state to continue
 	 *         receiving data/events or false if it's essentially "done".
 	 */
@@ -32,7 +40,9 @@ public:
 		{ return true; }
 
 	/**
-	 * Subclasses may override this to receive file sequentially.
+	 * Subclasses may override this method to receive file sequentially.
+	 * @param data points to start of the next chunk of file data.
+	 * @param len length in bytes of the chunk of data pointed to by \a data.
 	 * @return true if the analyzer is still in a valid state to continue
 	 *         receiving data/events or false if it's essentially "done".
 	 */
@@ -40,7 +50,7 @@ public:
 		{ return true; }
 
 	/**
-	 * Subclasses may override this to specifically handle an EOF signal,
+	 * Subclasses may override this method to specifically handle an EOF signal,
 	 * which means no more data is going to be incoming and the analyzer
 	 * may be deleted/cleaned up soon.
 	 * @return true if the analyzer is still in a valid state to continue
@@ -50,7 +60,10 @@ public:
 		{ return true; }
 
 	/**
-	 * Subclasses may override this to handle missing data in a file stream.
+	 * Subclasses may override this method to handle missing data in a file.
+	 * @param offset the byte offset within full file at which the missing
+	 *        data chunk occurs.
+	 * @param len the number of missing bytes.
 	 * @return true if the analyzer is still in a valid state to continue
 	 *         receiving data/events or false if it's essentially "done".
 	 */
@@ -73,8 +86,10 @@ public:
 	File* GetFile() const { return file; }
 
 	/**
+	 * Retrieves an analyzer tag field from full analyzer argument record.
+	 * @param args an \c AnalyzerArgs (script-layer type) value.
 	 * @return the analyzer tag equivalent of the 'tag' field from the
-	 *         AnalyzerArgs value \a args.
+	 *         \c AnalyzerArgs value \a args.
 	 */
 	static FA_Tag ArgsTag(const RecordVal* args)
 		{
@@ -84,6 +99,13 @@ public:
 		}
 
 protected:
+
+	/**
+	 * Constructor.  Only derived classes are meant to be instantiated.
+	 * @param arg_args an \c AnalyzerArgs (script-layer type) value specifiying
+	 *        tunable options, if any, related to a particular analyzer type.
+	 * @param arg_file the file to which the the analyzer is being attached.
+	 */
 	Analyzer(RecordVal* arg_args, File* arg_file)
 	    : tag(file_analysis::Analyzer::ArgsTag(arg_args)),
 	      args(arg_args->Ref()->AsRecordVal()),
@@ -91,9 +113,10 @@ protected:
 		{}
 
 private:
-	FA_Tag tag;
+
-	RecordVal* args;
+	FA_Tag tag; /**< The particular analyzer type of the analyzer instance. */
-	File* file;
+	RecordVal* args; /**< \c AnalyzerArgs val gives tunable analyzer params. */
+	File* file; /**< The file to which the analyzer is attached. */
 };
 
 typedef file_analysis::Analyzer* (*AnalyzerInstantiator)(RecordVal* args,

src/file_analysis/AnalyzerSet.h

@@ -16,67 +16,144 @@ class File;
 declare(PDict,Analyzer);
 
 /**
- * A set of file analysis analyzers indexed by AnalyzerArgs.  Allows queueing
+ * A set of file analysis analyzers indexed by an \c AnalyzerArgs (script-layer
- * of addition/removals so that those modifications can happen at well-defined
+ * type) value.  Allows queueing of addition/removals so that those
- * times (e.g. to make sure a loop iterator isn't invalidated).
+ * modifications can happen at well-defined times (e.g. to make sure a loop
+ * iterator isn't invalidated).
  */
 class AnalyzerSet {
 public:
+
+	/**
+	 * Constructor.  Nothing special.
+	 * @param arg_file the file to which all analyzers in the set are attached.
+	 */
 	AnalyzerSet(File* arg_file);
 
+	/**
+	 * Destructor.  Any queued analyzer additions/removals are aborted and
+	 * will not occur.
+	 */
 	~AnalyzerSet();
 
 	/**
+	 * Attach an analyzer to #file immediately.
+	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
 	 * @return true if analyzer was instantiated/attached, else false.
 	 */
 	bool Add(RecordVal* args);
 
 	/**
+	 * Queue the attachment of an analyzer to #file.
+	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
 	 * @return true if analyzer was able to be instantiated, else false.
 	 */
 	bool QueueAdd(RecordVal* args);
 
 	/**
+	 * Remove an analyzer from #file immediately.
+	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
 	 * @return false if analyzer didn't exist and so wasn't removed, else true.
 	 */
 	bool Remove(const RecordVal* args);
 
 	/**
+	 * Queue the removal of an analyzer from #file.
+	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
 	 * @return true if analyzer exists at time of call, else false;
 	 */
 	bool QueueRemove(const RecordVal* args);
 
 	/**
-	 * Perform all queued modifications to the currently active analyzers.
+	 * Perform all queued modifications to the current analyzer set.
 	 */
 	void DrainModifications();
 
+	/**
+	 * Prepare the analyzer set to be iterated over.
+	 * @see Dictionary#InitForIteration
+	 * @return an iterator that may be used to loop over analyzers in the set.
+	 */
 	IterCookie* InitForIteration() const
 		{ return analyzer_map.InitForIteration(); }
 
+	/**
+	 * Get next entry in the analyzer set.
+	 * @see Dictionary#NextEntry
+	 * @param c a set iterator.
+	 * @return the next analyzer in the set or a null pointer if there is no
+	 *         more left (in that case the cookie is also deleted).
+	 */
 	file_analysis::Analyzer* NextEntry(IterCookie* c)
 		{ return analyzer_map.NextEntry(c); }
 
 protected:
+
+	/**
+	 * Get a hash key which represents an analyzer instance.
+	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
+	 * @return the hash key calculated from \a args
+	 */
 	HashKey* GetKey(const RecordVal* args) const;
+
+	/**
+	 * Create an instance of a file analyzer.
+	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
+	 * @return a new file analyzer instance.
+	 */
 	file_analysis::Analyzer* InstantiateAnalyzer(RecordVal* args) const;
+
+	/**
+	 * Insert an analyzer instance in to the set.
+	 * @param a an analyzer instance.
+	 * @param key the hash key which represents the analyzer's \c AnalyzerArgs.
+	 */
 	void Insert(file_analysis::Analyzer* a, HashKey* key);
+
+	/**
+	 * Remove an analyzer instance from the set.
+	 * @param tag enumarator which specifies type of the analyzer to remove,
+	 *        just used for debugging messages.
+	 * @param key the hash key which represents the analyzer's \c AnalyzerArgs.
+	 */
 	bool Remove(FA_Tag tag, HashKey* key);
 
 private:
-	File* file;
+
+	File* file;                                  /**< File which owns the set */
 	CompositeHash* analyzer_hash;                /**< AnalyzerArgs hashes. */
 	PDict(file_analysis::Analyzer) analyzer_map; /**< Indexed by AnalyzerArgs. */
 
+	/**
+	 * Abstract base class for analyzer set modifications.
+	 */
 	class Modification {
 	public:
 		virtual ~Modification() {}
+
+		/**
+		 * Perform the modification on an analyzer set.
+		 * @param set the analyzer set on which the modification will happen.
+		 * @return true if the modification altered \a set.
+		 */
 		virtual bool Perform(AnalyzerSet* set) = 0;
+
+		/**
+		 * Don't perform the modification on the analyzer set and clean up.
+		 */
 		virtual void Abort() = 0;
 	};
 
+	/**
+	 * Represents a request to add an analyzer to an analyzer set.
+	 */
 	class AddMod : public Modification {
 	public:
+		/**
+		 * Construct request which can add an analyzer to an analyzer set.
+		 * @param arg_a an analyzer instance to add to an analyzer set.
+		 * @param arg_key hash key representing the analyzer's \c AnalyzerArgs.
+		 */
 		AddMod(file_analysis::Analyzer* arg_a, HashKey* arg_key)
 			: Modification(), a(arg_a), key(arg_key) {}
 		virtual ~AddMod() {}
@@ -88,8 +165,16 @@ private:
 		HashKey* key;
 	};
 
+	/**
+	 * Represents a request to remove an analyzer from an analyzer set.
+	 */
 	class RemoveMod : public Modification {
 	public:
+		/**
+		 * Construct request which can remove an analyzer from an analyzer set.
+		 * @param arg_a an analyzer instance to add to an analyzer set.
+		 * @param arg_key hash key representing the analyzer's \c AnalyzerArgs.
+		 */
 		RemoveMod(FA_Tag arg_tag, HashKey* arg_key)
 			: Modification(), tag(arg_tag), key(arg_key) {}
 		virtual ~RemoveMod() {}
@@ -102,7 +187,7 @@ private:
 	};
 
 	typedef queue<Modification*> ModQueue;
-	ModQueue mod_queue;
+	ModQueue mod_queue; /**< A queue of analyzer additions/removals requests. */
 };
 
 } // namespace file_analysiss

src/file_analysis/DataEvent.h

@@ -12,17 +12,50 @@
 namespace file_analysis {
 
 /**
- * An analyzer to send file data to script-layer events.
+ * An analyzer to send file data to script-layer via events.
  */
 class DataEvent : public file_analysis::Analyzer {
 public:
+
+	/**
+	 * Generates the event, if any, specified by the "chunk_event" field of this
+	 * analyzer's \c AnalyzerArgs.  This is for non-sequential file data input.
+	 * @param data pointer to start of file data chunk.
+	 * @param len number of bytes in the data chunk.
+	 * @param offset number of bytes from start of file at which chunk occurs.
+	 * @return always true
+	 */
 	virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);
 
+	/**
+	 * Generates the event, if any, specified by the "stream_event" field of
+	 * this analyzer's \c AnalyzerArgs.  This is for sequential file data input.
+	 * @param data pointer to start of file data chunk.
+	 * @param len number of bytes in the data chunk.
+	 * @return always true
+	 */
 	virtual bool DeliverStream(const u_char* data, uint64 len);
 
+	/**
+	 * Create a new instance of a DataEvent analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new DataEvent analyzer instance or a null pointer if
+	 *         no "chunk_event" or "stream_event" field was specfied in \a args.
+	 */
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
 
 protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @param ce pointer to event handler which will be called to receive
+	 *        non-sequential file data.
+	 * @param se pointer to event handler which will be called to receive
+	 *        sequential file data.
+	 */
 	DataEvent(RecordVal* args, File* file,
 	          EventHandlerPtr ce, EventHandlerPtr se);
 

src/file_analysis/Extract.h

@@ -12,17 +12,44 @@
 namespace file_analysis {
 
 /**
- * An analyzer to extract files to disk.
+ * An analyzer to extract content of files to local disk.
  */
 class Extract : public file_analysis::Analyzer {
 public:
+
+	/**
+	 * Destructor.  Will close the file that was used for data extraction.
+	 */
 	virtual ~Extract();
 
+	/**
+	 * Write a chunk of file data to the local extraction file.
+	 * @param data pointer to a chunk of file data.
+	 * @param len number of bytes in the data chunk.
+	 * @param offset number of bytes from start of file at which chunk starts.
+	 * @return false if there was no extraction file open and the data couldn't
+	 *         be written, else true.
+	 */
 	virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);
 
+	/**
+	 * Create a new instance of an Extract analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new Extract analyzer instance or a null pointer if the
+	 *         the "extraction_file" field of \a args wasn't set.
+	 */
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
 
 protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @param arg_filename a file system path which specifies the local file
+	 *        to which the contents of the file will be extracted/written.
+	 */
 	Extract(RecordVal* args, File* file, const string& arg_filename);
 
 private:

src/file_analysis/File.cc

@@ -1,11 +1,9 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include <string>
-#include <openssl/md5.h>
 
 #include "File.h"
 #include "FileTimer.h"
-#include "FileID.h"
 #include "Analyzer.h"
 #include "Manager.h"
 #include "Reporter.h"
@@ -51,8 +49,6 @@ int File::bof_buffer_size_idx = -1;
 int File::bof_buffer_idx = -1;
 int File::mime_type_idx = -1;
 
-string File::salt;
-
 void File::StaticInit()
 	{
 	if ( id_idx != -1 )
@@ -72,42 +68,27 @@ void File::StaticInit()
 	bof_buffer_size_idx = Idx("bof_buffer_size");
 	bof_buffer_idx = Idx("bof_buffer");
 	mime_type_idx = Idx("mime_type");
-
-	salt = BifConst::FileAnalysis::salt->CheckString();
 	}
 
-File::File(const string& unique, Connection* conn, AnalyzerTag::Tag tag,
+File::File(const string& file_id, Connection* conn, AnalyzerTag::Tag tag,
            bool is_orig)
-	: id(""), unique(unique), val(0), postpone_timeout(false),
+	: id(file_id), val(0), postpone_timeout(false), first_chunk(true),
-		first_chunk(true), missed_bof(false), need_reassembly(false), done(false),
+	  missed_bof(false), need_reassembly(false), done(false), analyzers(this)
-		analyzers(this)
 	{
 	StaticInit();
 
-	char tmp[20];
+	DBG_LOG(DBG_FILE_ANALYSIS, "Creating new File object %s", file_id.c_str());
-	uint64 hash[2];
-	string msg(unique + salt);
-	MD5(reinterpret_cast<const u_char*>(msg.data()), msg.size(),
-	    reinterpret_cast<u_char*>(hash));
-	uitoa_n(hash[0], tmp, sizeof(tmp), 62);
-
-	DBG_LOG(DBG_FILE_ANALYSIS, "Creating new File object %s (%s)", tmp,
-	        unique.c_str());
 
 	val = new RecordVal(fa_file_type);
-	val->Assign(id_idx, new StringVal(tmp));
+	val->Assign(id_idx, new StringVal(file_id.c_str()));
-	id = FileID(tmp);
 
 	if ( conn )
 		{
 		// add source, connection, is_orig fields
-		val->Assign(source_idx, new StringVal(::Analyzer::GetTagName(tag)));
+		SetSource(::Analyzer::GetTagName(tag));
 		val->Assign(is_orig_idx, new Val(is_orig, TYPE_BOOL));
 		UpdateConnectionFields(conn);
 		}
-	else
-		// use the unique file handle as source
-		val->Assign(source_idx, new StringVal(unique.c_str()));
 
 	UpdateLastActivityTime();
 	}
@@ -187,6 +168,18 @@ int File::Idx(const string& field)
 	return rval;
 	}
 
+string File::GetSource() const
+	{
+	Val* v = val->Lookup(source_idx);
+
+	return v ? v->AsString()->CheckString() : string();
+	}
+
+void File::SetSource(const string& source)
+	{
+	val->Assign(source_idx, new StringVal(source.c_str()));
+	}
+
 double File::GetTimeoutInterval() const
 	{
 	return LookupFieldDefaultInterval(timeout_interval_idx);
@@ -423,7 +416,7 @@ void File::Gap(uint64 offset, uint64 len)
 
 bool File::FileEventAvailable(EventHandlerPtr h)
 	{
-	return h && ! file_mgr->IsIgnored(unique);
+	return h && ! file_mgr->IsIgnored(id);
 	}
 
 void File::FileEvent(EventHandlerPtr h)

src/file_analysis/File.h

@@ -10,7 +10,6 @@
 #include "Conn.h"
 #include "Val.h"
 #include "AnalyzerSet.h"
-#include "FileID.h"
 #include "BroString.h"
 
 namespace file_analysis {
@@ -20,13 +19,30 @@ namespace file_analysis {
  */
 class File {
 public:
+
+	/**
+	 * Destructor.  Nothing fancy, releases a reference to the wrapped
+	 * \c fa_file value.
+	 */
 	~File();
 
 	/**
-	 * @return the #val record.
+	 * @return the wrapped \c fa_file record value, #val.
 	 */
 	RecordVal* GetVal() const { return val; }
 
+	/**
+	 * @return the value of the "source" field from #val record or an empty
+	 * string if it's not initialized.
+	 */
+	string GetSource() const;
+
+	/**
+	 * Set the "source" field from #val record to \a source.
+	 * @param source the new value of the "source" field.
+	 */
+	void SetSource(const string& source);
+
 	/**
 	 * @return value (seconds) of the "timeout_interval" field from #val record.
 	 */
@@ -34,18 +50,14 @@ public:
 
 	/**
 	 * Set the "timeout_interval" field from #val record to \a interval seconds.
+	 * @param interval the new value of the "timeout_interval" field.
 	 */
 	void SetTimeoutInterval(double interval);
 
 	/**
 	 * @return value of the "id" field from #val record.
 	 */
-	FileID GetID() const { return id; }
+	string GetID() const { return id; }
-
-	/**
-	 * @return the string which uniquely identifies the file.
-	 */
-	string GetUnique() const { return unique; }
 
 	/**
 	 * @return value of "last_active" field in #val record;
@@ -59,13 +71,15 @@ public:
 
 	/**
 	 * Set "total_bytes" field of #val record to \a size.
+	 * @param size the new value of the "total_bytes" field.
 	 */
 	void SetTotalBytes(uint64 size);
 
 	/**
-	 * Compares "seen_bytes" field to "total_bytes" field of #val record
+	 * Compares "seen_bytes" field to "total_bytes" field of #val record to
-	 * and returns true if the comparison indicates the full file was seen.
+	 * determine if the full file has been seen.
-	 * If "total_bytes" hasn't been set yet, it returns false.
+	 * @return false if "total_bytes" hasn't been set yet or "seen_bytes" is
+	 *         less than it, else true.
 	 */
 	bool IsComplete() const;
 
@@ -79,23 +93,30 @@ public:
 	/**
 	 * Queues attaching an analyzer.  Only one analyzer per type can be attached
 	 * at a time unless the arguments differ.
+	 * @param args an \c AnalyzerArgs value representing a file analyzer.
 	 * @return false if analyzer can't be instantiated, else true.
 	 */
 	bool AddAnalyzer(RecordVal* args);
 
 	/**
 	 * Queues removal of an analyzer.
+	 * @param args an \c AnalyzerArgs value representing a file analyzer.
 	 * @return true if analyzer was active at time of call, else false.
 	 */
 	bool RemoveAnalyzer(const RecordVal* args);
 
 	/**
 	 * Pass in non-sequential data and deliver to attached analyzers.
+	 * @param data pointer to start of a chunk of file data.
+	 * @param len number of bytes in the data chunk.
+	 * @param offset number of bytes from start of file at which chunk occurs.
 	 */
 	void DataIn(const u_char* data, uint64 len, uint64 offset);
 
 	/**
 	 * Pass in sequential data and deliver to attached analyzers.
+	 * @param data pointer to start of a chunk of file data.
+	 * @param len number of bytes in the data chunk.
 	 */
 	void DataIn(const u_char* data, uint64 len);
 
@@ -106,10 +127,13 @@ public:
 
 	/**
 	 * Inform attached analyzers about a gap in file stream.
+	 * @param offset number of bytes in to file at which missing chunk starts.
+	 * @param len length in bytes of the missing chunk of file data.
 	 */
 	void Gap(uint64 offset, uint64 len);
 
 	/**
+	 * @param h pointer to an event handler.
 	 * @return true if event has a handler and the file isn't ignored.
 	 */
 	bool FileEventAvailable(EventHandlerPtr h);
@@ -117,11 +141,14 @@ public:
 	/**
 	 * Raises an event related to the file's life-cycle, the only parameter
 	 * to that event is the \c fa_file record..
+	 * @param h pointer to an event handler.
 	 */
 	void FileEvent(EventHandlerPtr h);
 
 	/**
 	 * Raises an event related to the file's life-cycle.
+	 * @param h pointer to an event handler.
+	 * @param vl list of argument values to pass to event call.
 	 */
 	void FileEvent(EventHandlerPtr h, val_list* vl);
 
@@ -130,35 +157,51 @@ protected:
 
 	/**
 	 * Constructor; only file_analysis::Manager should be creating these.
+	 * @param file_id an identifier string for the file in pretty hash form
+	 *        (similar to connection uids).
+	 * @param conn a network connection over which the file is transferred.
+	 * @param tag the network protocol over which the file is transferred.
+	 * @param is_orig true if the file is being transferred from the originator
+	 *        of the connection to the responder.  False indicates the other
+	 *        direction.
 	 */
-	File(const string& unique, Connection* conn = 0,
+	File(const string& file_id, Connection* conn = 0,
 	     AnalyzerTag::Tag tag = AnalyzerTag::Error, bool is_orig = false);
 
 	/**
 	 * Updates the "conn_ids" and "conn_uids" fields in #val record with the
 	 * \c conn_id and UID taken from \a conn.
+	 * @param conn the connection over which a part of the file has been seen.
 	 */
 	void UpdateConnectionFields(Connection* conn);
 
 	/**
 	 * Increment a byte count field of #val record by \a size.
+	 * @param size number of bytes by which to increment.
+	 * @param field_idx the index of the field in \c fa_file to increment.
 	 */
 	void IncrementByteCount(uint64 size, int field_idx);
 
 	/**
 	 * Wrapper to RecordVal::LookupWithDefault for the field in #val at index
 	 * \a idx which automatically unrefs the Val and returns a converted value.
+	 * @param idx the index of a field of type "count" in \c fa_file.
+	 * @return the value of the field, which may be it &default.
 	 */
 	uint64 LookupFieldDefaultCount(int idx) const;
 
 	/**
 	 * Wrapper to RecordVal::LookupWithDefault for the field in #val at index
 	 * \a idx which automatically unrefs the Val and returns a converted value.
+	 * @param idx the index of a field of type "interval" in \c fa_file.
+	 * @return the value of the field, which may be it &default.
 	 */
 	double LookupFieldDefaultInterval(int idx) const;
 
 	/**
 	 * Buffers incoming data at the beginning of a file.
+	 * @param data pointer to a data chunk to buffer.
+	 * @param len number of bytes in the data chunk.
 	 * @return true if buffering is still required, else false
 	 */
 	bool BufferBOF(const u_char* data, uint64 len);
@@ -171,11 +214,15 @@ protected:
 	/**
 	 * Does mime type detection and assigns type (if available) to \c mime_type
 	 * field in #val.
+	 * @param data pointer to a chunk of file data.
+	 * @param len number of bytes in the data chunk.
 	 * @return whether mime type was available.
 	 */
 	bool DetectMIME(const u_char* data, uint64 len);
 
 	/**
+	 * Lookup a record field index/offset by name.
+	 * @param field_name the name of the \c fa_file record field.
 	 * @return the field offset in #val record corresponding to \a field_name.
 	 */
 	static int Idx(const string& field_name);
@@ -186,15 +233,14 @@ protected:
 	static void StaticInit();
 
 private:
-	FileID id;                 /**< A pretty hash that likely identifies file */
+	string id;                 /**< A pretty hash that likely identifies file */
-	string unique;             /**< A string that uniquely identifies file */
 	RecordVal* val;            /**< \c fa_file from script layer. */
 	bool postpone_timeout;     /**< Whether postponing timeout is requested. */
 	bool first_chunk;          /**< Track first non-linear chunk. */
 	bool missed_bof;           /**< Flags that we missed start of file. */
 	bool need_reassembly;      /**< Whether file stream reassembly is needed. */
 	bool done;                 /**< If this object is about to be deleted. */
-	AnalyzerSet analyzers;
+	AnalyzerSet analyzers;     /**< A set of attached file analyzer. */
 
 	struct BOF_Buffer {
 		BOF_Buffer() : full(false), replayed(false), size(0) {}
@@ -207,8 +253,6 @@ private:
 		BroString::CVec chunks;
 	} bof_buffer;              /**< Beginning of file buffer. */
 
-	static string salt;
-
 	static int id_idx;
 	static int parent_id_idx;
 	static int source_idx;

src/file_analysis/FileID.h

@@ -1,34 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-
-#ifndef FILE_ANALYSIS_FILEID_H
-#define FILE_ANALYSIS_FILEID_H
-
-namespace file_analysis {
-
-/**
- * A simple string wrapper class to help enforce some type safety between
- * methods of FileAnalysis::Manager, some of which use a unique string to
- * identify files, and others which use a pretty hash (the FileID) to identify
- * files.  A FileID is primarily used in methods which interface with the
- * script-layer, while the unique strings are used for methods which interface
- * with protocol analyzers or anything that sends data to the file analysis
- * framework.
- */
-struct FileID {
-	string id;
-
-	explicit FileID(const string arg_id) : id(arg_id) {}
-	FileID(const FileID& other) : id(other.id) {}
-
-	const char* c_str() const { return id.c_str(); }
-
-	bool operator==(const FileID& rhs) const { return id == rhs.id; }
-	bool operator<(const FileID& rhs) const { return id < rhs.id; }
-
-	FileID& operator=(const FileID& rhs) { id = rhs.id; return *this; }
-	FileID& operator=(const string& rhs) { id = rhs; return *this; }
-};
-
-} // namespace file_analysis
-
-#endif

src/file_analysis/FileTimer.cc

@@ -5,7 +5,7 @@
 
 using namespace file_analysis;
 
-FileTimer::FileTimer(double t, const FileID& id, double interval)
+FileTimer::FileTimer(double t, const string& id, double interval)
     : Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id)
 	{
 	DBG_LOG(DBG_FILE_ANALYSIS, "New %f second timeout timer for %s",

src/file_analysis/FileTimer.h

@@ -5,7 +5,6 @@
 
 #include <string>
 #include "Timer.h"
-#include "FileID.h"
 
 namespace file_analysis {
 
@@ -14,16 +13,25 @@ namespace file_analysis {
  */
 class FileTimer : public Timer {
 public:
-	FileTimer(double t, const FileID& id, double interval);
+
+	/**
+	 * Constructor, nothing interesting about it.
+	 * @param t unix time at which the timer should start ticking.
+	 * @param id the file identifier which will be checked for inactivity.
+	 * @param interval amount of time after \a t to check for inactivity.
+	 */
+	FileTimer(double t, const string& id, double interval);
 
 	/**
 	 * Check inactivity of file_analysis::File corresponding to #file_id,
 	 * reschedule if active, else call file_analysis::Manager::Timeout.
+	 * @param t current unix time
+	 * @param is_expire true if all pending timers are being expired.
 	 */
 	void Dispatch(double t, int is_expire);
 
 private:
-	FileID file_id;
+	string file_id;
 };
 
 } // namespace file_analysis

src/file_analysis/Hash.h

@@ -17,17 +17,50 @@ namespace file_analysis {
  */
 class Hash : public file_analysis::Analyzer {
 public:
+
+	/**
+	 * Destructor.
+	 */
 	virtual ~Hash();
 
+	/**
+	 * Incrementally hash next chunk of file contents.
+	 * @param data pointer to start of a chunk of a file data.
+	 * @param len number of bytes in the data chunk.
+	 * @return false if the digest is in an invalid state, else true.
+	 */
 	virtual bool DeliverStream(const u_char* data, uint64 len);
 
+	/**
+	 * Finalizes the hash and raises a "file_hash" event.
+	 * @return always false so analyze will be deteched from file.
+	 */
 	virtual bool EndOfFile();
 
+	/**
+	 * Missing data can't be handled, so just indicate the this analyzer should
+	 * be removed from receiving further data.  The hash will not be finalized.
+	 * @param offset byte offset in file at which missing chunk starts.
+	 * @param len number of missing bytes.
+	 * @return always false so analyzer will detach from file.
+	 */
 	virtual bool Undelivered(uint64 offset, uint64 len);
 
 protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @param hv specific hash calculator object.
+	 * @param kind human readable name of the hash algorithm to use.
+	 */
 	Hash(RecordVal* args, File* file, HashVal* hv, const char* kind);
 
+	/**
+	 * If some file contents have been seen, finalizes the hash of them and
+	 * raises the "file_hash" event with the results.
+	 */
 	void Finalize();
 
 private:
@@ -36,34 +69,85 @@ private:
 	const char* kind;
 };
 
+/**
+ * An analyzer to produce an MD5 hash of file contents.
+ */
 class MD5 : public Hash {
 public:
+
+	/**
+	 * Create a new instance of the MD5 hashing file analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new MD5 analyzer instance or a null pointer if there's no
+	 *         handler for the "file_hash" event.
+	 */
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
 		{ return file_hash ? new MD5(args, file) : 0; }
 
 protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 */
 	MD5(RecordVal* args, File* file)
 		: Hash(args, file, new MD5Val(), "md5")
 		{}
 };
 
+/**
+ * An analyzer to produce a SHA1 hash of file contents.
+ */
 class SHA1 : public Hash {
 public:
+
+	/**
+	 * Create a new instance of the SHA1 hashing file analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new MD5 analyzer instance or a null pointer if there's no
+	 *         handler for the "file_hash" event.
+	 */
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
 		{ return file_hash ? new SHA1(args, file) : 0; }
 
 protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 */
 	SHA1(RecordVal* args, File* file)
 		: Hash(args, file, new SHA1Val(), "sha1")
 		{}
 };
 
+/**
+ * An analyzer to produce a SHA256 hash of file contents.
+ */
 class SHA256 : public Hash {
 public:
+
+	/**
+	 * Create a new instance of the SHA256 hashing file analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new MD5 analyzer instance or a null pointer if there's no
+	 *         handler for the "file_hash" event.
+	 */
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
 		{ return file_hash ? new SHA256(args, file) : 0; }
 
 protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 */
 	SHA256(RecordVal* args, File* file)
 		: Hash(args, file, new SHA256Val(), "sha256")
 		{}

src/file_analysis/Manager.cc

@@ -2,6 +2,7 @@
 
 #include <vector>
 #include <string>
+#include <openssl/md5.h>
 
 #include "Manager.h"
 #include "File.h"
@@ -24,7 +25,7 @@ Manager::~Manager()
 
 void Manager::Terminate()
 	{
-	vector<FileID> keys;
+	vector<string> keys;
 	for ( IDMap::iterator it = id_map.begin(); it != id_map.end(); ++it )
 		keys.push_back(it->first);
 
@@ -32,66 +33,79 @@ void Manager::Terminate()
 		Timeout(keys[i], true);
 	}
 
+string Manager::HashHandle(const string& handle) const
+	{
+	static string salt;
+
+	if ( salt.empty() )
+		salt = BifConst::FileAnalysis::salt->CheckString();
+
+	char tmp[20];
+	uint64 hash[2];
+	string msg(handle + salt);
+
+	MD5(reinterpret_cast<const u_char*>(msg.data()), msg.size(),
+	    reinterpret_cast<u_char*>(hash));
+	uitoa_n(hash[0], tmp, sizeof(tmp), 62);
+
+	return tmp;
+	}
+
 void Manager::SetHandle(const string& handle)
 	{
-	current_handle = handle;
+	if ( handle.empty() )
+		return;
+
+	current_file_id = HashHandle(handle);
 	}
 
 void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
                      AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
 	{
-	if ( IsDisabled(tag) )
-		return;
-
 	GetFileHandle(tag, conn, is_orig);
-	DataIn(data, len, offset, GetFile(current_handle, conn, tag, is_orig));
+	File* file = GetFile(current_file_id, conn, tag, is_orig);
-	}
 
-void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
-                     const string& unique)
-	{
-	DataIn(data, len, offset, GetFile(unique));
-	}
-
-void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
-                     File* file)
-	{
 	if ( ! file )
 		return;
 
 	file->DataIn(data, len, offset);
 
 	if ( file->IsComplete() )
-		RemoveFile(file->GetUnique());
+		RemoveFile(file->GetID());
 	}
 
 void Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
                      Connection* conn, bool is_orig)
 	{
-	if ( IsDisabled(tag) )
-		return;
-
 	GetFileHandle(tag, conn, is_orig);
-
 	// Sequential data input shouldn't be going over multiple conns, so don't
 	// do the check to update connection set.
-	DataIn(data, len, GetFile(current_handle, conn, tag, is_orig, false));
+	File* file = GetFile(current_file_id, conn, tag, is_orig, false);
-	}
 
-void Manager::DataIn(const u_char* data, uint64 len, const string& unique)
-	{
-	DataIn(data, len, GetFile(unique));
-	}
-
-void Manager::DataIn(const u_char* data, uint64 len, File* file)
-	{
 	if ( ! file )
 		return;
 
 	file->DataIn(data, len);
 
 	if ( file->IsComplete() )
-		RemoveFile(file->GetUnique());
+		RemoveFile(file->GetID());
+	}
+
+void Manager::DataIn(const u_char* data, uint64 len, const string& file_id,
+                     const string& source)
+	{
+	File* file = GetFile(file_id);
+
+	if ( ! file )
+		return;
+
+	if ( file->GetSource().empty() )
+		file->SetSource(source);
+
+	file->DataIn(data, len);
+
+	if ( file->IsComplete() )
+		RemoveFile(file->GetID());
 	}
 
 void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn)
@@ -102,35 +116,22 @@ void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn)
 
 void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
 	{
-	if ( IsDisabled(tag) )
+	// Don't need to create a file if we're just going to remove it right away.
-		return;
-
 	GetFileHandle(tag, conn, is_orig);
-	EndOfFile(current_handle);
+	RemoveFile(current_file_id);
 	}
 
-void Manager::EndOfFile(const string& unique)
+void Manager::EndOfFile(const string& file_id)
 	{
-	RemoveFile(unique);
+	RemoveFile(file_id);
 	}
 
 void Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag,
                   Connection* conn, bool is_orig)
 	{
-	if ( IsDisabled(tag) )
-		return;
-
 	GetFileHandle(tag, conn, is_orig);
-	Gap(offset, len, GetFile(current_handle, conn, tag, is_orig));
+	File* file = GetFile(current_file_id, conn, tag, is_orig);
-	}
 
-void Manager::Gap(uint64 offset, uint64 len, const string& unique)
-	{
-	Gap(offset, len, GetFile(unique));
-	}
-
-void Manager::Gap(uint64 offset, uint64 len, File* file)
-	{
 	if ( ! file )
 		return;
 
@@ -140,52 +141,33 @@ void Manager::Gap(uint64 offset, uint64 len, File* file)
 void Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
                       bool is_orig)
 	{
-	if ( IsDisabled(tag) )
-		return;
-
 	GetFileHandle(tag, conn, is_orig);
-	SetSize(size, GetFile(current_handle, conn, tag, is_orig));
+	File* file = GetFile(current_file_id, conn, tag, is_orig);
-	}
 
-void Manager::SetSize(uint64 size, const string& unique)
-	{
-	SetSize(size, GetFile(unique));
-	}
-
-void Manager::SetSize(uint64 size, File* file)
-	{
 	if ( ! file )
 		return;
 
 	file->SetTotalBytes(size);
 
 	if ( file->IsComplete() )
-		RemoveFile(file->GetUnique());
+		RemoveFile(file->GetID());
 	}
 
-bool Manager::PostponeTimeout(const FileID& file_id) const
+bool Manager::SetTimeoutInterval(const string& file_id, double interval) const
 	{
 	File* file = Lookup(file_id);
 
 	if ( ! file )
 		return false;
 
+	if ( interval > 0 )
 		file->postpone_timeout = true;
-	return true;
-	}
-
-bool Manager::SetTimeoutInterval(const FileID& file_id, double interval) const
-	{
-	File* file = Lookup(file_id);
-
-	if ( ! file )
-		return false;
 
 	file->SetTimeoutInterval(interval);
 	return true;
 	}
 
-bool Manager::AddAnalyzer(const FileID& file_id, RecordVal* args) const
+bool Manager::AddAnalyzer(const string& file_id, RecordVal* args) const
 	{
 	File* file = Lookup(file_id);
 
@@ -195,7 +177,7 @@ bool Manager::AddAnalyzer(const FileID& file_id, RecordVal* args) const
 	return file->AddAnalyzer(args);
 	}
 
-bool Manager::RemoveAnalyzer(const FileID& file_id, const RecordVal* args) const
+bool Manager::RemoveAnalyzer(const string& file_id, const RecordVal* args) const
 	{
 	File* file = Lookup(file_id);
 
@@ -205,32 +187,23 @@ bool Manager::RemoveAnalyzer(const FileID& file_id, const RecordVal* args) const
 	return file->RemoveAnalyzer(args);
 	}
 
-File* Manager::GetFile(const string& unique, Connection* conn,
+File* Manager::GetFile(const string& file_id, Connection* conn,
                        AnalyzerTag::Tag tag, bool is_orig, bool update_conn)
 	{
-	if ( unique.empty() )
+	if ( file_id.empty() )
 		return 0;
 
-	if ( IsIgnored(unique) )
+	if ( IsIgnored(file_id) )
 		return 0;
 
-	File* rval = str_map[unique];
+	File* rval = id_map[file_id];
 
 	if ( ! rval )
 		{
-		rval = str_map[unique] = new File(unique, conn, tag, is_orig);
+		rval = id_map[file_id] = new File(file_id, conn, tag, is_orig);
-		FileID id = rval->GetID();
-
-		if ( id_map[id] )
-			{
-			reporter->Error("Evicted duplicate file ID: %s", id.c_str());
-			RemoveFile(unique);
-			}
-
-		id_map[id] = rval;
 		rval->ScheduleInactivityTimer();
 
-		if ( IsIgnored(unique) )
+		if ( IsIgnored(file_id) )
 			return 0;
 		}
 	else
@@ -244,7 +217,7 @@ File* Manager::GetFile(const string& unique, Connection* conn,
 	return rval;
 	}
 
-File* Manager::Lookup(const FileID& file_id) const
+File* Manager::Lookup(const string& file_id) const
 	{
 	IDMap::const_iterator it = id_map.find(file_id);
 
@@ -254,7 +227,7 @@ File* Manager::Lookup(const FileID& file_id) const
 	return it->second;
 	}
 
-void Manager::Timeout(const FileID& file_id, bool is_terminating)
+void Manager::Timeout(const string& file_id, bool is_terminating)
 	{
 	File* file = Lookup(file_id);
 
@@ -277,53 +250,50 @@ void Manager::Timeout(const FileID& file_id, bool is_terminating)
 	DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s",
 	        file->GetID().c_str());
 
-	RemoveFile(file->GetUnique());
+	RemoveFile(file->GetID());
 	}
 
-bool Manager::IgnoreFile(const FileID& file_id)
+bool Manager::IgnoreFile(const string& file_id)
+	{
+	if ( id_map.find(file_id) == id_map.end() )
+		return false;
+
+	DBG_LOG(DBG_FILE_ANALYSIS, "Ignore FileID %s", file_id.c_str());
+
+	ignored.insert(file_id);
+
+	return true;
+	}
+
+bool Manager::RemoveFile(const string& file_id)
 	{
 	IDMap::iterator it = id_map.find(file_id);
 
 	if ( it == id_map.end() )
 		return false;
 
-	DBG_LOG(DBG_FILE_ANALYSIS, "Ignore FileID %s", file_id.c_str());
+	DBG_LOG(DBG_FILE_ANALYSIS, "Remove FileID %s", file_id.c_str());
-
-	ignored.insert(it->second->GetUnique());
-
-	return true;
-	}
-
-bool Manager::RemoveFile(const string& unique)
-	{
-	StrMap::iterator it = str_map.find(unique);
-
-	if ( it == str_map.end() )
-		return false;
 
 	it->second->EndOfFile();
 
-	FileID id = it->second->GetID();
-
-	DBG_LOG(DBG_FILE_ANALYSIS, "Remove FileID %s", id.c_str());
-
-	if ( ! id_map.erase(id) )
-		reporter->Error("No mapping for fileID %s", id.c_str());
-
-	ignored.erase(unique);
 	delete it->second;
-	str_map.erase(unique);
+	id_map.erase(file_id);
+	ignored.erase(file_id);
+
 	return true;
 	}
 
-bool Manager::IsIgnored(const string& unique)
+bool Manager::IsIgnored(const string& file_id)
 	{
-	return ignored.find(unique) != ignored.end();
+	return ignored.find(file_id) != ignored.end();
 	}
 
 void Manager::GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig)
 	{
-	current_handle.clear();
+	current_file_id.clear();
+
+	if ( IsDisabled(tag) )
+		return;
 
 	if ( ! get_file_handle )
 		return;

src/file_analysis/Manager.h

@@ -18,7 +18,6 @@
 
 #include "File.h"
 #include "FileTimer.h"
-#include "FileID.h"
 
 namespace file_analysis {
 
@@ -27,7 +26,15 @@ namespace file_analysis {
  */
 class Manager {
 public:
+
+	/**
+	 * Constructor.
+	 */
 	Manager();
+
+	/**
+	 * Destructor.  Times out any currently active file analyses.
+	 */
 	~Manager();
 
 	/**
@@ -36,141 +43,220 @@ public:
 	void Terminate();
 
 	/**
-	 * Take in a unique file handle string to identifiy incoming file data.
+	 * Creates a file identifier from a unique file handle string.
+	 * @param handle a unique string which identifies a single file.
+	 * @return a prettified MD5 hash of \a handle, truncated to 64-bits.
+	 */
+	string HashHandle(const string& handle) const;
+
+	/**
+	 * Take in a unique file handle string to identify next piece of
+	 * incoming file data/information.
+	 * @param handle a unique string which identifies a single file.
 	 */
 	void SetHandle(const string& handle);
 
 	/**
 	 * Pass in non-sequential file data.
+	 * @param data pointer to start of a chunk of file data.
+	 * @param len number of bytes in the data chunk.
+	 * @param offset number of bytes from start of file that data chunk occurs.
+	 * @param tag network protocol over which the file data is transferred.
+	 * @param conn network connection over which the file data is transferred.
+	 * @param is_orig true if the file is being sent from connection originator
+	 *        or false if is being sent in the opposite direction.
 	 */
 	void DataIn(const u_char* data, uint64 len, uint64 offset,
 		    AnalyzerTag::Tag tag, Connection* conn, bool is_orig);
-	void DataIn(const u_char* data, uint64 len, uint64 offset,
-		    const string& unique);
-	void DataIn(const u_char* data, uint64 len, uint64 offset,
-		    File* file);
 
 	/**
 	 * Pass in sequential file data.
+	 * @param data pointer to start of a chunk of file data.
+	 * @param len number of bytes in the data chunk.
+	 * @param tag network protocol over which the file data is transferred.
+	 * @param conn network connection over which the file data is transferred.
+	 * @param is_orig true if the file is being sent from connection originator
+	 *        or false if is being sent in the opposite direction.
 	 */
 	void DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
 	            Connection* conn, bool is_orig);
-	void DataIn(const u_char* data, uint64 len, const string& unique);
-	void DataIn(const u_char* data, uint64 len, File* file);
 
 	/**
-	 * Signal the end of file data.
+	 * Pass in sequential file data from external source (e.g. input framework).
+	 * @param data pointer to start of a chunk of file data.
+	 * @param len number of bytes in the data chunk.
+	 * @param file_id an identifier for the file (usually a hash of \a source).
+	 * @param source uniquely identifies the file and should also describe
+	 *        in human-readable form where the file input is coming from (e.g.
+	 *        a local file path).
+	 */
+	void DataIn(const u_char* data, uint64 len, const string& file_id,
+	            const string& source);
+
+	/**
+	 * Signal the end of file data regardless of which direction it is being
+	 * sent over the connection.
+	 * @param tag network protocol over which the file data is transferred.
+	 * @param conn network connection over which the file data is transferred.
 	 */
 	void EndOfFile(AnalyzerTag::Tag tag, Connection* conn);
+
+	/**
+	 * Signal the end of file data being transferred over a connection in
+	 * a particular direction.
+	 * @param tag network protocol over which the file data is transferred.
+	 * @param conn network connection over which the file data is transferred.
+	 */
 	void EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig);
-	void EndOfFile(const string& unique);
+
+	/**
+	 * Signal the end of file data being transferred using the file identifier.
+	 * @param file_id the file identifier/hash.
+	 */
+	void EndOfFile(const string& file_id);
 
 	/**
 	 * Signal a gap in the file data stream.
+	 * @param offset number of bytes in to file at which missing chunk starts.
+	 * @param len length in bytes of the missing chunk of file data.
+	 * @param tag network protocol over which the file data is transferred.
+	 * @param conn network connection over which the file data is transferred.
+	 * @param is_orig true if the file is being sent from connection originator
+	 *        or false if is being sent in the opposite direction.
 	 */
 	void Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, Connection* conn,
 	         bool is_orig);
-	void Gap(uint64 offset, uint64 len, const string& unique);
-	void Gap(uint64 offset, uint64 len, File* file);
 
 	/**
 	 * Provide the expected number of bytes that comprise a file.
+	 * @param size the number of bytes in the full file.
+	 * @param tag network protocol over which the file data is transferred.
+	 * @param conn network connection over which the file data is transferred.
+	 * @param is_orig true if the file is being sent from connection originator
+	 *        or false if is being sent in the opposite direction.
 	 */
 	void SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
 	             bool is_orig);
-	void SetSize(uint64 size, const string& unique);
-	void SetSize(uint64 size, File* file);
 
 	/**
 	 * Starts ignoring a file, which will finally be removed from internal
 	 * mappings on EOF or TIMEOUT.
+	 * @param file_id the file identifier/hash.
 	 * @return false if file identifier did not map to anything, else true.
 	 */
-	bool IgnoreFile(const FileID& file_id);
+	bool IgnoreFile(const string& file_id);
-
-	/**
-	 * If called during a \c file_timeout event handler, requests deferral of
-	 * analysis timeout.
-	 */
-	bool PostponeTimeout(const FileID& file_id) const;
 
 	/**
 	 * Set's an inactivity threshold for the file.
+	 * @param file_id the file identifier/hash.
+	 * @param interval the amount of time in which no activity is seen for
+	 *        the file identified by \a file_id that will cause the file
+	 *        to be considered stale, timed out, and then resource reclaimed.
+	 * @return false if file identifier did not map to anything, else true.
 	 */
-	bool SetTimeoutInterval(const FileID& file_id, double interval) const;
+	bool SetTimeoutInterval(const string& file_id, double interval) const;
 
 	/**
 	 * Queue attachment of an analzer to the file identifier.  Multiple
 	 * analyzers of a given type can be attached per file identifier at a time
 	 * as long as the arguments differ.
+	 * @param file_id the file identifier/hash.
+	 * @param args a \c AnalyzerArgs value which describes a file analyzer.
 	 * @return false if the analyzer failed to be instantiated, else true.
 	 */
-	bool AddAnalyzer(const FileID& file_id, RecordVal* args) const;
+	bool AddAnalyzer(const string& file_id, RecordVal* args) const;
 
 	/**
 	 * Queue removal of an analyzer for a given file identifier.
+	 * @param file_id the file identifier/hash.
+	 * @param args a \c AnalyzerArgs value which describes a file analyzer.
 	 * @return true if the analyzer is active at the time of call, else false.
 	 */
-	bool RemoveAnalyzer(const FileID& file_id, const RecordVal* args) const;
+	bool RemoveAnalyzer(const string& file_id, const RecordVal* args) const;
 
 	/**
-	 * @return whether the file mapped to \a unique is being ignored.
+	 * Tells whether analysis for a file is active or ignored.
+	 * @param file_id the file identifier/hash.
+	 * @return whether the file mapped to \a file_id is being ignored.
 	 */
-	bool IsIgnored(const string& unique);
+	bool IsIgnored(const string& file_id);
 
 protected:
 	friend class FileTimer;
 
-	typedef map<string, File*> StrMap;
+	typedef set<string> IDSet;
-	typedef set<string> StrSet;
+	typedef map<string, File*> IDMap;
-	typedef map<FileID, File*> IDMap;
 
 	/**
-	 * @return the File object mapped to \a unique or a null pointer if analysis
+	 * Create a new file to be analyzed or retrieve an existing one.
-	 *         is being ignored for the associated file.  An File object may be
+	 * @param file_id the file identifier/hash.
-	 *         created if a mapping doesn't exist, and if it did exist, the
+	 * @param conn network connection, if any, over which the file is
-	 *         activity time is refreshed along with any connection-related
+	 *        transferred.
-	 *         fields.
+	 * @param tag network protocol, if any, over which the file is transferred.
+	 * @param is_orig true if the file is being sent from connection originator
+	 *        or false if is being sent in the opposite direction (or if it
+	 *        this file isn't related to a connection).
+	 * @param update_conn whether we need to update connection-related field
+	 *        in the \c fa_file record value associated with the file.
+	 * @return the File object mapped to \a file_id or a null pointer if
+	 *         analysis is being ignored for the associated file.  An File
+	 *         object may be created if a mapping doesn't exist, and if it did
+	 *         exist, the activity time is refreshed along with any
+	 *         connection-related fields.
 	 */
-	File* GetFile(const string& unique, Connection* conn = 0,
+	File* GetFile(const string& file_id, Connection* conn = 0,
 	              AnalyzerTag::Tag tag = AnalyzerTag::Error,
 	              bool is_orig = false, bool update_conn = true);
 
 	/**
+	 * Try to retrieve a file that's being analyzed, using its identifier/hash.
+	 * @param file_id the file identifier/hash.
 	 * @return the File object mapped to \a file_id, or a null pointer if no
 	 *         mapping exists.
 	 */
-	File* Lookup(const FileID& file_id) const;
+	File* Lookup(const string& file_id) const;
 
 	/**
 	 * Evaluate timeout policy for a file and remove the File object mapped to
 	 * \a file_id if needed.
+	 * @param file_id the file identifier/hash.
+	 * @param is_termination whether the Manager (and probably Bro) is in a
+	 *        terminating state.  If true, then the timeout cannot be postponed.
 	 */
-	void Timeout(const FileID& file_id, bool is_terminating = ::terminating);
+	void Timeout(const string& file_id, bool is_terminating = ::terminating);
 
 	/**
-	 * Immediately remove file_analysis::File object associated with \a unique.
+	 * Immediately remove file_analysis::File object associated with \a file_id.
-	 * @return false if file string did not map to anything, else true.
+	 * @param file_id the file identifier/hash.
+	 * @return false if file id string did not map to anything, else true.
 	 */
-	bool RemoveFile(const string& unique);
+	bool RemoveFile(const string& file_id);
 
 	/**
-	 * Sets #current_handle to a unique file handle string based on what the
+	 * Sets #current_file_id to a hash of a unique file handle string based on
-	 * \c get_file_handle event derives from the connection params.  The
+	 * what the \c get_file_handle event derives from the connection params.
-	 * event queue is flushed so that we can get the handle value immediately.
+	 * Event queue is flushed so that we can get the handle value immediately.
+	 * @param tag network protocol over which the file is transferred.
+	 * @param conn network connection over which the file is transferred.
+	 * @param is_orig true if the file is being sent from connection originator
+	 *        or false if is being sent in the opposite direction.
 	 */
 	void GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig);
 
 	/**
-	 * @return whether file analysis is disabled for the given analyzer.
+	 * Check if analysis is available for files transferred over a given
+	 * network protocol.
+	 * @param tag the network protocol over which files can be transferred and
+	 *        analyzed by the file analysis framework.
+	 * @return whether file analysis is disabled for the analyzer given by
+	 *         \a tag.
 	 */
 	static bool IsDisabled(AnalyzerTag::Tag tag);
 
 private:
-	StrMap str_map;	/**< Map unique string to file_analysis::File. */
 	IDMap id_map;	/**< Map file ID to file_analysis::File records. */
-	StrSet ignored;	/**< Ignored files.  Will be finally removed on EOF. */
+	IDSet ignored;	/**< Ignored files.  Will be finally removed on EOF. */
-	string current_handle;	/**< Last file handle set by get_file_handle event.*/
+	string current_file_id;	/**< Hash of what get_file_handle event sets.*/
 
 	static TableVal* disabled;	/**< Table of disabled analyzers. */
 };

src/input.bif

@@ -9,6 +9,7 @@ module Input;
 
 type TableDescription: record;
 type EventDescription: record;
+type AnalysisDescription: record;
 
 function Input::__create_table_stream%(description: Input::TableDescription%) : bool
 	%{
@@ -22,6 +23,12 @@ function Input::__create_event_stream%(description: Input::EventDescription%) :
 	return new Val(res, TYPE_BOOL);
 	%}
 
+function Input::__create_analysis_stream%(description: Input::AnalysisDescription%) : bool
+	%{
+	bool res = input_mgr->CreateAnalysisStream(description->AsRecordVal());
+	return new Val(res, TYPE_BOOL);
+	%}
+
 function Input::__remove_stream%(id: string%) : bool
 	%{
 	bool res = input_mgr->RemoveStream(id->AsString()->CheckString());

src/input/Manager.cc

@@ -15,6 +15,7 @@
 #include "EventHandler.h"
 #include "NetVar.h"
 #include "Net.h"
+#include "../file_analysis/Manager.h"
 
 
 #include "CompHash.h"
@@ -148,6 +149,14 @@ public:
         ~EventStream();
 };
 
+class Manager::AnalysisStream: public Manager::Stream {
+public:
+	string file_id;
+
+	AnalysisStream();
+	~AnalysisStream();
+};
+
 Manager::TableStream::TableStream() : Manager::Stream::Stream()
 	{
 	stream_type = TABLE_STREAM;
@@ -198,6 +207,15 @@ Manager::TableStream::~TableStream()
 		}
 	}
 
+Manager::AnalysisStream::AnalysisStream() : Manager::Stream::Stream()
+	{
+	stream_type = ANALYSIS_STREAM;
+	}
+
+Manager::AnalysisStream::~AnalysisStream()
+	{
+	}
+
 Manager::Manager()
 	{
 	end_of_data = internal_handler("Input::end_of_data");
@@ -274,7 +292,8 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 
 	RecordType* rtype = description->Type()->AsRecordType();
 	if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0)
-		|| same_type(rtype, BifType::Record::Input::EventDescription, 0) ) )
+		|| same_type(rtype, BifType::Record::Input::EventDescription, 0)
+		|| same_type(rtype, BifType::Record::Input::AnalysisDescription, 0) ) )
 		{
 		reporter->Error("Streamdescription argument not of right type for new input stream");
 		return false;
@@ -680,6 +699,40 @@ bool Manager::CreateTableStream(RecordVal* fval)
 	return true;
 	}
 
+bool Manager::CreateAnalysisStream(RecordVal* fval)
+	{
+	RecordType* rtype = fval->Type()->AsRecordType();
+	if ( ! same_type(rtype, BifType::Record::Input::AnalysisDescription, 0) )
+		{
+		reporter->Error("AnalysisDescription argument not of right type");
+		return false;
+		}
+
+	AnalysisStream* stream = new AnalysisStream();
+		{
+		if ( ! CreateStream(stream, fval) )
+			{
+			delete stream;
+			return false;
+			}
+		}
+
+	stream->file_id = file_mgr->HashHandle(stream->name);
+
+	assert(stream->reader);
+
+	// reader takes in a byte stream as the only field
+	Field** fields = new Field*[1];
+	fields[0] = new Field("bytestream", 0, TYPE_STRING, TYPE_VOID, false);
+	stream->reader->Init(1, fields);
+
+	readers[stream->reader] = stream;
+
+	DBG_LOG(DBG_INPUT, "Successfully created analysis stream %s",
+		stream->name.c_str());
+
+	return true;
+	}
 
 bool Manager::IsCompatibleType(BroType* t, bool atomic_only)
 	{
@@ -966,6 +1019,15 @@ void Manager::SendEntry(ReaderFrontend* reader, Value* *vals)
 		readFields = SendEventStreamEvent(i, type, vals);
 		}
 
+	else if ( i->stream_type == ANALYSIS_STREAM )
+		{
+		readFields = 1;
+		assert(vals[0]->type == TYPE_STRING);
+		file_mgr->DataIn(reinterpret_cast<u_char*>(vals[0]->val.string_val.data),
+		                 vals[0]->val.string_val.length,
+		                 static_cast<AnalysisStream*>(i)->file_id, i->name);
+		}
+
 	else
 		assert(false);
 
@@ -1179,7 +1241,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader)
 	DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", i->name.c_str());
 #endif
 
-	if ( i->stream_type == EVENT_STREAM )
+	if ( i->stream_type != TABLE_STREAM )
 		{
 		// just signal the end of the data source
 		SendEndOfData(i);
@@ -1288,6 +1350,9 @@ void Manager::SendEndOfData(ReaderFrontend* reader)
 void Manager::SendEndOfData(const Stream *i)
 	{
 	SendEvent(end_of_data, 2, new StringVal(i->name.c_str()), new StringVal(i->info->source));
+
+	if ( i->stream_type == ANALYSIS_STREAM )
+		file_mgr->EndOfFile(static_cast<const AnalysisStream*>(i)->file_id);
 	}
 
 void Manager::Put(ReaderFrontend* reader, Value* *vals)
@@ -1310,6 +1375,15 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals)
 		readFields = SendEventStreamEvent(i, type, vals);
 		}
 
+	else if ( i->stream_type == ANALYSIS_STREAM )
+		{
+		readFields = 1;
+		assert(vals[0]->type == TYPE_STRING);
+		file_mgr->DataIn(reinterpret_cast<u_char*>(vals[0]->val.string_val.data),
+		                 vals[0]->val.string_val.length,
+		                 static_cast<AnalysisStream*>(i)->file_id, i->name);
+		}
+
 	else
 		assert(false);
 
@@ -1577,6 +1651,12 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals)
 		success = true;
 		}
 
+	else if ( i->stream_type == ANALYSIS_STREAM )
+		{
+		// can't do anything
+		success = true;
+		}
+
 	else
 		{
 		assert(false);

src/input/Manager.h

@@ -55,6 +55,18 @@ public:
 	 */
 	bool CreateEventStream(RecordVal* description);
 
+	/**
+	 * Creates a new input stream which will forward the data from the data
+	 * source on to the file analysis framework.  The internal BiF defined
+	 * in input.bif just forward here.  For an input reader to be compatible
+	 * with this method, it must be able to accept a filter of a single string
+	 * type (i.e. they read a byte stream).
+	 *
+	 * @param description A record of the script type \c
+	 * Input::AnalysisDescription
+	 */
+	bool CreateAnalysisStream(RecordVal* description);
+
 	/**
 	 * Force update on a input stream. Forces a re-read of the whole
 	 * input source. Usually used when an input stream is opened in
@@ -138,6 +150,7 @@ private:
 	class Stream;
 	class TableStream;
 	class EventStream;
+	class AnalysisStream;
 
 	// Actual RemoveStream implementation -- the function's public and
 	// protected definitions are wrappers around this function.
@@ -202,7 +215,7 @@ private:
 	Stream* FindStream(const string &name);
 	Stream* FindStream(ReaderFrontend* reader);
 
-	enum StreamType { TABLE_STREAM, EVENT_STREAM };
+	enum StreamType { TABLE_STREAM, EVENT_STREAM, ANALYSIS_STREAM };
 
 	map<ReaderFrontend*, Stream*> readers;
 

testing/btest/Baseline/core.tunnels.ayiya/http.log

@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-14-38-11
+#open	2013-05-21-21-11-20
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1257655301.652206	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	10102	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1257655301.652206	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	10102	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1257655302.514424	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	2	GET	ipv6.google.com	/csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-
+1257655302.514424	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	2	GET	ipv6.google.com	/csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-	-
-1257655303.603569	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	3	GET	ipv6.google.com	/gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-
+1257655303.603569	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	3	GET	ipv6.google.com	/gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-	-
-#close	2013-03-22-14-38-11
+#close	2013-05-21-21-11-20

testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-14-37-45
+#open	2013-05-21-21-11-21
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1333458850.340368	arKYeMETxOg	10.131.17.170	51803	173.199.115.168	80	1	GET	cdn.epicgameads.com	/ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879	http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296	Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)	0	31461	200	OK	-	-	-	(empty)	-	-	-	application/x-shockwave-flash	-	-
+1333458850.340368	arKYeMETxOg	10.131.17.170	51803	173.199.115.168	80	1	GET	cdn.epicgameads.com	/ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879	http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296	Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)	0	31461	200	OK	-	-	-	(empty)	-	-	-	application/x-shockwave-flash	-	-	-
-1333458850.399501	arKYeMETxOg	10.131.17.170	51803	173.199.115.168	80	2	GET	cdn.epicgameads.com	/ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881	http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207	Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)	0	31461	200	OK	-	-	-	(empty)	-	-	-	application/x-shockwave-flash	-	-
+1333458850.399501	arKYeMETxOg	10.131.17.170	51803	173.199.115.168	80	2	GET	cdn.epicgameads.com	/ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881	http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207	Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)	0	31461	200	OK	-	-	-	(empty)	-	-	-	application/x-shockwave-flash	-	-	-
-#close	2013-03-22-14-37-45
+#close	2013-05-21-21-11-21

testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-28-21-35-15
+#open	2013-05-21-21-11-22
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1333458850.375568	arKYeMETxOg	10.131.47.185	1923	79.101.110.141	80	1	GET	o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com	/videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1	http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf	Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11	0	56320	206	Partial Content	-	-	-	(empty)	-	-	-	application/octet-stream	-	-
+1333458850.375568	arKYeMETxOg	10.131.47.185	1923	79.101.110.141	80	1	GET	o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com	/videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1	http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf	Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11	0	56320	206	Partial Content	-	-	-	(empty)	-	-	-	application/octet-stream	-	-	-
-#close	2013-03-28-21-35-15
+#close	2013-05-21-21-11-22

testing/btest/Baseline/core.tunnels.teredo/http.log

@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-14-37-44
+#open	2013-05-21-21-11-21
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1210953057.917183	3PKsZ2Uye21	192.168.2.16	1578	75.126.203.78	80	1	POST	download913.avast.com	/cgi-bin/iavs4stats.cgi	-	Syncer/4.80 (av_pro-1169;f)	589	0	204	<empty>	-	-	-	(empty)	-	-	-	text/plain	-	-
+1210953057.917183	3PKsZ2Uye21	192.168.2.16	1578	75.126.203.78	80	1	POST	download913.avast.com	/cgi-bin/iavs4stats.cgi	-	Syncer/4.80 (av_pro-1169;f)	589	0	204	<empty>	-	-	-	(empty)	-	-	-	text/plain	-	-	-
-1210953061.585996	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1210953061.585996	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1210953073.381474	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1210953073.381474	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1210953074.674817	c4Zw9TmAE05	192.168.2.16	1580	67.228.110.120	80	1	GET	www.wireshark.org	/	http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	11845	200	OK	-	-	-	(empty)	-	-	-	application/xml	-	-
+1210953074.674817	c4Zw9TmAE05	192.168.2.16	1580	67.228.110.120	80	1	GET	www.wireshark.org	/	http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	11845	200	OK	-	-	-	(empty)	-	-	-	application/xml	-	-	-
-#close	2013-03-22-14-37-44
+#close	2013-05-21-21-11-21

testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-14-37-44
+#open	2013-05-21-21-11-22
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1340127577.361683	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1340127577.361683	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1340127577.379360	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1340127577.379360	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-	-
-#close	2013-03-22-14-37-44
+#close	2013-05-21-21-11-22

testing/btest/Baseline/istate.events-ssl/receiver.http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-21-05-55
+#open	2013-05-21-21-11-32
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1363986354.505533	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-
+1369170691.550143	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-	-
-#close	2013-03-22-21-05-56
+#close	2013-05-21-21-11-33

testing/btest/Baseline/istate.events-ssl/sender.http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-04-10-15-49-37
+#open	2013-05-21-21-11-32
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1365608977.146651	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-
+1369170691.550143	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-	-
-#close	2013-04-10-15-49-38
+#close	2013-05-21-21-11-33

testing/btest/Baseline/istate.events/receiver.http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-21-03-17
+#open	2013-05-21-21-11-40
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1363986197.076696	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-
+1369170699.511968	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-	-
-#close	2013-03-22-21-03-18
+#close	2013-05-21-21-11-41

testing/btest/Baseline/istate.events/sender.http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-04-10-15-48-08
+#open	2013-05-21-21-11-40
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1365608887.935644	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-
+1369170699.511968	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-	-
-#close	2013-04-10-15-48-09
+#close	2013-05-21-21-11-41

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/QJO04kPdawk-file

@@ -0,0 +1 @@
+test2

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/TJdltRTxco1-file

@@ -0,0 +1 @@
+test

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/TaUJcEIboHh-file

@@ -0,0 +1,21 @@
+{
+  "data": "",
+  "form": {
+    "example": "test",
+    "example2": "test2",
+    "example3": "test3"
+  },
+  "origin": "141.142.228.5",
+  "json": null,
+  "url": "http://httpbin.org/post",
+  "args": {},
+  "headers": {
+    "Content-Type": "multipart/form-data; boundary=----------------------------4ebf00fbcf09",
+    "User-Agent": "curl/7.30.0",
+    "Connection": "close",
+    "Accept": "*/*",
+    "Content-Length": "350",
+    "Host": "httpbin.org"
+  },
+  "files": {}
+}

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/dDH5dHdsRH4-file

@@ -0,0 +1 @@
+test3

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out

@@ -0,0 +1,53 @@
+FILE_NEW
+TJdltRTxco1, 0, 0
+FILE_BOF_BUFFER
+test^M^J
+MIME_TYPE
+text/plain
+FILE_STATE_REMOVE
+TJdltRTxco1, 6, 0
+[orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp]
+source: HTTP
+MD5: 9f06243abcb89c70e0c331c61d871fa7
+SHA1: fde773a18bb29f5ed65e6f0a7aa717fd1fa485d4
+SHA256: 837ccb607e312b170fac7383d7ccfd61fa5072793f19a25e75fbacb56539b86b
+FILE_NEW
+QJO04kPdawk, 0, 0
+FILE_BOF_BUFFER
+test2^M^J
+MIME_TYPE
+text/plain
+FILE_STATE_REMOVE
+QJO04kPdawk, 7, 0
+[orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp]
+source: HTTP
+MD5: d68af81ef370b3873d50f09140068810
+SHA1: 51a7b6f2d91f6a87822dc04560f2972bc14fc97e
+SHA256: de0edd0ac4a705aff70f34734e90a1d0a1d8b76abe4bb53f3ea934bc105b3b17
+FILE_NEW
+dDH5dHdsRH4, 0, 0
+FILE_BOF_BUFFER
+test3^M^J
+MIME_TYPE
+text/plain
+FILE_STATE_REMOVE
+dDH5dHdsRH4, 7, 0
+[orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp]
+source: HTTP
+MD5: 1a3d75d44753ad246f0bd333cdaf08b0
+SHA1: 4f98809ab09272dfcc58266e3f23ae2393f70e76
+SHA256: 018c67a2c30ed9977e1dddfe98cac542165dac355cf9764c91a362613e752933
+FILE_NEW
+TaUJcEIboHh, 0, 0
+FILE_BOF_BUFFER
+{^J  "data":
+MIME_TYPE
+text/plain
+FILE_STATE_REMOVE
+TaUJcEIboHh, 465, 0
+[orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp]
+total bytes: 465
+source: HTTP
+MD5: 226244811006caf4ac904344841168dd
+SHA1: 7222902b8b8e68e25c0422e7f8bdf344efeda54d
+SHA256: dd485ecf240e12807516b0a27718fc3ab9a17c1158a452967343c98cefba07a0

testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	file_analysis
-#open	2013-04-23-15-41-01
+#open	2013-05-21-16-47-14
-#fields	id	parent_id	source	is_orig	last_active	seen_bytes	total_bytes	missing_bytes	overflow_bytes	timeout_interval	bof_buffer_size	mime_type	timedout	conn_uids	analyzers	extracted_files	md5	sha1	sha256
+#fields	id	parent_id	source	is_orig	last_active	seen_bytes	total_bytes	missing_bytes	overflow_bytes	timeout_interval	bof_buffer_size	mime_type	timedout	conn_uids	extracted_files	md5	sha1	sha256
-#types	string	string	string	bool	time	count	count	count	count	interval	count	string	bool	table[string]	table[enum]	table[string]	string	string	string
+#types	string	string	string	bool	time	count	count	count	count	interval	count	string	bool	table[string]	table[string]	string	string	string
-Cx92a0ym5R8	-	HTTP	F	1362692527.009775	4705	4705	0	0	120.000000	1024	text/plain	F	UWkUyAuUGXf	FileAnalysis::ANALYZER_SHA1,FileAnalysis::ANALYZER_EXTRACT,FileAnalysis::ANALYZER_DATA_EVENT,FileAnalysis::ANALYZER_MD5,FileAnalysis::ANALYZER_SHA256	Cx92a0ym5R8-file	397168fd09991a0e712254df7bc639ac	1dd7ac0398df6cbc0696445a91ec681facf4dc47	4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
+Cx92a0ym5R8	-	HTTP	F	1362692527.009775	4705	4705	0	0	120.000000	1024	text/plain	F	UWkUyAuUGXf	Cx92a0ym5R8-file	397168fd09991a0e712254df7bc639ac	1dd7ac0398df6cbc0696445a91ec681facf4dc47	4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
-#close	2013-04-23-15-41-01
+#close	2013-05-21-16-47-14

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-14-38-21
+#open	2013-05-21-21-11-23
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1315799856.264750	UWkUyAuUGXf	10.0.1.104	64216	193.40.5.162	80	1	GET	lepo.it.da.ut.ee	/~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf	-	Wget/1.12 (darwin10.8.0)	0	346	404	Not Found	-	-	-	(empty)	-	-	-	text/html	-	-
+1315799856.264750	UWkUyAuUGXf	10.0.1.104	64216	193.40.5.162	80	1	GET	lepo.it.da.ut.ee	/~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf	-	Wget/1.12 (darwin10.8.0)	0	346	404	Not Found	-	-	-	(empty)	-	-	-	text/html	-	-	-
-#close	2013-03-22-14-38-21
+#close	2013-05-21-21-11-23

testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.wikipedia/http.select

@@ -1,14 +1,14 @@
-1300475168.78402|j4u32Pc5bif|141.142.220.118|48649|208.80.152.118|80|1|GET|bits.wikimedia.org|/skins-1.5/monobook/main.css|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.78402|j4u32Pc5bif|141.142.220.118|48649|208.80.152.118|80|1|GET|bits.wikimedia.org|/skins-1.5/monobook/main.css|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475168.91602|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/6/63/Wikipedia-logo.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.91602|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/6/63/Wikipedia-logo.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475168.91618|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.91618|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475168.91836|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/b/bd/Bookshelf-40x201_6.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.91836|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/b/bd/Bookshelf-40x201_6.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475168.9523|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.9523|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475168.95231|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.95231|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475168.95482|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.95482|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475168.96269|i2rO3KD1Syg|141.142.220.118|35642|208.80.152.2|80|1|GET|meta.wikimedia.org|/images/wikimedia-button.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.96269|i2rO3KD1Syg|141.142.220.118|35642|208.80.152.2|80|1|GET|meta.wikimedia.org|/images/wikimedia-button.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475168.97593|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.97593|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475168.97644|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.97644|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475168.97926|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475168.97926|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475169.01459|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475169.01459|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475169.01462|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475169.01462|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||
-1300475169.01493|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||
+1300475169.01493|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||||

testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log

@@ -3,21 +3,21 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-14-38-24
+#open	2013-05-21-21-11-23
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1300475168.784020	j4u32Pc5bif	141.142.220.118	48649	208.80.152.118	80	1	GET	bits.wikimedia.org	/skins-1.5/monobook/main.css	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.784020	j4u32Pc5bif	141.142.220.118	48649	208.80.152.118	80	1	GET	bits.wikimedia.org	/skins-1.5/monobook/main.css	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475168.916018	VW0XPVINV8a	141.142.220.118	49997	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/6/63/Wikipedia-logo.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.916018	VW0XPVINV8a	141.142.220.118	49997	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/6/63/Wikipedia-logo.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475168.916183	3PKsZ2Uye21	141.142.220.118	49996	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.916183	3PKsZ2Uye21	141.142.220.118	49996	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475168.918358	GSxOnSLghOa	141.142.220.118	49998	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/b/bd/Bookshelf-40x201_6.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.918358	GSxOnSLghOa	141.142.220.118	49998	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/b/bd/Bookshelf-40x201_6.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475168.952307	Tw8jXtpTGu6	141.142.220.118	50000	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.952307	Tw8jXtpTGu6	141.142.220.118	50000	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475168.952296	P654jzLoe3a	141.142.220.118	49999	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.952296	P654jzLoe3a	141.142.220.118	49999	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475168.954820	0Q4FH8sESw5	141.142.220.118	50001	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.954820	0Q4FH8sESw5	141.142.220.118	50001	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475168.962687	i2rO3KD1Syg	141.142.220.118	35642	208.80.152.2	80	1	GET	meta.wikimedia.org	/images/wikimedia-button.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.962687	i2rO3KD1Syg	141.142.220.118	35642	208.80.152.2	80	1	GET	meta.wikimedia.org	/images/wikimedia-button.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475168.975934	VW0XPVINV8a	141.142.220.118	49997	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.975934	VW0XPVINV8a	141.142.220.118	49997	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475168.976436	3PKsZ2Uye21	141.142.220.118	49996	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.976436	3PKsZ2Uye21	141.142.220.118	49996	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475168.979264	GSxOnSLghOa	141.142.220.118	49998	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.979264	GSxOnSLghOa	141.142.220.118	49998	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475169.014619	Tw8jXtpTGu6	141.142.220.118	50000	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475169.014619	Tw8jXtpTGu6	141.142.220.118	50000	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475169.014593	P654jzLoe3a	141.142.220.118	49999	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475169.014593	P654jzLoe3a	141.142.220.118	49999	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-1300475169.014927	0Q4FH8sESw5	141.142.220.118	50001	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475169.014927	0Q4FH8sESw5	141.142.220.118	50001	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-	-
-#close	2013-03-22-14-38-24
+#close	2013-05-21-21-11-23

testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log

@@ -9,13 +9,13 @@
 1329843175.680248	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	PASV	-	-	-	227	Entering Passive Mode (199,233,217,249,221,90)	(empty)	T	141.142.220.235	199.233.217.249	56666	-
 1329843175.791528	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	LIST	-	-	-	226	Transfer complete.	(empty)	-	-	-	-	-
 1329843179.815947	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	PASV	-	-	-	227	Entering Passive Mode (199,233,217,249,221,91)	(empty)	T	141.142.220.235	199.233.217.249	56667	-
-1329843193.984222	arKYeMETxOg	141.142.220.235	37604	199.233.217.249	56666	<ftp-data>	-	-	-	-	-	-	-	(empty)	-	-	-	-	ftp-item-Rqjkzoroau4-0.dat
+1329843193.984222	arKYeMETxOg	141.142.220.235	37604	199.233.217.249	56666	<ftp-data>	-	-	-	-	-	-	-	(empty)	-	-	-	-	ftp-item-Rqjkzoroau4.dat
-1329843193.984222	k6kgXLOoSKl	141.142.220.235	59378	199.233.217.249	56667	<ftp-data>	-	-	-	-	-	-	-	(empty)	-	-	-	-	ftp-item-BTsa70Ua9x7-1.dat
+1329843193.984222	k6kgXLOoSKl	141.142.220.235	59378	199.233.217.249	56667	<ftp-data>	-	-	-	-	-	-	-	(empty)	-	-	-	-	ftp-item-BTsa70Ua9x7.dat
 1329843179.926563	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	RETR	ftp://199.233.217.249/./robots.txt	text/plain	77	226	Transfer complete.	(empty)	-	-	-	-	-
 1329843194.040188	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	PORT	141,142,220,235,131,46	-	-	200	PORT command successful.	(empty)	F	199.233.217.249	141.142.220.235	33582	-
 1329843194.095782	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	LIST	-	-	-	226	Transfer complete.	(empty)	-	-	-	-	-
 1329843197.672179	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	PORT	141,142,220,235,147,203	-	-	200	PORT command successful.	(empty)	F	199.233.217.249	141.142.220.235	37835	-
-1329843199.968212	nQcgTWjvg4c	199.233.217.249	61920	141.142.220.235	33582	<ftp-data>	-	-	-	-	-	-	-	(empty)	-	-	-	-	ftp-item-VLQvJybrm38-2.dat
+1329843199.968212	nQcgTWjvg4c	199.233.217.249	61920	141.142.220.235	33582	<ftp-data>	-	-	-	-	-	-	-	(empty)	-	-	-	-	ftp-item-VLQvJybrm38.dat
 1329843197.727769	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	RETR	ftp://199.233.217.249/./robots.txt	text/plain	77	226	Transfer complete.	(empty)	-	-	-	-	-
-1329843200.079930	j4u32Pc5bif	199.233.217.249	61918	141.142.220.235	37835	<ftp-data>	-	-	-	-	-	-	-	(empty)	-	-	-	-	ftp-item-zrfwSs9K1yk-3.dat
+1329843200.079930	j4u32Pc5bif	199.233.217.249	61918	141.142.220.235	37835	<ftp-data>	-	-	-	-	-	-	-	(empty)	-	-	-	-	ftp-item-zrfwSs9K1yk.dat
 #close	2013-04-12-16-32-25

testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-14-38-28
+#open	2013-05-21-21-11-24
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1237440095.634312	UWkUyAuUGXf	192.168.3.103	54102	128.146.216.51	80	1	POST	www.osu.edu	/	-	curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3	2001	60731	200	OK	100	Continue	-	(empty)	-	-	-	text/html	-	-
+1237440095.634312	UWkUyAuUGXf	192.168.3.103	54102	128.146.216.51	80	1	POST	www.osu.edu	/	-	curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3	2001	60731	200	OK	100	Continue	-	(empty)	-	-	-	text/html	-	-	-
-#close	2013-03-22-14-38-28
+#close	2013-05-21-21-11-24

testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-14-38-28
+#open	2013-05-21-21-11-25
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1128727435.634189	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	http-item-BFymS6bFgT3-0.dat
+1128727435.634189	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-	http-item-BFymS6bFgT3.dat
-#close	2013-03-22-14-38-28
+#close	2013-05-21-21-11-25

testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log

@@ -3,56 +3,56 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-25-20-20-22
+#open	2013-05-21-21-11-25
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1354328870.191989	UWkUyAuUGXf	128.2.6.136	46562	173.194.75.103	80	1	OPTIONS	www.google.com	*	-	-	0	962	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328870.191989	UWkUyAuUGXf	128.2.6.136	46562	173.194.75.103	80	1	OPTIONS	www.google.com	*	-	-	0	962	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328874.237327	arKYeMETxOg	128.2.6.136	46563	173.194.75.103	80	1	OPTIONS	www.google.com	HTTP/1.1	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328874.237327	arKYeMETxOg	128.2.6.136	46563	173.194.75.103	80	1	OPTIONS	www.google.com	HTTP/1.1	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328874.299063	k6kgXLOoSKl	128.2.6.136	46564	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328874.299063	k6kgXLOoSKl	128.2.6.136	46564	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328874.342591	nQcgTWjvg4c	128.2.6.136	46565	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328874.342591	nQcgTWjvg4c	128.2.6.136	46565	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328874.364020	j4u32Pc5bif	128.2.6.136	46566	173.194.75.103	80	1	GET	www.google.com	/	-	-	0	43911	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328874.364020	j4u32Pc5bif	128.2.6.136	46566	173.194.75.103	80	1	GET	www.google.com	/	-	-	0	43911	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328878.470424	TEfuqmmG4bh	128.2.6.136	46567	173.194.75.103	80	1	GET	www.google.com	/	-	-	0	43983	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328878.470424	TEfuqmmG4bh	128.2.6.136	46567	173.194.75.103	80	1	GET	www.google.com	/	-	-	0	43983	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328882.575456	FrJExwHcSal	128.2.6.136	46568	173.194.75.103	80	1	GET	www.google.com	/HTTP/1.1	-	-	0	1207	403	Forbidden	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328882.575456	FrJExwHcSal	128.2.6.136	46568	173.194.75.103	80	1	GET	www.google.com	/HTTP/1.1	-	-	0	1207	403	Forbidden	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328882.928027	5OKnoww6xl4	128.2.6.136	46569	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328882.928027	5OKnoww6xl4	128.2.6.136	46569	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328882.968948	3PKsZ2Uye21	128.2.6.136	46570	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328882.968948	3PKsZ2Uye21	128.2.6.136	46570	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328882.990373	VW0XPVINV8a	128.2.6.136	46571	173.194.75.103	80	1	GET	www.google.com	/	-	-	0	43913	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328882.990373	VW0XPVINV8a	128.2.6.136	46571	173.194.75.103	80	1	GET	www.google.com	/	-	-	0	43913	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328887.114613	fRFu0wcOle6	128.2.6.136	46572	173.194.75.103	80	0	-	-	-	-	-	0	961	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328887.114613	fRFu0wcOle6	128.2.6.136	46572	173.194.75.103	80	0	-	-	-	-	-	0	961	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328891.161077	qSsw6ESzHV4	128.2.6.136	46573	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328891.161077	qSsw6ESzHV4	128.2.6.136	46573	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328891.204740	iE6yhOq3SF	128.2.6.136	46574	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328891.204740	iE6yhOq3SF	128.2.6.136	46574	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328891.245592	GSxOnSLghOa	128.2.6.136	46575	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328891.245592	GSxOnSLghOa	128.2.6.136	46575	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328891.287655	qCaWGmzFtM5	128.2.6.136	46576	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328891.287655	qCaWGmzFtM5	128.2.6.136	46576	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328891.309065	70MGiRM1Qf4	128.2.6.136	46577	173.194.75.103	80	1	CCM_POST	www.google.com	/	-	-	0	963	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328891.309065	70MGiRM1Qf4	128.2.6.136	46577	173.194.75.103	80	1	CCM_POST	www.google.com	/	-	-	0	963	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328895.355012	h5DsfNtYzi1	128.2.6.136	46578	173.194.75.103	80	1	CCM_POST	www.google.com	/HTTP/1.1	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328895.355012	h5DsfNtYzi1	128.2.6.136	46578	173.194.75.103	80	1	CCM_POST	www.google.com	/HTTP/1.1	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328895.416133	P654jzLoe3a	128.2.6.136	46579	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328895.416133	P654jzLoe3a	128.2.6.136	46579	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328895.459490	Tw8jXtpTGu6	128.2.6.136	46580	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328895.459490	Tw8jXtpTGu6	128.2.6.136	46580	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328895.480865	c4Zw9TmAE05	128.2.6.136	46581	173.194.75.103	80	1	CCM_POST	www.google.com	/	-	-	0	963	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328895.480865	c4Zw9TmAE05	128.2.6.136	46581	173.194.75.103	80	1	CCM_POST	www.google.com	/	-	-	0	963	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328899.526682	EAr0uf4mhq	128.2.6.136	46582	173.194.75.103	80	1	CONNECT	www.google.com	/	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328899.526682	EAr0uf4mhq	128.2.6.136	46582	173.194.75.103	80	1	CONNECT	www.google.com	/	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328903.572533	GvmoxJFXdTa	128.2.6.136	46583	173.194.75.103	80	1	CONNECT	www.google.com	/HTTP/1.1	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328903.572533	GvmoxJFXdTa	128.2.6.136	46583	173.194.75.103	80	1	CONNECT	www.google.com	/HTTP/1.1	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328903.634196	0Q4FH8sESw5	128.2.6.136	46584	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328903.634196	0Q4FH8sESw5	128.2.6.136	46584	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328903.676395	slFea8xwSmb	128.2.6.136	46585	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328903.676395	slFea8xwSmb	128.2.6.136	46585	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328903.697693	UfGkYA2HI2g	128.2.6.136	46586	173.194.75.103	80	1	CONNECT	www.google.com	/	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328903.697693	UfGkYA2HI2g	128.2.6.136	46586	173.194.75.103	80	1	CONNECT	www.google.com	/	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328907.743696	i2rO3KD1Syg	128.2.6.136	46587	173.194.75.103	80	1	TRACE	www.google.com	/	-	-	0	960	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328907.743696	i2rO3KD1Syg	128.2.6.136	46587	173.194.75.103	80	1	TRACE	www.google.com	/	-	-	0	960	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328911.790590	2cx26uAvUPl	128.2.6.136	46588	173.194.75.103	80	1	TRACE	www.google.com	/HTTP/1.1	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328911.790590	2cx26uAvUPl	128.2.6.136	46588	173.194.75.103	80	1	TRACE	www.google.com	/HTTP/1.1	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328911.853464	BWaU4aSuwkc	128.2.6.136	46589	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328911.853464	BWaU4aSuwkc	128.2.6.136	46589	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328911.897044	10XodEwRycf	128.2.6.136	46590	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328911.897044	10XodEwRycf	128.2.6.136	46590	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328911.918511	zno26fFZkrh	128.2.6.136	46591	173.194.75.103	80	1	TRACE	www.google.com	/	-	-	0	960	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328911.918511	zno26fFZkrh	128.2.6.136	46591	173.194.75.103	80	1	TRACE	www.google.com	/	-	-	0	960	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328915.964678	v5rgkJBig5l	128.2.6.136	46592	173.194.75.103	80	1	DELETE	www.google.com	/	-	-	0	961	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328915.964678	v5rgkJBig5l	128.2.6.136	46592	173.194.75.103	80	1	DELETE	www.google.com	/	-	-	0	961	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328920.010458	eWZCH7OONC1	128.2.6.136	46593	173.194.75.103	80	1	DELETE	www.google.com	/HTTP/1.1	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328920.010458	eWZCH7OONC1	128.2.6.136	46593	173.194.75.103	80	1	DELETE	www.google.com	/HTTP/1.1	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328920.072101	0Pwk3ntf8O3	128.2.6.136	46594	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328920.072101	0Pwk3ntf8O3	128.2.6.136	46594	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328920.114526	0HKorjr8Zp7	128.2.6.136	46595	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328920.114526	0HKorjr8Zp7	128.2.6.136	46595	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328920.136714	yC2d6kVg709	128.2.6.136	46596	173.194.75.103	80	1	DELETE	www.google.com	/	-	-	0	961	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328920.136714	yC2d6kVg709	128.2.6.136	46596	173.194.75.103	80	1	DELETE	www.google.com	/	-	-	0	961	405	Method Not Allowed	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328924.183211	VcgagLjnO92	128.2.6.136	46597	173.194.75.103	80	1	PUT	www.google.com	/	-	-	0	934	411	Length Required	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328924.183211	VcgagLjnO92	128.2.6.136	46597	173.194.75.103	80	1	PUT	www.google.com	/	-	-	0	934	411	Length Required	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328924.224567	bdRoHfaPBo3	128.2.6.136	46598	173.194.75.103	80	1	PUT	www.google.com	/HTTP/1.1	-	-	0	934	411	Length Required	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328924.224567	bdRoHfaPBo3	128.2.6.136	46598	173.194.75.103	80	1	PUT	www.google.com	/HTTP/1.1	-	-	0	934	411	Length Required	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328924.287402	zHqb7t7kv28	128.2.6.136	46599	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328924.287402	zHqb7t7kv28	128.2.6.136	46599	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328924.328257	rrZWoMUQpv8	128.2.6.136	46600	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328924.328257	rrZWoMUQpv8	128.2.6.136	46600	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328924.350343	xNYSS2hJkle	128.2.6.136	46601	173.194.75.103	80	1	PUT	www.google.com	/	-	-	0	934	411	Length Required	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328924.350343	xNYSS2hJkle	128.2.6.136	46601	173.194.75.103	80	1	PUT	www.google.com	/	-	-	0	934	411	Length Required	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328924.391728	vMVjlplKKbd	128.2.6.136	46602	173.194.75.103	80	1	POST	www.google.com	/	-	-	0	934	411	Length Required	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328924.391728	vMVjlplKKbd	128.2.6.136	46602	173.194.75.103	80	1	POST	www.google.com	/	-	-	0	934	411	Length Required	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328924.433150	3omNawSNrxj	128.2.6.136	46603	173.194.75.103	80	1	POST	www.google.com	/HTTP/1.1	-	-	0	934	411	Length Required	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328924.433150	3omNawSNrxj	128.2.6.136	46603	173.194.75.103	80	1	POST	www.google.com	/HTTP/1.1	-	-	0	934	411	Length Required	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328924.496732	Rv8AJVfi9Zi	128.2.6.136	46604	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328924.496732	Rv8AJVfi9Zi	128.2.6.136	46604	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328924.537671	wEyF3OvvcQe	128.2.6.136	46605	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328924.537671	wEyF3OvvcQe	128.2.6.136	46605	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328924.559704	E490YZTUozc	128.2.6.136	46606	173.194.75.103	80	1	HEAD	www.google.com	/	-	-	0	0	200	OK	-	-	-	(empty)	-	-	-	-	-	-
+1354328924.559704	E490YZTUozc	128.2.6.136	46606	173.194.75.103	80	1	HEAD	www.google.com	/	-	-	0	0	200	OK	-	-	-	(empty)	-	-	-	-	-	-	-
-1354328928.625437	YIeWJmXWNWj	128.2.6.136	46607	173.194.75.103	80	1	HEAD	www.google.com	/	-	-	0	0	200	OK	-	-	-	(empty)	-	-	-	-	-	-
+1354328928.625437	YIeWJmXWNWj	128.2.6.136	46607	173.194.75.103	80	1	HEAD	www.google.com	/	-	-	0	0	200	OK	-	-	-	(empty)	-	-	-	-	-	-	-
-1354328932.692706	ydiZblvsYri	128.2.6.136	46608	173.194.75.103	80	1	HEAD	www.google.com	/HTTP/1.1	-	-	0	0	400	Bad Request	-	-	-	(empty)	-	-	-	-	-	-
+1354328932.692706	ydiZblvsYri	128.2.6.136	46608	173.194.75.103	80	1	HEAD	www.google.com	/HTTP/1.1	-	-	0	0	400	Bad Request	-	-	-	(empty)	-	-	-	-	-	-	-
-1354328932.754657	HFYOnBqSE5e	128.2.6.136	46609	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328932.754657	HFYOnBqSE5e	128.2.6.136	46609	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-1354328932.796568	JcUvhfWUMgd	128.2.6.136	46610	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-
+1354328932.796568	JcUvhfWUMgd	128.2.6.136	46610	173.194.75.103	80	0	-	-	-	-	-	0	925	400	Bad Request	-	-	-	(empty)	-	-	-	text/html	-	-	-
-#close	2013-03-25-20-20-22
+#close	2013-05-21-21-11-25

testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log

@@ -3,12 +3,12 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-16-25-59
+#open	2013-05-21-21-11-25
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
-1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	1	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2675	200	OK	-	-	-	(empty)	-	-	-	text/plain	-	-
+1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	1	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2675	200	OK	-	-	-	(empty)	-	-	-	text/plain	-	-	-
-1258577884.960135	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	2	GET	www.mozilla.org	/script/urchin.js	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	21421	200	OK	-	-	-	(empty)	-	-	-	text/plain	-	-
+1258577884.960135	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	2	GET	www.mozilla.org	/script/urchin.js	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	21421	200	OK	-	-	-	(empty)	-	-	-	text/plain	-	-	-
-1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	3	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	94	200	OK	-	-	-	(empty)	-	-	-	image/gif	-	-
+1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	3	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	94	200	OK	-	-	-	(empty)	-	-	-	image/gif	-	-	-
-1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	4	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2349	200	OK	-	-	-	(empty)	-	-	-	image/png	e0029eea80812e9a8e57b8d05d52938a	-
+1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	4	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2349	200	OK	-	-	-	(empty)	-	-	-	image/png	e0029eea80812e9a8e57b8d05d52938a	-	-
-1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	5	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	27579	200	OK	-	-	-	(empty)	-	-	-	image/png	30aa926344f58019d047e85ba049ca1e	-
+1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	5	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	27579	200	OK	-	-	-	(empty)	-	-	-	image/png	30aa926344f58019d047e85ba049ca1e	-	-
-#close	2013-03-22-16-25-59
+#close	2013-05-21-21-11-25

testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log

@@ -3,12 +3,12 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-14-38-28
+#open	2013-05-21-21-11-25
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	md5	extracted_request_files	extracted_response_files
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	vector[string]	vector[string]
-1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	1	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2675	200	OK	-	-	-	(empty)	-	-	-	-	-
+1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	1	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2675	200	OK	-	-	-	(empty)	-	-	-	-	-	-
-1258577884.960135	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	2	GET	www.mozilla.org	/script/urchin.js	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	21421	200	OK	-	-	-	(empty)	-	-	-	-	-
+1258577884.960135	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	2	GET	www.mozilla.org	/script/urchin.js	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	21421	200	OK	-	-	-	(empty)	-	-	-	-	-	-
-1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	3	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	94	200	OK	-	-	-	(empty)	-	-	-	-	-
+1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	3	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	94	200	OK	-	-	-	(empty)	-	-	-	-	-	-
-1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	4	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2349	200	OK	-	-	-	(empty)	-	-	-	-	-
+1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	4	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2349	200	OK	-	-	-	(empty)	-	-	-	-	-	-
-1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	5	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	27579	200	OK	-	-	-	(empty)	-	-	-	-	-
+1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	5	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	27579	200	OK	-	-	-	(empty)	-	-	-	-	-	-
-#close	2013-03-22-14-38-28
+#close	2013-05-21-21-11-25

testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http-item-QJO04kPdawk.dat

@@ -0,0 +1 @@
+test2

testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http-item-TJdltRTxco1.dat

@@ -0,0 +1 @@
+test

testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http-item-TaUJcEIboHh.dat

@@ -0,0 +1,21 @@
+{
+  "data": "",
+  "form": {
+    "example": "test",
+    "example2": "test2",
+    "example3": "test3"
+  },
+  "origin": "141.142.228.5",
+  "json": null,
+  "url": "http://httpbin.org/post",
+  "args": {},
+  "headers": {
+    "Content-Type": "multipart/form-data; boundary=----------------------------4ebf00fbcf09",
+    "User-Agent": "curl/7.30.0",
+    "Connection": "close",
+    "Accept": "*/*",
+    "Content-Length": "350",
+    "Host": "httpbin.org"
+  },
+  "files": {}
+}

testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http-item-dDH5dHdsRH4.dat

@@ -0,0 +1 @@
+test3

testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#open	2013-05-21-21-31-32
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extracted_request_files	extracted_response_files
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	vector[string]	vector[string]
+1369159408.455878	UWkUyAuUGXf	141.142.228.5	57262	54.243.88.146	80	1	POST	httpbin.org	/post	-	curl/7.30.0	370	465	200	OK	-	-	-	(empty)	-	-	-	text/plain	-	http-item-TJdltRTxco1.dat,http-item-QJO04kPdawk.dat,http-item-dDH5dHdsRH4.dat	http-item-TaUJcEIboHh.dat
+#close	2013-05-21-21-31-32

testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log

@@ -9,5 +9,5 @@
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	NICK	bloed	-	-	-	-	-
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	-	-	-	-
 1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	JOIN	#easymovies	(empty)	-	-	-	-
-1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	FAKE_MIME	irc-dcc-item-wqKMAamJVSb-0.dat
+1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	application/zip	irc-dcc-item-wqKMAamJVSb.dat
 #close	2013-03-27-18-49-16

testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log

@@ -6,7 +6,7 @@
 #open	2013-03-26-20-43-14
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	filename	content_len	mime_type	md5	extraction_file	excerpt
 #types	time	string	addr	port	addr	port	count	string	count	string	string	string	string
-1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	79	text/plain	-	smtp-entity-cwR7l6Zctxb-0.dat	(empty)
+1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	79	text/plain	-	smtp-entity-cwR7l6Zctxb.dat	(empty)
 1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	1918	text/html	-	-	(empty)
-1254722770.692804	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	NEWS.txt	10823	text/plain	-	smtp-entity-Ltd7QO7jEv3-1.dat	(empty)
+1254722770.692804	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	NEWS.txt	10823	text/plain	-	smtp-entity-Ltd7QO7jEv3.dat	(empty)
 #close	2013-03-26-20-43-14

testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro

@@ -20,7 +20,7 @@ redef default_file_timeout_interval = 2sec;
 event file_timeout(f: fa_file)
 	{
 	if ( timeout_cnt < 1 )
-		FileAnalysis::postpone_timeout(f);
+		FileAnalysis::set_timeout_interval(f, f$timeout_interval);
 	else
 		terminate();
 	++timeout_cnt;

testing/btest/scripts/base/frameworks/file-analysis/http/multipart.bro

@@ -0,0 +1,13 @@
+# @TEST-EXEC: bro -r $TRACES/http/multipart.trace $SCRIPTS/file-analysis-test.bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff TJdltRTxco1-file
+# @TEST-EXEC: btest-diff QJO04kPdawk-file
+# @TEST-EXEC: btest-diff dDH5dHdsRH4-file
+# @TEST-EXEC: btest-diff TaUJcEIboHh-file
+
+redef test_file_analysis_source = "HTTP";
+
+redef test_get_file_name = function(f: fa_file): string
+	{
+	return fmt("%s-file", f$id);
+	};

testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro

@@ -18,28 +18,12 @@ redef test_get_file_name = function(f: fa_file): string
 T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	4242
 @TEST-END-FILE
 
-module A;
-
-type Val: record {
-	s: string;
-};
-
-event line(description: Input::EventDescription, tpe: Input::Event, s: string)
-	{
-	FileAnalysis::data_stream(description$source, s);
-	}
-
-event Input::end_of_data(name: string, source: string) 
-	{
-	FileAnalysis::eof(source);
-	}
-
 event bro_init()
 	{
-	Input::add_event([$source="../input.log", $reader=Input::READER_BINARY,
+	local source: string = "../input.log";
-	                  $mode=Input::MANUAL, $name="input", $fields=Val,
+	Input::add_analysis([$source=source, $reader=Input::READER_BINARY,
-	                  $ev=line, $want_record=F]);
+	                     $mode=Input::MANUAL, $name=source]);
-	Input::remove("input");
+	Input::remove(source);
 	}
 
 event file_state_remove(f: fa_file) &priority=-10

testing/btest/scripts/base/protocols/ftp/ftp-extract.bro

@@ -3,10 +3,10 @@
 # @TEST-EXEC: bro -r $TRACES/ftp/ipv4.trace %INPUT
 # @TEST-EXEC: btest-diff conn.log
 # @TEST-EXEC: btest-diff ftp.log
-# @TEST-EXEC: btest-diff ftp-item-Rqjkzoroau4-0.dat
+# @TEST-EXEC: btest-diff ftp-item-Rqjkzoroau4.dat
-# @TEST-EXEC: btest-diff ftp-item-BTsa70Ua9x7-1.dat
+# @TEST-EXEC: btest-diff ftp-item-BTsa70Ua9x7.dat
-# @TEST-EXEC: btest-diff ftp-item-VLQvJybrm38-2.dat
+# @TEST-EXEC: btest-diff ftp-item-VLQvJybrm38.dat
-# @TEST-EXEC: btest-diff ftp-item-zrfwSs9K1yk-3.dat
+# @TEST-EXEC: btest-diff ftp-item-zrfwSs9K1yk.dat
 
 redef FTP::logged_commands += {"LIST"};
 redef FTP::extract_file_types=/.*/;

testing/btest/scripts/base/protocols/http/http-extract-files.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT
 # @TEST-EXEC: btest-diff http.log
-# @TEST-EXEC: btest-diff http-item-BFymS6bFgT3-0.dat
+# @TEST-EXEC: btest-diff http-item-BFymS6bFgT3.dat
 
 redef HTTP::extract_file_types += /text\/html/;

testing/btest/scripts/base/protocols/http/multipart-extract.bro

@@ -0,0 +1,8 @@
+# @TEST-EXEC: bro -C -r $TRACES/http/multipart.trace %INPUT
+# @TEST-EXEC: btest-diff http.log
+# @TEST-EXEC: btest-diff http-item-TJdltRTxco1.dat
+# @TEST-EXEC: btest-diff http-item-QJO04kPdawk.dat
+# @TEST-EXEC: btest-diff http-item-dDH5dHdsRH4.dat
+# @TEST-EXEC: btest-diff http-item-TaUJcEIboHh.dat
+
+redef HTTP::extract_file_types += /.*/;

testing/btest/scripts/base/protocols/irc/dcc-extract.test

@@ -1,26 +1,10 @@
 # This tests that the contents of a DCC transfer negotiated with IRC can be
-# correctly extracted.  The mime type of the file transferred is normalized
+# correctly extracted.
-# to prevent sensitivity to libmagic version being used.
 
 # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT
 # @TEST-EXEC: btest-diff irc.log
-# @TEST-EXEC: btest-diff irc-dcc-item-wqKMAamJVSb-0.dat
+# @TEST-EXEC: btest-diff irc-dcc-item-wqKMAamJVSb.dat
 # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT IRC::extraction_prefix="test"
-# @TEST-EXEC: test -e test-wqKMAamJVSb-0.dat
+# @TEST-EXEC: test -e test-wqKMAamJVSb.dat
 
 redef IRC::extract_file_types=/.*/;
-
-event bro_init()
-	{
-	Log::remove_default_filter(IRC::LOG);
-	Log::add_filter(IRC::LOG, [$name="normalized-mime-types",
-	                           $pred=function(rec: IRC::Info): bool
-		{
-		if ( rec?$dcc_mime_type )
-			{
-			rec$dcc_mime_type = "FAKE_MIME";
-			}
-		return T;
-		}
-	]);
-	}

testing/btest/scripts/base/protocols/smtp/mime-extract.test

@@ -1,10 +1,10 @@
 # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
 # @TEST-EXEC: btest-diff smtp_entities.log
-# @TEST-EXEC: btest-diff smtp-entity-cwR7l6Zctxb-0.dat
+# @TEST-EXEC: btest-diff smtp-entity-cwR7l6Zctxb.dat
-# @TEST-EXEC: btest-diff smtp-entity-Ltd7QO7jEv3-1.dat
+# @TEST-EXEC: btest-diff smtp-entity-Ltd7QO7jEv3.dat
 # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test"
-# @TEST-EXEC: test -e test-cwR7l6Zctxb-0.dat
+# @TEST-EXEC: test -e test-cwR7l6Zctxb.dat
-# @TEST-EXEC: test -e test-Ltd7QO7jEv3-1.dat
+# @TEST-EXEC: test -e test-Ltd7QO7jEv3.dat
 
 @load base/protocols/smtp