Merge remote-tracking branch 'origin/topic/jsiwek/file-analysis' into topic/seth/file-analysis-exe-analyzer

Conflicts:
	src/file_analysis/ActionSet.cc

scripts/base/frameworks/file-analysis/main.bro

@@ -1,8 +1,7 @@
-##! TODO add some comments here
+##! An interface for driving the analysis of files, possibly independent of
+##! any network protocol over which they're transported.
 
 @load base/file_analysis.bif
-
-# TODO: do logging here?
 @load base/frameworks/logging
 
 module FileAnalysis;
@@ -13,10 +12,6 @@ export {
 		LOG
 	};
 
-	## The default buffer size used to reassemble files.
-	# TODO: what's a reasonable default?
-	const default_reassembly_buffer_size: count = 1024*1024 &redef;
-
 	## The default buffer size used for storing the beginning of files.
 	const default_bof_buffer_size: count = 1024 &redef;
 
@@ -24,29 +19,32 @@ export {
 	## before giving up.
 	const default_timeout_interval: interval = 2 mins &redef;
 
-	# Needed a forward declaration for event parameters...
+	## A structure which represents a desired file analysis action to take.
-	type Info: record {};
-
 	type ActionArgs: record {
+		## The type of action.
 		act: Action;
+		## The local filename to which to write an extracted file.  Must be
+		## set when *act* is :bro:see:`FileAnalysis::ACTION_EXTRACT`.
 		extract_filename: string &optional;
-		chunk_event: event(info: Info, data: string, off: count) &optional;
-		stream_event: event(info: Info, data: string) &optional;
 	};
 
+	## A structure which contains the results of certain file analysis actions.
 	type ActionResults: record {
+		## An MD5 digest of the file contents.
 		md5: string &optional;
+		## An SHA1 digest of the file contents.
 		sha1: string &optional;
+		## An SHA256 digest of the file contents.
 		sha256: string &optional;
 	};
 
-	## Contains all metadata related to the analysis of a given file, some
+	## Contains all metadata related to the analysis of a given file.
-	## of which is logged.
 	type Info: record {
-		## Unique identifier associated with a single file.
+		## An identifier associated with a single file.
 		file_id: string &log;
-		## Unique identifier associated with the file if it was extracted
+
-		## from a container file as part of the analysis.
+		## Identifier associated with a container file from which this one was
+		## extracted as part of the file analysis.
 		parent_file_id: string &log &optional;
 
 		## An identification of the source of the file data.  E.g. it may be
@@ -62,18 +60,17 @@ export {
 
 		## Number of bytes provided to the file analysis engine for the file.
 		seen_bytes: count &log &default=0;
-		## Total number of bytes that are supposed to comprise the file content.
+
+		## Total number of bytes that are supposed to comprise the full file.
 		total_bytes: count &log &optional;
 
 		## The number of bytes in the file stream that were completely missed
 		## during the process of analysis e.g. due to dropped packets.
-		## analysis that had to be discarded due to a reassembly buffer size
-		## of *reassembly_buffer_size* being filled.
 		missing_bytes: count &log &default=0;
 
 		## The number of not all-in-sequence bytes in the file stream that
 		## were delivered to file actions/analyzers due to reassembly buffer
-		## size of *reassembly_buffer_size* being filled.
+		## overflow.
 		overflow_bytes: count &log &default=0;
 
 		## The amount of time between receiving new data for this file that
@@ -81,61 +78,246 @@ export {
 		timeout_interval: interval &log &default=default_timeout_interval;
 
 		## The number of bytes at the beginning of a file to save for later
-		## inspection in *bof_buffer* field of
+		## inspection in *bof_buffer* field.
-		## :bro:see:`FileAnalysis::ActionResults`.
 		bof_buffer_size: count &log &default=default_bof_buffer_size;
 
 		## The content of the beginning of a file up to *bof_buffer_size* bytes.
 		## This is also the buffer that's used for file/mime type detection.
 		bof_buffer: string &optional;
 
-		## An initial guess at file type.
+		## A file type provided by libmagic against the *bof_buffer*, or
+		## in the cases where no buffering of the beginning of file occurs,
+		## an initial guess of the file type based on the first data seen.
 		file_type: string &log &optional;
-		## An initial guess at mime type.
+
+		## A mime type provided by libmagic against the *bof_buffer*, or
+		## in the cases where no buffering of the beginning of file occurs,
+		## an initial guess of the mime type based on the first data seen.
 		mime_type: string &log &optional;
 
 		## Actions that have been added to the analysis of this file.
-		## Not meant to be modified directly by scripts.
+		## Only meant for inspection by user scripts, not direct modification.
 		actions: table[ActionArgs] of ActionResults;
 	} &redef;
 
-	## TODO: document
+	## Fields that are derived from existing ones, and are set just in time
+	## for logging purposes.
+	redef record FileAnalysis::Info += {
+		## Whether the file analysis timed out at least once for the file.
+		timedout: bool &log &default=F;
+
+		## Connection UIDS over which the file was transferred.
+		conn_uids: set[string] &log &optional;
+
+		## A set of action types taken during the file analysis.
+		actions_taken: set[Action] &log &optional;
+
+		## Local filenames of file extraction actions.
+		extracted_files: set[string] &log &optional;
+
+		## An MD5 digest of the file contents.
+		md5: string &log &optional;
+
+		## A SHA1 digest of the file contents.
+		sha1: string &log &optional;
+
+		## A SHA256 digest of the file contents.
+		sha256: string &log &optional;
+	};
+
+	## Redefined here just so the *info* parameters of the events have the
+	## right type information.
+	redef record ActionArgs += {
+		## An event which will be generated for all new file contents,
+		## chunk-wise.
+		chunk_event: event(info: Info, data: string, off: count) &optional;
+
+		## An event which will be generated for all new file contents,
+		## stream-wise.
+		stream_event: event(info: Info, data: string) &optional;
+	};
+
+	## Evaluated every time a significant event occurs during the course of
+	## file analysis.  Fields of the *info* argument may be modified or
+	## other actions may be added or removed inside the body of any handlers
+	## of this hook.
 	global policy: hook(trig: Trigger, info: Info);
 
+	## A table that can be used to disable file analysis completely for
+	## any files transferred over given network protocol analyzers.
 	const disable: table[AnalyzerTag] of bool = table() &redef;
 
-	# TODO: wrapper functions for BiFs ?
-
 	## Event that can be handled to access the Info record as it is sent on
 	## to the logging framework.
 	global log_file_analysis: event(rec: Info);
 
 	## The salt concatenated to unique file handle strings generated by
-	## :bro:see:`FileAnalysis::handle_callbacks` before hashing them
+	## :bro:see:`get_file_handle` before hashing them in to a file id
-	## in to a file id (the *file_id* field of :bro:see:`FileAnalysis::Info`).
+	## (the *file_id* field of :bro:see:`FileAnalysis::Info`).
 	## Provided to help mitigate the possiblility of manipulating parts of
 	## network connections that factor in to the file handle in order to
 	## generate two handles that would hash to the same file id.
 	const salt = "I recommend changing this." &redef;
+
+	## Postpones the timeout of file analysis for a given file.
+	## When used within a :bro:see:`FileAnalysis::policy` handler for
+	## :bro:see:`FileAnalysis::TRIGGER_TIMEOUT`, the analysis will delay
+	## timing out for the period of time indicated by the *timeout_interval*
+	## field of :bro:see:`FileAnalysis::Info`.
+	##
+	## file_id: the file identifier string from the *file_id* field of
+	##          :bro:see:`FileAnalysis::Info`.
+	##
+	## Returns: true if the timeout will be postponed, or false if analysis
+	##          for the *file_id* isn't currently active.
+	global postpone_timeout: function(file_id: string): bool;
+
+	## Adds an action to the analysis of a given file.
+	##
+	## file_id: the file identifier string from the *file_id* field of
+	##          :bro:see:`FileAnalysis::Info`.
+	##
+	## args: the action type to add along with any arguments it takes.
+	##
+	## Returns: true if the action will be added, or false if analysis
+	##          for the *file_id* isn't currently active or the *args*
+	##          were invalid for the action type.
+	global add_action: function(file_id: string, args: ActionArgs): bool;
+
+	## Removes an action from the analysis of a given file.
+	##
+	## file_id: the file identifier string from the *file_id* field of
+	##          :bro:see:`FileAnalysis::Info`.
+	##
+	## args: the action (type and args) to remove.
+	##
+	## Returns: true if the action will be removed, or false if analysis
+	##          for the *file_id* isn't currently active.
+	global remove_action: function(file_id: string, args: ActionArgs): bool;
+
+	## Stops/ignores any further analysis of a given file.
+	##
+	## file_id: the file identifier string from the *file_id* field of
+	##          :bro:see:`FileAnalysis::Info`.
+	##
+	## Returns: true if analysis for the given file will be ignored for the
+	##          rest of it's contents, or false if analysis for the *file_id*
+	##          isn't currently active.
+	global stop: function(file_id: string): bool;
+
+	## Sends a sequential stream of data in for file analysis.
+	## Meant for use when providing external file analysis input (e.g.
+	## from the input framework).
+	##
+	## source: a string that uniquely identifies the logical file that the
+	##         data is a part of and describes its source.
+	##
+	## data: bytestring contents of the file to analyze.
+	global data_stream: function(source: string, data: string);
+
+	## Sends a non-sequential chunk of data in for file analysis.
+	## Meant for use when providing external file analysis input (e.g.
+	## from the input framework).
+	##
+	## source: a string that uniquely identifies the logical file that the
+	##         data is a part of and describes its source.
+	##
+	## data: bytestring contents of the file to analyze.
+	##
+	## offset: the offset within the file that this chunk starts.
+	global data_chunk: function(source: string, data: string, offset: count);
+
+	## Signals a content gap in the file bytestream.
+	## Meant for use when providing external file analysis input (e.g.
+	## from the input framework).
+	##
+	## source: a string that uniquely identifies the logical file that the
+	##         data is a part of and describes its source.
+	##
+	## offset: the offset within the file that this gap starts.
+	##
+	## len: the number of bytes that are missing.
+	global gap: function(source: string, offset: count, len: count);
+
+	## Signals the total size of a file.
+	## Meant for use when providing external file analysis input (e.g.
+	## from the input framework).
+	##
+	## source: a string that uniquely identifies the logical file that the
+	##         data is a part of and describes its source.
+	##
+	## size: the number of bytes that comprise the full file.
+	global set_size: function(source: string, size: count);
+
+	## Signals the end of a file.
+	## Meant for use when providing external file analysis input (e.g.
+	## from the input framework).
+	##
+	## source: a string that uniquely identifies the logical file that the
+	##         data is a part of and describes its source.
+	global eof: function(source: string);
 }
 
+function postpone_timeout(file_id: string): bool
+	{
+	return __postpone_timeout(file_id);
+	}
+
+function add_action(file_id: string, args: ActionArgs): bool
+	{
+	return __add_action(file_id, args);
+	}
+
+function remove_action(file_id: string, args: ActionArgs): bool
+	{
+	return __remove_action(file_id, args);
+	}
+
+function stop(file_id: string): bool
+	{
+	return __stop(file_id);
+	}
+
+function data_stream(source: string, data: string)
+	{
+	__data_stream(source, data);
+	}
+
+function data_chunk(source: string, data: string, offset: count)
+	{
+	__data_chunk(source, data, offset);
+	}
+
+function gap(source: string, offset: count, len: count)
+	{
+	__gap(source, offset, len);
+	}
+
+function set_size(source: string, size: count)
+	{
+	__set_size(source, size);
+	}
+
+function eof(source: string)
+	{
+	__eof(source);
+	}
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(FileAnalysis::LOG,
 	                   [$columns=Info, $ev=log_file_analysis]);
 	}
 
-redef record FileAnalysis::Info += {
+hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
-	conn_uids: set[string] &log &optional;
+	&priority=5
-	actions_taken: set[Action] &log &optional;
+	{
-	extracted_files: set[string] &log &optional;
+	if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
-	md5: string &log &optional;
+	info$timedout = T;
-	sha1: string &log &optional;
+	}
-	sha256: string &log &optional;
-};
 
 hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
-	&priority=-10
+	&priority=-5
 	{
 	if ( trig != FileAnalysis::TRIGGER_EOF &&
 	     trig != FileAnalysis::TRIGGER_DONE ) return;

scripts/base/protocols/ftp/file-analysis.bro

@@ -18,9 +18,6 @@ export {
 
 		local info: FTP::Info = ftp_data_expected[c$id$resp_h, c$id$resp_p];
 
-		local rval = fmt("%s %s %s", ANALYZER_FTP_DATA, c$start_time,
-		                 id_string(c$id));
-
 		if ( info$passive )
 			# FTP client initiates data channel.
 			if ( is_orig )
@@ -28,12 +25,14 @@ export {
 				return "";
 			else
 				# Do care about FTP server data.
-				return rval;
+				return cat(ANALYZER_FTP_DATA, " ", c$start_time, " ",
+				           id_string(c$id));
 		else
 			# FTP server initiates dta channel.
 			if ( is_orig )
 				# Do care about FTP server data.
-				return rval;
+				return cat(ANALYZER_FTP_DATA, " ", c$start_time, " ",
+				           id_string(c$id));
 			else
 				# Don't care about FTP client data.
 				return "";

scripts/base/protocols/ftp/main.bro

@@ -66,7 +66,7 @@ export {
 		## Reply message from the server in response to the command.
 		reply_msg:        string      &log &optional;
 		## Arbitrary tags that may indicate a particular attribute of this command.
-		tags:             set[string] &log &default=set();
+		tags:             set[string] &log;
 
 		## Expected FTP data channel.
 		data_channel:     ExpectedDataChannel &log &optional;
@@ -109,6 +109,7 @@ export {
 # Add the state tracking information variable to the connection record
 redef record connection += {
 	ftp: Info &optional;
+	ftp_data_reuse: bool &default=F;
 };
 
 # Configure DPD
@@ -337,7 +338,6 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
 		}
 	}
 
-
 event expected_connection_seen(c: connection, a: count) &priority=10
 	{
 	local id = c$id;
@@ -357,8 +357,15 @@ event file_transferred(c: connection, prefix: string, descr: string,
 		}
 	}
 
+event connection_reused(c: connection) &priority=5
+	{
+	if ( "ftp-data" in c$service )
+		c$ftp_data_reuse = T;
+	}
+	
 event connection_state_remove(c: connection) &priority=-5
 	{
+	if ( c$ftp_data_reuse ) return;
 	delete ftp_data_expected[c$id$resp_h, c$id$resp_p];
 	}
 

scripts/base/protocols/http/file-analysis.bro

@@ -18,11 +18,11 @@ export {
 		if ( ! c?$http ) return "";
 
 		if ( c$http$range_request )
-			return fmt("%s %s %s %s", ANALYZER_HTTP, is_orig, c$id$orig_h,
+			return cat(ANALYZER_HTTP, " ", is_orig, " ", c$id$orig_h,
-			           build_url(c$http));
+			           " ", build_url(c$http));
 
-		return fmt("%s %s %s %s %s", ANALYZER_HTTP, c$start_time, is_orig,
+		return cat(ANALYZER_HTTP, " ", c$start_time, " ", is_orig,
-		           c$http$trans_depth, id_string(c$id));
+		           " ", c$http$trans_depth, " ", id_string(c$id));
 		}
 }
 

scripts/base/protocols/http/file-ident.bro

@@ -9,7 +9,8 @@ module HTTP;
 
 export {
 	redef enum Notice::Type += {
-		## Indicates when the file extension doesn't seem to match the file contents.
+		## Indicates when the file extension doesn't seem to match the file
+		## contents.
 		Incorrect_File_Type,
 	};
 
@@ -18,9 +19,10 @@ export {
 		mime_type:    string   &log &optional;
 	};
 	
-	## Mapping between mime types and regular expressions for URLs
+	## Mapping between mime type strings (without character set) and
-	## The :bro:enum:`HTTP::Incorrect_File_Type` notice is generated if the pattern 
+	## regular expressions for URLs.
-	## doesn't match the mime type that was discovered.
+	## The :bro:enum:`HTTP::Incorrect_File_Type` notice is generated if the
+	## pattern doesn't match the mime type that was discovered.
 	const mime_types_extensions: table[string] of pattern = {
 		["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/,
 	} &redef;
@@ -49,17 +51,66 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
 
 		c$http$mime_type = info$mime_type;
 
-		if ( info$mime_type !in mime_types_extensions ) next;
+		local mime_str: string = split1(info$mime_type, /;/)[1];
+
+		if ( mime_str !in mime_types_extensions ) next;
 		if ( ! c$http?$uri ) next;
-		if ( mime_types_extensions[info$mime_type] in c$http$uri ) next;
+		if ( mime_types_extensions[mime_str] in c$http$uri ) next;
 
 		local url = build_url_http(c$http);
 
 		if ( url == ignored_incorrect_file_type_urls ) next;
 
-		local message = fmt("%s %s %s", info$mime_type, c$http$method, url);
+		local message = fmt("%s %s %s", mime_str, c$http$method, url);
 		NOTICE([$note=Incorrect_File_Type,
 		        $msg=message,
 		        $conn=c]);
 		}
 	}
+
+hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+	&priority=5
+	{
+	if ( trig != FileAnalysis::TRIGGER_NEW_CONN ) return;
+	if ( ! info?$mime_type ) return;
+	if ( ! info?$source ) return;
+	if ( info$source != "HTTP" ) return;
+	if ( ! info?$conns ) return;
+
+	# Spread the mime around (e.g. for partial content, TRIGGER_TYPE only
+	# happens once for the first connection, but if there's subsequent
+	# connections to transfer the same file, they'll be lacking the mime_type
+	# field if we don't do this).
+	for ( cid in info$conns )
+		{
+		local c: connection = info$conns[cid];
+
+		if ( ! c?$http ) next;
+
+		c$http$mime_type = info$mime_type;
+		}
+	}
+
+# Tracks byte-range request / partial content response mime types, indexed
+# by [connection, uri] pairs.  This is needed because a person can pipeline
+# byte-range requests over multiple connections to the same uri.  Without
+# the tracking, only the first request in the pipeline for each connection
+# would get a mime_type field assigned to it (by the FileAnalysis policy hooks).
+global partial_types: table[conn_id, string] of string &read_expire=5mins;
+
+# Priority 4 so that it runs before the handler that will write to http.log.
+event http_message_done(c: connection, is_orig: bool, stat: http_message_stat)
+	&priority=4
+	{
+	if ( ! c$http$range_request ) return;
+	if ( ! c$http?$uri ) return;
+
+	if ( c$http?$mime_type )
+		{
+		partial_types[c$id, c$http$uri] = c$http$mime_type;
+		return;
+		}
+
+	if ( [c$id, c$http$uri] in partial_types )
+		c$http$mime_type = partial_types[c$id, c$http$uri];
+	}

scripts/base/protocols/irc/file-analysis.bro

@@ -15,7 +15,7 @@ export {
 	function get_file_handle(c: connection, is_orig: bool): string
 		{
 		if ( is_orig ) return "";
-		return fmt("%s %s %s", ANALYZER_IRC_DATA, c$start_time,
+		return cat(ANALYZER_IRC_DATA, " ", c$start_time, " ",
 		           id_string(c$id));
 		}
 }

scripts/base/protocols/smtp/entities.bro

@@ -112,6 +112,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
 		local c: connection = info$conns[cid];
 
 		if ( ! c?$smtp ) next;
+		if ( ! c$smtp?$current_entity ) next;
 
 		if ( c$smtp$current_entity$extract_file )
 			{
@@ -180,6 +181,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
 			local c: connection = info$conns[cid];
 
 			if ( ! c?$smtp ) next;
+			if ( ! c$smtp?$current_entity ) next;
 
 			c$smtp$current_entity$mime_type = info$mime_type;
 			}

scripts/base/protocols/smtp/file-analysis.bro

@@ -17,8 +17,8 @@ export {
 		{
 		if ( ! c?$smtp ) return "";
 
-		return fmt("%s %s %s %s", ANALYZER_SMTP, c$start_time,
+		return cat(ANALYZER_SMTP, " ", c$start_time, " ",
-		           c$smtp$trans_depth, c$smtp_state$mime_level);
+		           c$smtp$trans_depth, " ", c$smtp_state$mime_level);
 		}
 }
 

src/Event.h

@@ -118,6 +118,9 @@ public:
 
 	void Describe(ODesc* d) const;
 
+	EventHandlerPtr Tail() { return tail ? tail->handler : EventHandlerPtr(); }
+	val_list* TailArgs() { return tail ? tail->args : 0; }
+
 protected:
 	void QueueEvent(Event* event);
 

src/event.bif

@@ -6986,7 +6986,8 @@ event bro_script_loaded%(path: string, level: count%);
 ## belongs.  All incoming data to the framework is buffered, and depends
 ## on a handler for this event to return a string value that uniquely
 ## identifies a file.  Among all handlers of this event, exactly one must
-## call :bro:see:`return_file_handle`.
+## call :bro:see:`return_file_handle`.  Handlers of this event must not
+## change any global state.
 ##
 ## tag: The analyzer which is carrying the file data.
 ##

src/file_analysis.bif

@@ -10,18 +10,23 @@ type Info: record;
 type ActionArgs: record;
 type ActionResults: record;
 
-## An enumeration of possibly-interesting "events" that can occur over
+## An enumeration of significant things that can occur over the course of
-## the course of analyzing files.  The :bro:see:`FileAnalysis::policy`
+## analyzing files.  The :bro:see:`FileAnalysis::policy` hook is called each
-## hook is called each time a trigger occurs.
+## time a trigger occurs.
 enum Trigger %{
+
 	## Raised when any part of a new file is detected.
 	TRIGGER_NEW,
+
+	## Raised when file is detected being transported over a new network
+	## connection (other than the first).
+	TRIGGER_NEW_CONN,
+
 	## Raised when file analysis has likely seen a complete file.  That
 	## is when a number of bytes indicated by the *total_bytes* field of
-	## :bro:see:`FileAnalysis::Info` have been processed.  Note that
+	## :bro:see:`FileAnalysis::Info` have been processed.
-	## the *undelivered* field does not have to be zero for this to have
-	## occurred.
 	TRIGGER_DONE,
+
 	## Raised when file analysis for a given file is aborted due
 	## to not seeing any data for it recently.  Note that this doesn't
 	## necessarily mean the full file wasn't seen (e.g. if the
@@ -30,45 +35,56 @@ enum Trigger %{
 	## during a :bro:see:`FileAnalysis::policy` handler for this trigger to
 	## defer the timeout until later.
 	TRIGGER_TIMEOUT,
+
 	## Raised when the beginning of a file is detected.
 	TRIGGER_BOF,
-	## Raised when the beginning of a file is available and that beginning
+
-	## is at least the number of bytes indicated by the *bof_buffer_size*
+	## Raised when the beginning of a file is available in the *bof_buffer*
-	## field of :bro:see:`FileAnalysis::Info`.
+	## field of :bro:see:`FileAnalysis::Info` and that beginning
+	## is at least the number of bytes indicated by the *bof_buffer_size* field.
 	TRIGGER_BOF_BUFFER,
-	## Raised when an initial guess at the file/mime type of a file is matched
+
-	## based on magic numbers.
+	## Raised when an initial guess at the file/mime type of a file is matched.
 	TRIGGER_TYPE,
+
 	## Raised to signal that no more file data is incoming and it couldn't be
-	## determined whether the full file was actually seen.
+	## determined whether the full file was actually seen and analyzed.
 	TRIGGER_EOF,
-	## The reassembly buffer for the file filled and had to be discarded.
+
-	## The *undelivered* field of :bro:see:`FileAnalysis::Info` will
-	## indicate the number of bytes, if any, that were not all-in-sequence.
-	## TODO: Is it possible to extend the reassembly buffer when "handling"
-	## this trigger?
-	TRIGGER_REASSEMBLY_BUFFER_FULL,
 	## Raised when there's a missing chunk of data in the file stream.
 	TRIGGER_GAP,
 %}
 
+## An enumeration of various file analysis actions that can be taken.
 enum Action %{
+
+	## Extract a file to local filesystem
 	ACTION_EXTRACT,
+
+	## Calculate an MD5 digest of the file's contents.
 	ACTION_MD5,
+
+	## Calculate an SHA1 digest of the file's contents.
 	ACTION_SHA1,
+
+	## Calculate an SHA256 digest of the file's contents.
 	ACTION_SHA256,
+
+	## Deliver the file contents to the script-layer in an event.
 	ACTION_DATA_EVENT,
 	ACTION_PE_ANALYZER,
 %}
 
-function FileAnalysis::postpone_timeout%(file_id: string%): bool
+## :bro:see:`FileAnalysis::postpone_timeout`.
+function FileAnalysis::__postpone_timeout%(file_id: string%): bool
 	%{
 	using file_analysis::FileID;
 	bool result = file_mgr->PostponeTimeout(FileID(file_id->CheckString()));
 	return new Val(result, TYPE_BOOL);
 	%}
 
-function FileAnalysis::add_action%(file_id: string, args: any%): bool
+## :bro:see:`FileAnalysis::add_action`.
+function FileAnalysis::__add_action%(file_id: string, args: any%): bool
 	%{
 	using file_analysis::FileID;
 	using BifType::Record::FileAnalysis::ActionArgs;
@@ -78,7 +94,8 @@ function FileAnalysis::add_action%(file_id: string, args: any%): bool
 	return new Val(result, TYPE_BOOL);
 	%}
 
-function FileAnalysis::remove_action%(file_id: string, args: any%): bool
+## :bro:see:`FileAnalysis::remove_action`.
+function FileAnalysis::__remove_action%(file_id: string, args: any%): bool
 	%{
 	using file_analysis::FileID;
 	using BifType::Record::FileAnalysis::ActionArgs;
@@ -88,39 +105,45 @@ function FileAnalysis::remove_action%(file_id: string, args: any%): bool
 	return new Val(result, TYPE_BOOL);
 	%}
 
-function FileAnalysis::stop%(file_id: string%): bool
+## :bro:see:`FileAnalysis::stop`.
+function FileAnalysis::__stop%(file_id: string%): bool
 	%{
 	using file_analysis::FileID;
 	bool result = file_mgr->IgnoreFile(FileID(file_id->CheckString()));
 	return new Val(result, TYPE_BOOL);
 	%}
 
-function FileAnalysis::data_stream%(source: string, data: string%): any
+## :bro:see:`FileAnalysis::data_stream`.
+function FileAnalysis::__data_stream%(source: string, data: string%): any
 	%{
 	file_mgr->DataIn(data->Bytes(), data->Len(), source->CheckString());
 	return 0;
 	%}
 
-function FileAnalysis::data_chunk%(source: string, data: string,
+## :bro:see:`FileAnalysis::data_chunk`.
+function FileAnalysis::__data_chunk%(source: string, data: string,
                                      offset: count%): any
 	%{
 	file_mgr->DataIn(data->Bytes(), data->Len(), offset, source->CheckString());
 	return 0;
 	%}
 
-function FileAnalysis::gap%(source: string, offset: count, len: count%): any
+## :bro:see:`FileAnalysis::gap`.
+function FileAnalysis::__gap%(source: string, offset: count, len: count%): any
 	%{
 	file_mgr->Gap(offset, len, source->CheckString());
 	return 0;
 	%}
 
-function FileAnalysis::set_size%(source: string, size: count%): any
+## :bro:see:`FileAnalysis::set_size`.
+function FileAnalysis::__set_size%(source: string, size: count%): any
 	%{
 	file_mgr->SetSize(size, source->CheckString());
 	return 0;
 	%}
 
-function FileAnalysis::eof%(source: string%): any
+## :bro:see:`FileAnalysis::eof`.
+function FileAnalysis::__eof%(source: string%): any
 	%{
 	file_mgr->EndOfFile(source->CheckString());
 	return 0;

src/file_analysis/ActionSet.cc

@@ -11,13 +11,11 @@ using namespace file_analysis;
 
 // keep in order w/ declared enum values in file_analysis.bif
 static ActionInstantiator action_factory[] = {
-	Extract::Instantiate,
+	file_analysis::Extract::Instantiate,
-	MD5::Instantiate,
+	file_analysis::MD5::Instantiate,
-	SHA1::Instantiate,
+	file_analysis::SHA1::Instantiate,
-	SHA256::Instantiate,
+	file_analysis::SHA256::Instantiate,
-	DataEvent::Instantiate,
+	file_analysis::DataEvent::Instantiate,
-
-	PE_Analyzer::Instantiate,
 };
 
 static void action_del_func(void* v)

src/file_analysis/ActionSet.h

@@ -13,6 +13,11 @@ namespace file_analysis {
 class Info;
 declare(PDict,Action);
 
+/**
+ * A set of file analysis actions indexed by ActionArgs.  Allows queueing
+ * of addition/removals so that those modifications can happen at well-defined
+ * times (e.g. to make sure a loop iterator isn't invalidated).
+ */
 class ActionSet {
 public:
 

src/file_analysis/Info.cc

@@ -81,7 +81,8 @@ void Info::StaticInit()
 
 Info::Info(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
     : file_id(""), unique(unique), val(0), postpone_timeout(false),
-      need_reassembly(false), done(false), actions(this)
+      first_chunk(true), need_type(false), need_reassembly(false), done(false),
+      actions(this)
 	{
 	StaticInit();
 
@@ -134,11 +135,23 @@ void Info::UpdateConnectionFields(Connection* conn)
 
 	Val* conns = val->Lookup(conns_idx);
 
+	bool is_first = false;
+
 	if ( ! conns )
+		{
+		is_first = true;
 		val->Assign(conns_idx, conns = empty_connection_table());
+		}
 
 	Val* idx = get_conn_id_val(conn);
+	if ( ! conns->AsTableVal()->Lookup(idx) )
+		{
 		conns->AsTableVal()->Assign(idx, conn->BuildConnVal());
+		if ( ! is_first )
+			file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW_CONN,
+			                         this);
+		}
+
 	Unref(idx);
 	}
 
@@ -162,7 +175,7 @@ int Info::Idx(const string& field)
 	{
 	int rval = BifType::Record::FileAnalysis::Info->FieldOffset(field.c_str());
 	if ( rval < 0 )
-		reporter->InternalError("Unkown FileAnalysis::Info field: %s",
+		reporter->InternalError("Unknown FileAnalysis::Info field: %s",
 		                        field.c_str());
 	return rval;
 	}
@@ -172,6 +185,13 @@ double Info::GetTimeoutInterval() const
 	return LookupFieldDefaultInterval(timeout_interval_idx);
 	}
 
+string Info::GetSource() const
+	{
+	Val* v = val->Lookup(source_idx);
+	if ( ! v ) return "";
+	return v->AsStringVal()->CheckString();
+	}
+
 RecordVal* Info::GetResults(RecordVal* args) const
 	{
 	TableVal* actions_table = val->Lookup(actions_idx)->AsTableVal();
@@ -230,18 +250,6 @@ bool Info::BufferBOF(const u_char* data, uint64 len)
 
 	uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx);
 
-	/* Leaving out this optimization (I think) for now to keep things simpler.
-	// If first chunk satisfies desired size, do everything now without copying.
-	if ( bof_buffer.chunks.empty() && len >= desired_size )
-		{
-		bof_buffer.full = bof_buffer.replayed = true;
-		val->Assign(bof_buffer_idx, new StringVal(new BroString(data, len, 0)));
-		file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
-		// TODO: libmagic stuff
-		return false;
-		}
-	*/
-
 	bof_buffer.chunks.push_back(new BroString(data, len, 0));
 	bof_buffer.size += len;
 
@@ -254,18 +262,10 @@ bool Info::BufferBOF(const u_char* data, uint64 len)
 	return true;
 	}
 
-void Info::ReplayBOF()
+bool Info::DetectTypes(const u_char* data, uint64 len)
 	{
-	if ( bof_buffer.replayed ) return;
+	const char* desc = bro_magic_buffer(magic, data, len);
-	bof_buffer.replayed = true;
+	const char* mime = bro_magic_buffer(magic_mime, data, len);
-
-	if ( bof_buffer.chunks.empty() ) return;
-
-	BroString* bs = concatenate(bof_buffer.chunks);
-	const char* desc = bro_magic_buffer(magic, bs->Bytes(), bs->Len());
-	const char* mime = bro_magic_buffer(magic_mime, bs->Bytes(), bs->Len());
-
-	val->Assign(bof_buffer_idx, new StringVal(bs));
 
 	if ( desc )
 		val->Assign(file_type_idx, new StringVal(desc));
@@ -273,10 +273,29 @@ void Info::ReplayBOF()
 	if ( mime )
 		val->Assign(mime_type_idx, new StringVal(mime));
 
+	return desc || mime;
+	}
+
+void Info::ReplayBOF()
+	{
+	if ( bof_buffer.replayed ) return;
+	bof_buffer.replayed = true;
+
+	if ( bof_buffer.chunks.empty() )
+		{
+		// Since we missed the beginning, try file type detect on next data in.
+		need_type = true;
+		return;
+		}
+
+	BroString* bs = concatenate(bof_buffer.chunks);
+	val->Assign(bof_buffer_idx, new StringVal(bs));
+	bool have_type = DetectTypes(bs->Bytes(), bs->Len());
+
 	using BifEnum::FileAnalysis::TRIGGER_BOF_BUFFER;
 	file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
 
-	if ( desc || mime )
+	if ( have_type )
 		file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
 
 	for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i )
@@ -286,7 +305,17 @@ void Info::ReplayBOF()
 void Info::DataIn(const u_char* data, uint64 len, uint64 offset)
 	{
 	actions.DrainModifications();
-	// TODO: attempt libmagic stuff here before doing reassembly?
+
+	if ( first_chunk )
+		{
+		if ( DetectTypes(data, len) )
+			{
+			file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
+			actions.DrainModifications();
+			}
+
+		first_chunk = false;
+		}
 
 	Action* act = 0;
 	IterCookie* c = actions.InitForIteration();
@@ -316,6 +345,17 @@ void Info::DataIn(const u_char* data, uint64 len)
 
 	if ( BufferBOF(data, len) ) return;
 
+	if ( need_type )
+		{
+		if ( DetectTypes(data, len) )
+			{
+			file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
+			actions.DrainModifications();
+			}
+
+		need_type = false;
+		}
+
 	Action* act = 0;
 	IterCookie* c = actions.InitForIteration();
 

src/file_analysis/Info.h

@@ -32,6 +32,11 @@ public:
 	 */
 	double GetTimeoutInterval() const;
 
+	/**
+	 * @return value of the "source" field from #val record.
+	 */
+	string GetSource() const;
+
 	/**
 	 * @return value of the "file_id" field from #val record.
 	 */
@@ -155,10 +160,19 @@ protected:
 	 */
 	void ReplayBOF();
 
+	/**
+	 * Does file/mime type detection and assigns types (if available) to
+	 * corresponding fields in #val.
+	 * @return whether a file or mime type was available.
+	 */
+	bool DetectTypes(const u_char* data, uint64 len);
+
 	FileID file_id;            /**< A pretty hash that likely identifies file*/
 	string unique;             /**< A string that uniquely identifies file */
 	RecordVal* val;            /**< \c FileAnalysis::Info from script layer. */
 	bool postpone_timeout;     /**< Whether postponing timeout is requested. */
+	bool first_chunk;          /**< Track first non-linear chunk. */
+	bool need_type;            /**< Flags next data input to be magic typed. */
 	bool need_reassembly;      /**< Whether file stream reassembly is needed. */
 	bool done;                 /**< If this object is about to be deleted. */
 	ActionSet actions;

src/file_analysis/Manager.cc

@@ -34,12 +34,18 @@ void Manager::ReceiveHandle(const string& handle)
 	if ( pending.empty() )
 		reporter->InternalError("File analysis underflow");
 
+	int use_count = cache.front();
+	cache.pop();
+
+	for ( int i = 0; i < use_count; ++i )
+		{
 		PendingFile* pf = pending.front();
 		if ( ! handle.empty() )
 			pf->Finish(handle);
 		delete pf;
 		pending.pop();
 		}
+	}
 
 void Manager::EventDrainDone()
 	{
@@ -330,11 +336,50 @@ bool Manager::IsDisabled(AnalyzerTag::Tag tag)
 	return rval;
 	}
 
+static bool CheckArgEquality(AnalyzerTag::Tag tag, Connection* conn,
+                             bool is_orig, val_list* other_args)
+	{
+	if ( ! other_args ) return false;
+	if ( (*other_args)[0]->AsCount() != (bro_uint_t) tag ) return false;
+	if ( (*other_args)[2]->AsBool() != is_orig ) return false;
+
+	RecordVal* id = (*other_args)[1]->AsRecordVal()->Lookup(
+	        connection_type->FieldOffset("id"))->AsRecordVal();
+
+	PortVal* orig_p = id->Lookup(
+	        conn_id->FieldOffset("orig_p"))->AsPortVal();
+
+	if ( orig_p->Port() != ntohs(conn->OrigPort()) ) return false;
+	if ( orig_p->PortType() != conn->ConnTransport() ) return false;
+
+	PortVal* resp_p = id->Lookup(
+	        conn_id->FieldOffset("resp_p"))->AsPortVal();
+
+	if ( resp_p->Port() != ntohs(conn->RespPort()) ) return false;
+
+	if ( id->Lookup(conn_id->FieldOffset("orig_h"))->AsAddr() !=
+	     conn->OrigAddr() ) return false;
+
+	if ( id->Lookup(conn_id->FieldOffset("resp_h"))->AsAddr() !=
+	     conn->RespAddr() ) return false;
+
+	return true;
+	}
+
 bool Manager::QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
                                bool is_orig)
 	{
 	if ( ! get_file_handle ) return false;
 
+	if ( mgr.Tail() == get_file_handle &&
+	     CheckArgEquality(tag, conn, is_orig, mgr.TailArgs()) )
+		{
+		cache.front()++;
+		return true;
+		}
+
+	cache.push(1);
+
 	val_list* vl = new val_list();
 	vl->append(new Val(tag, TYPE_COUNT));
 	vl->append(conn->BuildConnVal());

src/file_analysis/Manager.h

@@ -130,6 +130,7 @@ protected:
 	typedef set<string> StrSet;
 	typedef map<FileID, Info*> IDMap;
 	typedef queue<PendingFile*> PendingQueue;
+	typedef queue<int> HandleCache;
 
 	/**
 	 * @return the Info object mapped to \a unique or a null pointer if analysis
@@ -164,22 +165,24 @@ protected:
 	 */
 	bool IsIgnored(const string& unique);
 
-	/**
-	 * @return whether file analysis is disabled for the given analyzer.
-	 */
-	static bool IsDisabled(AnalyzerTag::Tag tag);
-
 	/**
 	 * Queues \c get_file_handle event in order to retrieve unique file handle.
 	 * @return true if there is a handler for the event, else false.
 	 */
-	static bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
+	bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
 	                      bool is_orig);
 
+	/**
+	 * @return whether file analysis is disabled for the given analyzer.
+	 */
+	static bool IsDisabled(AnalyzerTag::Tag tag);
+
 	StrMap str_map; /**< Map unique strings to \c FileAnalysis::Info records. */
 	IDMap id_map;   /**< Map file IDs to \c FileAnalysis::Info records. */
 	StrSet ignored; /**< Ignored files.  Will be finally removed on EOF. */
 	PendingQueue pending; /**< Files awaiting a unique handle. */
+	HandleCache cache; /**< The number of times a received file handle can be
+	                        used to pop the #pending queue. */
 
 	static TableVal* disabled; /**< Table of disabled analyzers. */
 };

src/file_analysis/PendingFile.h

@@ -7,6 +7,10 @@
 
 namespace file_analysis {
 
+/**
+ * Provides buffering for file contents until the script-layer is able to
+ * return a unique file handle for it.
+ */
 class PendingFile {
 public:
 

testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-14-37-46
+#open	2013-03-28-21-35-15
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
-1333458850.375568	arKYeMETxOg	10.131.47.185	1923	79.101.110.141	80	1	GET	o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com	/videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1	http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf	Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11	0	56320	206	Partial Content	-	-	-	(empty)	-	-	-	-	-	-
+1333458850.375568	arKYeMETxOg	10.131.47.185	1923	79.101.110.141	80	1	GET	o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com	/videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1	http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf	Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11	0	56320	206	Partial Content	-	-	-	(empty)	-	-	-	application/octet-stream; charset=binary	-	-
-#close	2013-03-22-14-37-46
+#close	2013-03-28-21-35-15

testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout

@@ -1,5 +1,8 @@
 FileAnalysis::TRIGGER_NEW
 oDwT1BbzjM1, 0, 0
+FileAnalysis::TRIGGER_TYPE
+file type is set
+mime type is set
 FileAnalysis::TRIGGER_DONE
 oDwT1BbzjM1, 1022920, 0
 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
@@ -7,6 +10,9 @@ total bytes: 1022920
 source: HTTP
 FileAnalysis::TRIGGER_NEW
 oDwT1BbzjM1, 0, 0
+FileAnalysis::TRIGGER_TYPE
+file type is set
+mime type is set
 FileAnalysis::TRIGGER_TIMEOUT
 FileAnalysis::TRIGGER_TIMEOUT
 FileAnalysis::TRIGGER_EOF

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out

@@ -1,5 +1,9 @@
 FileAnalysis::TRIGGER_NEW
 7gZBKVUgy4l, 0, 0
+FileAnalysis::TRIGGER_TYPE
+file type is set
+mime type is set
+FileAnalysis::TRIGGER_NEW_CONN
 FileAnalysis::TRIGGER_DONE
 7gZBKVUgy4l, 555523, 0
 [orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp]

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out

@@ -1,5 +1,8 @@
 FileAnalysis::TRIGGER_NEW
 oDwT1BbzjM1, 0, 0
+FileAnalysis::TRIGGER_TYPE
+file type is set
+mime type is set
 FileAnalysis::TRIGGER_DONE
 oDwT1BbzjM1, 1022920, 0
 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
@@ -7,6 +10,9 @@ total bytes: 1022920
 source: HTTP
 FileAnalysis::TRIGGER_NEW
 oDwT1BbzjM1, 0, 0
+FileAnalysis::TRIGGER_TYPE
+file type is set
+mime type is set
 FileAnalysis::TRIGGER_TIMEOUT
 FileAnalysis::TRIGGER_EOF
 oDwT1BbzjM1, 206024, 0

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out

@@ -1,5 +1,9 @@
 FileAnalysis::TRIGGER_NEW
 uHS14uhRKGe, 0, 0
+FileAnalysis::TRIGGER_TYPE
+file type is set
+mime type is set
+FileAnalysis::TRIGGER_NEW_CONN
 FileAnalysis::TRIGGER_DONE
 uHS14uhRKGe, 498702, 0
 [orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp]

testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	file_analysis
-#open	2013-03-26-20-26-26
+#open	2013-03-29-18-28-57
-#fields	file_id	parent_file_id	source	last_active	seen_bytes	total_bytes	missing_bytes	overflow_bytes	timeout_interval	bof_buffer_size	file_type	mime_type	conn_uids	actions_taken	extracted_files	md5	sha1	sha256
+#fields	file_id	parent_file_id	source	last_active	seen_bytes	total_bytes	missing_bytes	overflow_bytes	timeout_interval	bof_buffer_size	file_type	mime_type	timedout	conn_uids	actions_taken	extracted_files	md5	sha1	sha256
-#types	string	string	string	time	count	count	count	count	interval	count	string	string	table[string]	table[enum]	table[string]	string	string	string
+#types	string	string	string	time	count	count	count	count	interval	count	string	string	bool	table[string]	table[enum]	table[string]	string	string	string
-Cx92a0ym5R8	-	HTTP	1362692527.009775	4705	4705	0	0	120.000000	1024	set	set	UWkUyAuUGXf	FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256	Cx92a0ym5R8-file	397168fd09991a0e712254df7bc639ac	1dd7ac0398df6cbc0696445a91ec681facf4dc47	4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
+Cx92a0ym5R8	-	HTTP	1362692527.009775	4705	4705	0	0	120.000000	1024	set	set	F	UWkUyAuUGXf	FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256	Cx92a0ym5R8-file	397168fd09991a0e712254df7bc639ac	1dd7ac0398df6cbc0696445a91ec681facf4dc47	4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
-#close	2013-03-26-20-26-26
+#close	2013-03-29-18-28-57

testing/btest/scripts/base/protocols/modbus/events.bro

@@ -8,141 +8,141 @@
 
 event modbus_message(c: connection, headers: ModbusHeaders, is_orig: bool)
 {
-    print "modbus_message", c, headers, is_orig;
+    print "modbus_message", c$id, headers, is_orig;
 }
 
 event modbus_exception(c: connection, headers: ModbusHeaders, code: count)
 {
-    print "modbus_exception", c, headers, code;
+    print "modbus_exception", c$id, headers, code;
 }
 
 event modbus_read_coils_request(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
 {
-    print "modbus_read_coils_request", c, headers, start_address, quantity;
+    print "modbus_read_coils_request", c$id, headers, start_address, quantity;
 }
 
 event modbus_read_coils_response(c: connection, headers: ModbusHeaders, coils: ModbusCoils)
 {
-    print "modbus_read_coils_response", c, headers, coils;
+    print "modbus_read_coils_response", c$id, headers, coils;
 }
 
 event modbus_read_discrete_inputs_request(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
 {
-    print "modbus_read_discrete_inputs_request", c, headers, start_address, quantity;
+    print "modbus_read_discrete_inputs_request", c$id, headers, start_address, quantity;
 }
 
 event modbus_read_discrete_inputs_response(c: connection, headers: ModbusHeaders, coils: ModbusCoils)
 {
-    print "modbus_read_discrete_inputs_response", c, headers, coils;
+    print "modbus_read_discrete_inputs_response", c$id, headers, coils;
 }
 
 event modbus_read_holding_registers_request(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
 {
-    print "modbus_read_holding_registers_request", c, headers, start_address, quantity;
+    print "modbus_read_holding_registers_request", c$id, headers, start_address, quantity;
 }
 
 event modbus_read_holding_registers_response(c: connection, headers: ModbusHeaders, registers: ModbusRegisters)
 {
-    print "modbus_read_holding_registers_response", c, headers, registers;
+    print "modbus_read_holding_registers_response", c$id, headers, registers;
 }
 
 event modbus_read_input_registers_request(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
 {
-    print "modbus_read_input_registers_request", c, headers, start_address, quantity;
+    print "modbus_read_input_registers_request", c$id, headers, start_address, quantity;
 }
 
 event modbus_read_input_registers_response(c: connection, headers: ModbusHeaders, registers: ModbusRegisters)
 {
-    print "modbus_read_input_registers_response", c, headers, registers;
+    print "modbus_read_input_registers_response", c$id, headers, registers;
 }
 
 event modbus_write_single_coil_request(c: connection, headers: ModbusHeaders, address: count, value: bool)
 {
-    print "modbus_write_single_coil_request", c, headers, address, value;
+    print "modbus_write_single_coil_request", c$id, headers, address, value;
 }
 
 event modbus_write_single_coil_response(c: connection, headers: ModbusHeaders, address: count, value: bool)
 {
-    print "modbus_write_single_coil_response", c, headers, address, value;
+    print "modbus_write_single_coil_response", c$id, headers, address, value;
 }
 
 event modbus_write_single_register_request(c: connection, headers: ModbusHeaders, address: count, value: count)
 {
-    print "modbus_write_single_register_request", c, headers, address, value;
+    print "modbus_write_single_register_request", c$id, headers, address, value;
 }
 
 event modbus_write_single_register_response(c: connection, headers: ModbusHeaders, address: count, value: count)
 {
-    print "modbus_write_single_register_response", c, headers, address, value;
+    print "modbus_write_single_register_response", c$id, headers, address, value;
 }
 
 event modbus_write_multiple_coils_request(c: connection, headers: ModbusHeaders, start_address: count, coils: ModbusCoils)
 {
-    print "modbus_write_multiple_coils_request", c, headers, start_address, coils;
+    print "modbus_write_multiple_coils_request", c$id, headers, start_address, coils;
 }
 
 event modbus_write_multiple_coils_response(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
 {
-    print "modbus_write_multiple_coils_response", c, headers, start_address, quantity;
+    print "modbus_write_multiple_coils_response", c$id, headers, start_address, quantity;
 }
 
 event modbus_write_multiple_registers_request(c: connection, headers: ModbusHeaders, start_address: count, registers: ModbusRegisters)
 {
-    print "modbus_write_multiple_registers_request", c, headers, start_address, registers;
+    print "modbus_write_multiple_registers_request", c$id, headers, start_address, registers;
 }
 
 event modbus_write_multiple_registers_response(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
 {
-    print "modbus_write_multiple_registers_response", c, headers, start_address, quantity;
+    print "modbus_write_multiple_registers_response", c$id, headers, start_address, quantity;
 }
 
 event modbus_read_file_record_request(c: connection, headers: ModbusHeaders)
 {
-    print "modbus_read_file_record_request", c, headers;
+    print "modbus_read_file_record_request", c$id, headers;
 }
 
 event modbus_read_file_record_response(c: connection, headers: ModbusHeaders)
 {
-    print "modbus_read_file_record_response", c, headers;
+    print "modbus_read_file_record_response", c$id, headers;
 }
 
 event modbus_write_file_record_request(c: connection, headers: ModbusHeaders)
 {
-    print "modbus_write_file_record_request", c, headers;
+    print "modbus_write_file_record_request", c$id, headers;
 }
 
 event modbus_write_file_record_response(c: connection, headers: ModbusHeaders)
 {
-    print "modbus_write_file_record_response", c, headers;
+    print "modbus_write_file_record_response", c$id, headers;
 }
 
 event modbus_mask_write_register_request(c: connection, headers: ModbusHeaders, address: count, and_mask: count, or_mask: count)
 {
-    print "modbus_mask_write_register_request", c, headers, address, and_mask, or_mask;
+    print "modbus_mask_write_register_request", c$id, headers, address, and_mask, or_mask;
 }
 
 event modbus_mask_write_register_response(c: connection, headers: ModbusHeaders, address: count, and_mask: count, or_mask: count)
 {
-    print "modbus_mask_write_register_response", c, headers, address, and_mask, or_mask;
+    print "modbus_mask_write_register_response", c$id, headers, address, and_mask, or_mask;
 }
 
 event modbus_read_write_multiple_registers_request(c: connection, headers: ModbusHeaders, read_start_address: count, read_quantity: count, write_start_address: count, write_registers: ModbusRegisters)
 {
-    print "modbus_read_write_multiple_registers_request", c, headers, read_start_address, read_quantity, write_start_address, write_registers;
+    print "modbus_read_write_multiple_registers_request", c$id, headers, read_start_address, read_quantity, write_start_address, write_registers;
 }
 
 event modbus_read_write_multiple_registers_response(c: connection, headers: ModbusHeaders, written_registers: ModbusRegisters)
 {
-    print "modbus_read_write_multiple_registers_response", c, headers, written_registers;
+    print "modbus_read_write_multiple_registers_response", c$id, headers, written_registers;
 }
 
 event modbus_read_fifo_queue_request(c: connection, headers: ModbusHeaders, start_address: count)
 {
-    print "modbus_read_fifo_queue_request", c, headers, start_address;
+    print "modbus_read_fifo_queue_request", c$id, headers, start_address;
 }
 
 event modbus_read_fifo_queue_response(c: connection, headers: ModbusHeaders, fifos: ModbusRegisters)
 {
-    print "modbus_read_fifo_queue_response", c, headers, fifos;
+    print "modbus_read_fifo_queue_response", c$id, headers, fifos;
 }
 

testing/scripts/diff-remove-mime-types

@@ -8,7 +8,7 @@ BEGIN { FS="\t"; OFS="\t"; type_col = -1; desc_col = -1 }
 /^#fields/ {
     for ( i = 2; i < NF; ++i )
         {
-        if ( $i == "mime_type" )
+        if ( $i == "mime_type" || $i == "file_type" )
             type_col = i-1;
         if ( $i == "mime_desc" )
             desc_col = i-1;