Merge remote-tracking branch 'origin/topic/jsiwek/faf-experimental' into topic/seth/file-analysis-exe-analyzer

Conflicts:
	src/file_analysis.bif

scripts/base/frameworks/file-analysis/main.bro

@@ -12,49 +12,40 @@ export {
 		LOG
 	};
 
-	## The default buffer size used for storing the beginning of files.
-	const default_bof_buffer_size: count = 1024 &redef;
-
-	## The default amount of time file analysis will wait for new file data
-	## before giving up.
-	const default_timeout_interval: interval = 2 mins &redef;
-
 	## A structure which represents a desired file analysis action to take.
 	type ActionArgs: record {
 		## The type of action.
 		act: Action;
+
 		## The local filename to which to write an extracted file.  Must be
 		## set when *act* is :bro:see:`FileAnalysis::ACTION_EXTRACT`.
 		extract_filename: string &optional;
-	};
 
-	## A structure which contains the results of certain file analysis actions.
+		## An event which will be generated for all new file contents,
-	type ActionResults: record {
+		## chunk-wise.
-		## An MD5 digest of the file contents.
+		chunk_event: event(f: fa_file, data: string, off: count) &optional;
-		md5: string &optional;
+
-		## An SHA1 digest of the file contents.
+		## An event which will be generated for all new file contents,
-		sha1: string &optional;
+		## stream-wise.
-		## An SHA256 digest of the file contents.
+		stream_event: event(f: fa_file, data: string) &optional;
-		sha256: string &optional;
+	} &redef;
-	};
 
 	## Contains all metadata related to the analysis of a given file.
+	## For the most part, fields here are derived from ones of the same name
+	## in :bro:see:`fa_file`.
 	type Info: record {
 		## An identifier associated with a single file.
-		file_id: string &log;
+		id: string &log;
 
 		## Identifier associated with a container file from which this one was
 		## extracted as part of the file analysis.
-		parent_file_id: string &log &optional;
+		parent_id: string &log &optional;
 
 		## An identification of the source of the file data.  E.g. it may be
 		## a network protocol over which it was transferred, or a local file
 		## path which was read, or some other input source.
 		source: string &log &optional;
 
-		## The set of connections over which the file was transferred.
-		conns: table[conn_id] of connection &optional;
-
 		## The time at which the last activity for the file was seen.
 		last_active: time &log;
 
@@ -75,15 +66,11 @@ export {
 
 		## The amount of time between receiving new data for this file that
 		## the analysis engine will wait before giving up on it.
-		timeout_interval: interval &log &default=default_timeout_interval;
+		timeout_interval: interval &log &optional;
 
 		## The number of bytes at the beginning of a file to save for later
 		## inspection in *bof_buffer* field.
-		bof_buffer_size: count &log &default=default_bof_buffer_size;
+		bof_buffer_size: count &log &optional;
-
-		## The content of the beginning of a file up to *bof_buffer_size* bytes.
-		## This is also the buffer that's used for file/mime type detection.
-		bof_buffer: string &optional;
 
 		## A file type provided by libmagic against the *bof_buffer*, or
 		## in the cases where no buffering of the beginning of file occurs,
@@ -95,25 +82,17 @@ export {
 		## an initial guess of the mime type based on the first data seen.
 		mime_type: string &log &optional;
 
-		## Actions that have been added to the analysis of this file.
-		## Only meant for inspection by user scripts, not direct modification.
-		actions: table[ActionArgs] of ActionResults;
-	} &redef;
-
-	## Fields that are derived from existing ones, and are set just in time
-	## for logging purposes.
-	redef record FileAnalysis::Info += {
 		## Whether the file analysis timed out at least once for the file.
 		timedout: bool &log &default=F;
 
 		## Connection UIDS over which the file was transferred.
-		conn_uids: set[string] &log &optional;
+		conn_uids: set[string] &log;
 
 		## A set of action types taken during the file analysis.
-		actions_taken: set[Action] &log &optional;
+		actions_taken: set[Action] &log;
 
 		## Local filenames of file extraction actions.
-		extracted_files: set[string] &log &optional;
+		extracted_files: set[string] &log;
 
 		## An MD5 digest of the file contents.
 		md5: string &log &optional;
@@ -123,25 +102,7 @@ export {
 
 		## A SHA256 digest of the file contents.
 		sha256: string &log &optional;
-	};
+	} &redef;
-
-	## Redefined here just so the *info* parameters of the events have the
-	## right type information.
-	redef record ActionArgs += {
-		## An event which will be generated for all new file contents,
-		## chunk-wise.
-		chunk_event: event(info: Info, data: string, off: count) &optional;
-
-		## An event which will be generated for all new file contents,
-		## stream-wise.
-		stream_event: event(info: Info, data: string) &optional;
-	};
-
-	## Evaluated every time a significant event occurs during the course of
-	## file analysis.  Fields of the *info* argument may be modified or
-	## other actions may be added or removed inside the body of any handlers
-	## of this hook.
-	global policy: hook(trig: Trigger, info: Info);
 
 	## A table that can be used to disable file analysis completely for
 	## any files transferred over given network protocol analyzers.
@@ -153,57 +114,52 @@ export {
 
 	## The salt concatenated to unique file handle strings generated by
 	## :bro:see:`get_file_handle` before hashing them in to a file id
-	## (the *file_id* field of :bro:see:`FileAnalysis::Info`).
+	## (the *id* field of :bro:see:`fa_file`).
 	## Provided to help mitigate the possiblility of manipulating parts of
 	## network connections that factor in to the file handle in order to
 	## generate two handles that would hash to the same file id.
 	const salt = "I recommend changing this." &redef;
 
 	## Postpones the timeout of file analysis for a given file.
-	## When used within a :bro:see:`FileAnalysis::policy` handler for
+	## When used within a :bro:see:`file_timeout` handler for, the analysis
-	## :bro:see:`FileAnalysis::TRIGGER_TIMEOUT`, the analysis will delay
+	## the analysis will delay timing out for the period of time indicated by
-	## timing out for the period of time indicated by the *timeout_interval*
+	## the *timeout_interval* field of :bro:see:`fa_file`.
-	## field of :bro:see:`FileAnalysis::Info`.
 	##
-	## file_id: the file identifier string from the *file_id* field of
+	## f: the file.
-	##          :bro:see:`FileAnalysis::Info`.
 	##
 	## Returns: true if the timeout will be postponed, or false if analysis
-	##          for the *file_id* isn't currently active.
+	##          for the *id* isn't currently active.
-	global postpone_timeout: function(file_id: string): bool;
+	global postpone_timeout: function(f: fa_file): bool;
 
 	## Adds an action to the analysis of a given file.
 	##
-	## file_id: the file identifier string from the *file_id* field of
+	## f: the file.
-	##          :bro:see:`FileAnalysis::Info`.
 	##
 	## args: the action type to add along with any arguments it takes.
 	##
 	## Returns: true if the action will be added, or false if analysis
-	##          for the *file_id* isn't currently active or the *args*
+	##          for the *id* isn't currently active or the *args*
 	##          were invalid for the action type.
-	global add_action: function(file_id: string, args: ActionArgs): bool;
+	global add_action: function(f: fa_file, args: ActionArgs): bool;
 
 	## Removes an action from the analysis of a given file.
 	##
-	## file_id: the file identifier string from the *file_id* field of
+	## f: the file.
-	##          :bro:see:`FileAnalysis::Info`.
 	##
 	## args: the action (type and args) to remove.
 	##
 	## Returns: true if the action will be removed, or false if analysis
-	##          for the *file_id* isn't currently active.
+	##          for the *id* isn't currently active.
-	global remove_action: function(file_id: string, args: ActionArgs): bool;
+	global remove_action: function(f: fa_file, args: ActionArgs): bool;
 
 	## Stops/ignores any further analysis of a given file.
 	##
-	## file_id: the file identifier string from the *file_id* field of
+	## f: the file.
-	##          :bro:see:`FileAnalysis::Info`.
 	##
 	## Returns: true if analysis for the given file will be ignored for the
-	##          rest of it's contents, or false if analysis for the *file_id*
+	##          rest of it's contents, or false if analysis for the *id*
 	##          isn't currently active.
-	global stop: function(file_id: string): bool;
+	global stop: function(f: fa_file): bool;
 
 	## Sends a sequential stream of data in for file analysis.
 	## Meant for use when providing external file analysis input (e.g.
@@ -258,24 +214,61 @@ export {
 	global eof: function(source: string);
 }
 
-function postpone_timeout(file_id: string): bool
+redef record fa_file += {
+	info: Info &optional;
+};
+
+function set_info(f: fa_file)
 	{
-	return __postpone_timeout(file_id);
+	if ( ! f?$info )
+		{
+		local tmp: Info;
+		f$info = tmp;
 		}
 
-function add_action(file_id: string, args: ActionArgs): bool
+	f$info$id = f$id;
-	{
+	if ( f?$parent_id ) f$info$parent_id = f$parent_id;
-	return __add_action(file_id, args);
+	if ( f?$source ) f$info$source = f$source;
+	f$info$last_active = f$last_active;
+	f$info$seen_bytes = f$seen_bytes;
+	if ( f?$total_bytes ) f$info$total_bytes = f$total_bytes;
+	f$info$missing_bytes = f$missing_bytes;
+	f$info$overflow_bytes = f$overflow_bytes;
+	f$info$timeout_interval = f$timeout_interval;
+	f$info$bof_buffer_size = f$bof_buffer_size;
+	if ( f?$file_type ) f$info$file_type = f$file_type;
+	if ( f?$mime_type ) f$info$mime_type = f$mime_type;
+	if ( f?$conns )
+		for ( cid in f$conns )
+			add f$info$conn_uids[f$conns[cid]$uid];
 	}
 
-function remove_action(file_id: string, args: ActionArgs): bool
+function postpone_timeout(f: fa_file): bool
 	{
-	return __remove_action(file_id, args);
+	return __postpone_timeout(f$id);
 	}
 
-function stop(file_id: string): bool
+function add_action(f: fa_file, args: ActionArgs): bool
 	{
-	return __stop(file_id);
+	if ( ! __add_action(f$id, args) ) return F;
+
+	set_info(f);
+	add f$info$actions_taken[args$act];
+
+	if ( args$act == FileAnalysis::ACTION_EXTRACT )
+		add f$info$extracted_files[args$extract_filename];
+
+	return T;
+	}
+
+function remove_action(f: fa_file, args: ActionArgs): bool
+	{
+	return __remove_action(f$id, args);
+	}
+
+function stop(f: fa_file): bool
+	{
+	return __stop(f$id);
 	}
 
 function data_stream(source: string, data: string)
@@ -309,53 +302,34 @@ event bro_init() &priority=5
 	                   [$columns=Info, $ev=log_file_analysis]);
 	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_timeout(f: fa_file) &priority=5
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
+	set_info(f);
-	info$timedout = T;
+	f$info$timedout = T;
 	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_hash(f: fa_file, kind: string, hash: string) &priority=5
-	&priority=-5
 	{
-	if ( trig != FileAnalysis::TRIGGER_EOF &&
+	set_info(f);
-	     trig != FileAnalysis::TRIGGER_DONE ) return;
+	switch ( kind ) {
+	case "md5":
+		f$info$md5 = hash;
+		break;
+	case "sha1":
+		f$info$sha1 = hash;
+		break;
+	case "sha256":
+		f$info$sha256 = hash;
+		break;
+	}
+	}
 
-	info$conn_uids = set();
+event file_state_remove(f: fa_file) &priority=5
-	if ( info?$conns )
-		for ( cid in info$conns )
-			add info$conn_uids[info$conns[cid]$uid];
-
-	info$actions_taken = set();
-	info$extracted_files = set();
-
-	for ( act in info$actions )
 	{
-		add info$actions_taken[act$act];
+	set_info(f);
-		local result: FileAnalysis::ActionResults = info$actions[act];
-
-		switch ( act$act ) {
-		case FileAnalysis::ACTION_EXTRACT:
-			add info$extracted_files[act$extract_filename];
-			break;
-		case FileAnalysis::ACTION_MD5:
-			if ( result?$md5 )
-				info$md5 = result$md5;
-			break;
-		case FileAnalysis::ACTION_SHA1:
-			if ( result?$sha1 )
-				info$sha1 = result$sha1;
-			break;
-		case FileAnalysis::ACTION_SHA256:
-			if ( result?$sha256 )
-				info$sha256 = result$sha256;
-			break;
-		case FileAnalysis::ACTION_DATA_EVENT:
-			# no direct result
-			break;
-		}
 	}
 
-	Log::write(FileAnalysis::LOG, info);
+event file_state_remove(f: fa_file) &priority=-5
+	{
+	Log::write(FileAnalysis::LOG, f$info);
 	}

scripts/base/init-bare.bro

@@ -316,6 +316,70 @@ type connection: record {
 	tunnel: EncapsulatingConnVector &optional;
 };
 
+const default_file_timeout_interval: interval = 2 mins &redef;
+const default_file_bof_buffer_size: count = 1024 &redef;
+
+## A file that Bro is analyzing.  This is Bro's type for describing the basic
+## internal metadata collected about a "file", which is essentially just a
+## byte stream that is e.g. pulled from a network connection or possibly
+## some other input source.
+type fa_file: record {
+	## An identifier associated with a single file.
+	id: string;
+
+	## Identifier associated with a container file from which this one was
+	## extracted as part of the file analysis.
+	parent_id: string &optional;
+
+	## An identification of the source of the file data.  E.g. it may be
+	## a network protocol over which it was transferred, or a local file
+	## path which was read, or some other input source.
+	source: string &optional;
+
+	## The set of connections over which the file was transferred.
+	conns: table[conn_id] of connection &optional;
+
+	## The time at which the last activity for the file was seen.
+	last_active: time;
+
+	## Number of bytes provided to the file analysis engine for the file.
+	seen_bytes: count &default=0;
+
+	## Total number of bytes that are supposed to comprise the full file.
+	total_bytes: count &optional;
+
+	## The number of bytes in the file stream that were completely missed
+	## during the process of analysis e.g. due to dropped packets.
+	missing_bytes: count &default=0;
+
+	## The number of not all-in-sequence bytes in the file stream that
+	## were delivered to file actions/analyzers due to reassembly buffer
+	## overflow.
+	overflow_bytes: count &default=0;
+
+	## The amount of time between receiving new data for this file that
+	## the analysis engine will wait before giving up on it.
+	timeout_interval: interval &default=default_file_timeout_interval;
+
+	## The number of bytes at the beginning of a file to save for later
+	## inspection in *bof_buffer* field.
+	bof_buffer_size: count &default=default_file_bof_buffer_size;
+
+	## The content of the beginning of a file up to *bof_buffer_size* bytes.
+	## This is also the buffer that's used for file/mime type detection.
+	bof_buffer: string &optional;
+
+	## A file type provided by libmagic against the *bof_buffer*, or
+	## in the cases where no buffering of the beginning of file occurs,
+	## an initial guess of the file type based on the first data seen.
+	file_type: string &optional;
+
+	## A mime type provided by libmagic against the *bof_buffer*, or
+	## in the cases where no buffering of the beginning of file occurs,
+	## an initial guess of the mime type based on the first data seen.
+	mime_type: string &optional;
+} &redef;
+
 ## Fields of a SYN packet.
 ##
 ## .. bro:see:: connection_SYN_packet

scripts/base/protocols/ftp/file-analysis.bro

@@ -5,22 +5,21 @@
 module FTP;
 
 export {
-	## Determines whether the default :bro:see:`get_file_handle` handler
-	## is used to return file handles to the file analysis framework.
-	## Redefine to true in order to provide a custom handler which overrides
-	## the default for FTP.
-	const disable_default_file_handle_provider: bool = F &redef;
-
 	## Default file handle provider for FTP.
-	function get_file_handle(c: connection, is_orig: bool): string
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+}
+
+function get_handle_string(c: connection): string
+	{
+	return fmt("%s %s %s", ANALYZER_FTP_DATA, c$start_time, id_string(c$id));
+	}
+
+function get_file_handle(c: connection, is_orig: bool): string
 	{
 	if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) return "";
 
 	local info: FTP::Info = ftp_data_expected[c$id$resp_h, c$id$resp_p];
 
-		local rval = fmt("%s %s %s", ANALYZER_FTP_DATA, c$start_time,
-		                 id_string(c$id));
-
 	if ( info$passive )
 		# FTP client initiates data channel.
 		if ( is_orig )
@@ -28,23 +27,21 @@ export {
 			return "";
 		else
 			# Do care about FTP server data.
-				return rval;
+			return get_handle_string(c);
 	else
 		# FTP server initiates dta channel.
 		if ( is_orig )
 			# Do care about FTP server data.
-				return rval;
+			return get_handle_string(c);
 		else
 			# Don't care about FTP client data.
 			return "";
 	}
-}
 
 module GLOBAL;
 
 event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
 	{
 	if ( tag != ANALYZER_FTP_DATA ) return;
-	if ( FTP::disable_default_file_handle_provider ) return;
+	set_file_handle(FTP::get_file_handle(c, is_orig));
-	return_file_handle(FTP::get_file_handle(c, is_orig));
 	}

scripts/base/protocols/ftp/file-extract.bro

@@ -24,21 +24,30 @@ redef record Info += {
 	extract_file:          bool &default=F;
 };
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+function get_extraction_name(f: fa_file): string
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_NEW ) return;
+	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
-	if ( ! info?$source ) return;
+	++extract_count;
-	if ( info$source != "FTP_DATA" ) return;
+	return r;
-	if ( ! info?$conns ) return;
+	}
 
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
+event file_new(f: fa_file) &priority=5
-	                          extract_count);
-	local extracting: bool = F;
-
-	for ( cid in info$conns )
 	{
-		local c: connection = info$conns[cid];
+	if ( ! f?$source ) return;
+	if ( f$source != "FTP_DATA" ) return;
+
+	if ( f?$mime_type && extract_file_types in f$mime_type )
+		{
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
+		                             $extract_filename=get_extraction_name(f)]);
+		return;
+		}
+
+	if ( ! f?$conns ) return;
+
+	for ( cid in f$conns )
+		{
+		local c: connection = f$conns[cid];
 
 		if ( [cid$resp_h, cid$resp_p] !in ftp_data_expected ) next;
 
@@ -46,59 +55,31 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
 
 		if ( ! s$extract_file ) next;
 
-		if ( ! extracting )
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
+		                             $extract_filename=get_extraction_name(f)]);
+		return;
+		}
+	}
+
+event file_state_remove(f: fa_file) &priority=4
 	{
-			FileAnalysis::add_action(info$file_id,
+	if ( ! f?$source ) return;
-			                         [$act=FileAnalysis::ACTION_EXTRACT,
+	if ( f$source != "FTP_DATA" ) return;
-			                          $extract_filename=fname]);
+	if ( ! f?$info ) return;
-			extracting = T;
-			++extract_count;
-			}
-		}
-	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+	for ( filename in f$info$extracted_files )
-	&priority=5
-	{
-	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
-	if ( ! info?$mime_type ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "FTP_DATA" ) return;
-	if ( extract_file_types !in info$mime_type ) return;
-
-	for ( act in info$actions )
-		if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
-
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
-	                          extract_count);
-	++extract_count;
-	FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
-	                                        $extract_filename=fname]);
-	}
-
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
-	&priority=-5
-	{
-	if ( trig != FileAnalysis::TRIGGER_EOF &&
-	     trig != FileAnalysis::TRIGGER_DONE ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "FTP_DATA" ) return;
-
-	for ( act in info$actions )
-		if ( act$act == FileAnalysis::ACTION_EXTRACT )
 		{
 		local s: FTP::Info;
 		s$ts = network_time();
 		s$tags = set();
 		s$user = "<ftp-data>";
-			s$extraction_file = act$extract_filename;
+		s$extraction_file = filename;
 
-			if ( info?$conns )
+		if ( f?$conns )
-				for ( cid in info$conns )
+			for ( cid in f$conns )
 				{
-					s$uid = info$conns[cid]$uid;
+				s$uid = f$conns[cid]$uid;
 				s$id = cid;
-					break;
 				}
 
 		Log::write(FTP::LOG, s);

scripts/base/protocols/http/file-analysis.bro

@@ -6,14 +6,11 @@
 module HTTP;
 
 export {
-	## Determines whether the default :bro:see:`get_file_handle` handler
-	## is used to return file handles to the file analysis framework.
-	## Redefine to true in order to provide a custom handler which overrides
-	## the default HTTP.
-	const disable_default_file_handle_provider: bool = F &redef;
-
 	## Default file handle provider for HTTP.
-	function get_file_handle(c: connection, is_orig: bool): string
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
 	{
 	if ( ! c?$http ) return "";
 
@@ -24,13 +21,11 @@ export {
 	return fmt("%s %s %s %s %s", ANALYZER_HTTP, c$start_time, is_orig,
 	           c$http$trans_depth, id_string(c$id));
 	}
-}
 
 module GLOBAL;
 
 event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
 	{
 	if ( tag != ANALYZER_HTTP ) return;
-	if ( HTTP::disable_default_file_handle_provider ) return;
+	set_file_handle(HTTP::get_file_handle(c, is_orig));
-	return_file_handle(HTTP::get_file_handle(c, is_orig));
 	}

scripts/base/protocols/http/file-extract.bro

@@ -18,74 +18,67 @@ export {
 		extraction_file:  string &log &optional;
 		
 		## Indicates if the response body is to be extracted or not.  Must be 
-		## set before or by the first :bro:enum:`FileAnalysis::TRIGGER_NEW`
+		## set before or by the first :bro:see:`file_new` for the file content.
-		## for the file content.
 		extract_file:     bool &default=F;
 	};
 }
 
 global extract_count: count = 0;
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+function get_extraction_name(f: fa_file): string
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
+	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
-	if ( ! info?$mime_type ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "HTTP" ) return;
-	if ( extract_file_types !in info$mime_type ) return;
-
-	for ( act in info$actions )
-		if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
-
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
-	                          extract_count);
 	++extract_count;
-	FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
+	return r;
+	}
+
+event file_new(f: fa_file) &priority=5
+	{
+	if ( ! f?$source ) return;
+	if ( f$source != "HTTP" ) return;
+	if ( ! f?$conns ) return;
+
+	local fname: string;
+	local c: connection;
+
+	if ( f?$mime_type && extract_file_types in f$mime_type )
+		{
+		fname = get_extraction_name(f);
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
 		                             $extract_filename=fname]);
 
-	if ( ! info?$conns ) return;
+		for ( cid in f$conns )
-
-	for ( cid in info$conns )
 			{
-		local c: connection = info$conns[cid];
+			c = f$conns[cid];
-
 			if ( ! c?$http ) next;
-
 			c$http$extraction_file = fname;
 			}
+
+		return;
 		}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
-	&priority=5
-	{
-	if ( trig != FileAnalysis::TRIGGER_NEW ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "HTTP" ) return;
-	if ( ! info?$conns ) return;
-
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
-	                          extract_count);
 	local extracting: bool = F;
 
-	for ( cid in info$conns )
+	for ( cid in f$conns )
 		{
-		local c: connection = info$conns[cid];
+		c = f$conns[cid];
 
 		if ( ! c?$http ) next;
 
-		if ( c$http$extract_file )
+		if ( ! c$http$extract_file ) next;
-			{
+
-			if ( ! extracting )
+		fname = get_extraction_name(f);
-				{
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
-				FileAnalysis::add_action(info$file_id,
-				                         [$act=FileAnalysis::ACTION_EXTRACT,
 		                             $extract_filename=fname]);
 		extracting = T;
-				++extract_count;
+		break;
 		}
 
+	if ( extracting )
+		for ( cid in f$conns )
+			{
+			c = f$conns[cid];
+			if ( ! c?$http ) next;
 			c$http$extraction_file = fname;
 			}
 	}
-	}

scripts/base/protocols/http/file-hash.bro

@@ -23,67 +23,46 @@ export {
 	                   &redef;
 }
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_new(f: fa_file) &priority=5
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
+	if ( ! f?$source ) return;
-	if ( ! info?$mime_type ) return;
+	if ( f$source != "HTTP" ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "HTTP" ) return;
 
-	if ( generate_md5 in info$mime_type )
+	if ( f?$mime_type && generate_md5 in f$mime_type )
-		FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]);
-	else if ( info?$conns )
 		{
-		for ( cid in info$conns )
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
+		return;
+		}
+
+	if ( ! f?$conns ) return;
+
+	for ( cid in f$conns )
 		{
-			local c: connection = info$conns[cid];
+		local c: connection = f$conns[cid];
 
 		if ( ! c?$http ) next;
 
-			if ( c$http$calc_md5 )
+		if ( ! c$http$calc_md5 ) next;
-				{
+
-				FileAnalysis::add_action(info$file_id,
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
-				                         [$act=FileAnalysis::ACTION_MD5]);
 		return;
 		}
 	}
-		}
-	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_state_remove(f: fa_file) &priority=4
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_DONE &&
+	if ( ! f?$source ) return;
-	     trig != FileAnalysis::TRIGGER_EOF ) return;
+	if ( f$source != "HTTP" ) return;
-	if ( ! info?$source ) return;
+	if ( ! f?$conns ) return;
-	if ( info$source != "HTTP" ) return;
+	if ( ! f?$info ) return;
-	if ( ! info?$conns ) return;
+	if ( ! f$info?$md5 ) return;
 
-	local act: FileAnalysis::ActionArgs = [$act=FileAnalysis::ACTION_MD5];
+	for ( cid in f$conns )
-
-	if ( act !in info$actions ) return;
-
-	local result = info$actions[act];
-
-	if ( ! result?$md5 ) return;
-
-	for ( cid in info$conns )
 		{
-		local c: connection = info$conns[cid];
+		local c: connection = f$conns[cid];
 
 		if ( ! c?$http ) next;
 
-		c$http$md5 = result$md5;
+		c$http$md5 = f$info$md5;
 		}
 	}
-
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
-	&priority=5
-	{
-	if ( trig != FileAnalysis::TRIGGER_GAP ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "HTTP" ) return;
-
-	FileAnalysis::remove_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]);
-	}

scripts/base/protocols/http/file-ident.bro

@@ -34,24 +34,22 @@ export {
 	const ignored_incorrect_file_type_urls = /^$/ &redef;
 }
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_new(f: fa_file) &priority=5
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
+	if ( ! f?$source ) return;
-	if ( ! info?$mime_type ) return;
+	if ( f$source != "HTTP" ) return;
-	if ( ! info?$source ) return;
+	if ( ! f?$mime_type ) return;
-	if ( info$source != "HTTP" ) return;
+	if ( ! f?$conns ) return;
-	if ( ! info?$conns ) return;
 
-	for ( cid in info$conns )
+	for ( cid in f$conns )
 		{
-		local c: connection = info$conns[cid];
+		local c: connection = f$conns[cid];
 
 		if ( ! c?$http ) next;
 
-		c$http$mime_type = info$mime_type;
+		c$http$mime_type = f$mime_type;
 
-		local mime_str: string = split1(info$mime_type, /;/)[1];
+		local mime_str: string = split1(f$mime_type, /;/)[1];
 
 		if ( mime_str !in mime_types_extensions ) next;
 		if ( ! c$http?$uri ) next;
@@ -68,26 +66,22 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
 		}
 	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_over_new_connection(f: fa_file) &priority=5
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_NEW_CONN ) return;
+	if ( ! f?$source ) return;
-	if ( ! info?$mime_type ) return;
+	if ( f$source != "HTTP" ) return;
-	if ( ! info?$source ) return;
+	if ( ! f?$mime_type ) return;
-	if ( info$source != "HTTP" ) return;
+	if ( ! f?$conns ) return;
-	if ( ! info?$conns ) return;
 
-	# Spread the mime around (e.g. for partial content, TRIGGER_TYPE only
+	# Spread the mime around (e.g. for partial content, file_type event only
 	# happens once for the first connection, but if there's subsequent
 	# connections to transfer the same file, they'll be lacking the mime_type
 	# field if we don't do this).
-	for ( cid in info$conns )
+	for ( cid in f$conns )
 		{
-		local c: connection = info$conns[cid];
+		local c: connection = f$conns[cid];
-
 		if ( ! c?$http ) next;
-
+		c$http$mime_type = f$mime_type;
-		c$http$mime_type = info$mime_type;
 		}
 	}
 

scripts/base/protocols/irc/dcc-send.bro

@@ -41,64 +41,29 @@ global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins;
 
 global extract_count: count = 0;
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+function set_dcc_mime(f: fa_file)
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_NEW ) return;
+	if ( ! f?$conns ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "IRC_DATA" ) return;
-	if ( ! info?$conns ) return;
 
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
+	for ( cid in f$conns )
-	                          extract_count);
-	local extracting: bool = F;
-
-	for ( cid in info$conns )
 		{
-		local c: connection = info$conns[cid];
+		local c: connection = f$conns[cid];
 
 		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
 
 		local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
 
-		if ( ! s$extract_file ) next;
+		s$dcc_mime_type = f$mime_type;
-
-		if ( ! extracting )
-			{
-			FileAnalysis::add_action(info$file_id,
-			                         [$act=FileAnalysis::ACTION_EXTRACT,
-			                          $extract_filename=fname]);
-			extracting = T;
-			++extract_count;
-			}
-
-		s$extraction_file = fname;
 		}
 	}
 
-function set_dcc_mime(info: FileAnalysis::Info)
+function set_dcc_extraction_file(f: fa_file, filename: string)
 	{
-	if ( ! info?$conns ) return;
+	if ( ! f?$conns ) return;
 
-	for ( cid in info$conns )
+	for ( cid in f$conns )
 		{
-		local c: connection = info$conns[cid];
+		local c: connection = f$conns[cid];
-
-		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
-
-		local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
-
-		s$dcc_mime_type = info$mime_type;
-		}
-	}
-
-function set_dcc_extraction_file(info: FileAnalysis::Info, filename: string)
-	{
-	if ( ! info?$conns ) return;
-
-	for ( cid in info$conns )
-		{
-		local c: connection = info$conns[cid];
 
 		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
 
@@ -108,13 +73,67 @@ function set_dcc_extraction_file(info: FileAnalysis::Info, filename: string)
 		}
 	}
 
-function log_dcc(info: FileAnalysis::Info)
+function get_extraction_name(f: fa_file): string
 	{
-	if ( ! info?$conns ) return;
+	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
+	++extract_count;
+	return r;
+	}
 
-	for ( cid in info$conns )
+# this handler sets the IRC::Info mime type
+event file_new(f: fa_file) &priority=5
 	{
-		local c: connection = info$conns[cid];
+	if ( ! f?$source ) return;
+	if ( f$source != "IRC_DATA" ) return;
+	if ( ! f?$mime_type ) return;
+
+	set_dcc_mime(f);
+	}
+
+# this handler check if file extraction is desired
+event file_new(f: fa_file) &priority=5
+	{
+	if ( ! f?$source ) return;
+	if ( f$source != "IRC_DATA" ) return;
+
+	local fname: string;
+
+	if ( f?$mime_type && extract_file_types in f$mime_type )
+		{
+		fname = get_extraction_name(f);
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
+		                             $extract_filename=fname]);
+		set_dcc_extraction_file(f, fname);
+		return;
+		}
+
+	if ( ! f?$conns ) return;
+
+	for ( cid in f$conns )
+		{
+		local c: connection = f$conns[cid];
+
+		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
+
+		local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
+
+		if ( ! s$extract_file ) next;
+
+		fname = get_extraction_name(f);
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
+		                             $extract_filename=fname]);
+		s$extraction_file = fname;
+		return;
+		}
+	}
+
+function log_dcc(f: fa_file)
+	{
+	if ( ! f?$conns ) return;
+
+	for ( cid in f$conns )
+		{
+		local c: connection = f$conns[cid];
 
 		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
 
@@ -137,37 +156,12 @@ function log_dcc(info: FileAnalysis::Info)
 		}
 	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_new(f: fa_file) &priority=-5
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
+	if ( ! f?$source ) return;
-	if ( ! info?$mime_type ) return;
+	if ( f$source != "IRC_DATA" ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "IRC_DATA" ) return;
 
-	set_dcc_mime(info);
+	log_dcc(f);
-
-	if ( extract_file_types !in info$mime_type ) return;
-
-	for ( act in info$actions )
-		if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
-
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
-	                          extract_count);
-	++extract_count;
-	FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
-	                                        $extract_filename=fname]);
-	set_dcc_extraction_file(info, fname);
-	}
-
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
-	&priority=-5
-	{
-	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "IRC_DATA" ) return;
-
-	log_dcc(info);
 	}
 
 event irc_dcc_message(c: connection, is_orig: bool,

scripts/base/protocols/irc/file-analysis.bro

@@ -5,26 +5,20 @@
 module IRC;
 
 export {
-	## Determines whether the default :bro:see:`get_file_handle` handler
-	## is used to return file handles to the file analysis framework.
-	## Redefine to true in order to provide a custom handler which overrides
-	## the default for IRC.
-	const disable_default_file_handle_provider: bool = F &redef;
-
 	## Default file handle provider for IRC.
-	function get_file_handle(c: connection, is_orig: bool): string
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
 	{
 	if ( is_orig ) return "";
-		return fmt("%s %s %s", ANALYZER_IRC_DATA, c$start_time,
+	return fmt("%s %s %s", ANALYZER_IRC_DATA, c$start_time, id_string(c$id));
-		           id_string(c$id));
 	}
-}
 
 module GLOBAL;
 
 event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
 	{
 	if ( tag != ANALYZER_IRC_DATA ) return;
-	if ( IRC::disable_default_file_handle_provider ) return;
+	set_file_handle(IRC::get_file_handle(c, is_orig));
-	return_file_handle(IRC::get_file_handle(c, is_orig));
 	}

scripts/base/protocols/smtp/entities-excerpt.bro

@@ -12,38 +12,26 @@ export {
 	};
 	
 	## This is the default value for how much of the entity body should be
-	## included for all MIME entities.
+	## included for all MIME entities.  The lesser of this value and
+	## :bro:see:`default_file_bof_buffer_size` will be used.
 	const default_entity_excerpt_len = 0 &redef;
 }
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_new(f: fa_file) &priority=5
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_NEW ) return;
+	if ( ! f?$source ) return;
-	if ( ! info?$source ) return;
+	if ( f$source != "SMTP" ) return;
-	if ( info$source != "SMTP" ) return;
+	if ( ! f?$bof_buffer ) return;
+	if ( ! f?$conns ) return;
 
-	if ( default_entity_excerpt_len > info$bof_buffer_size )
+	for ( cid in f$conns )
-		info$bof_buffer_size = default_entity_excerpt_len;
-	}
-
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
-	&priority=5
 		{
-	if ( trig != FileAnalysis::TRIGGER_BOF_BUFFER ) return;
+		local c: connection = f$conns[cid];
-	if ( ! info?$bof_buffer ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "SMTP" ) return;
-	if ( ! info?$conns ) return;
-
-	for ( cid in info$conns )
-		{
-		local c: connection = info$conns[cid];
 
 		if ( ! c?$smtp ) next;
 
 		if ( default_entity_excerpt_len > 0 )
 			c$smtp$current_entity$excerpt =
-			        info$bof_buffer[0:default_entity_excerpt_len];
+			        f$bof_buffer[0:default_entity_excerpt_len];
 		}
 	}

scripts/base/protocols/smtp/entities.bro

@@ -88,6 +88,13 @@ function set_session(c: connection, new_entity: bool)
 		}
 	}
 
+function get_extraction_name(f: fa_file): string
+	{
+	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
+	++extract_count;
+	return r;
+	}
+
 event mime_begin_entity(c: connection) &priority=10
 	{
 	if ( ! c?$smtp ) return;
@@ -95,21 +102,18 @@ event mime_begin_entity(c: connection) &priority=10
 	set_session(c, T);
 	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_new(f: fa_file) &priority=5
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_NEW ) return;
+	if ( ! f?$source ) return;
-	if ( ! info?$source ) return;
+	if ( f$source != "SMTP" ) return;
-	if ( info$source != "SMTP" ) return;
+	if ( ! f?$conns ) return;
-	if ( ! info?$conns ) return;
 
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
+	local fname: string;
-	                          extract_count);
 	local extracting: bool = F;
 
-	for ( cid in info$conns )
+	for ( cid in f$conns )
 		{
-		local c: connection = info$conns[cid];
+		local c: connection = f$conns[cid];
 
 		if ( ! c?$smtp ) next;
 		if ( ! c$smtp?$current_entity ) next;
@@ -118,8 +122,8 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
 			{
 			if ( ! extracting )
 				{
-				FileAnalysis::add_action(info$file_id,
+				fname = get_extraction_name(f);
-				                         [$act=FileAnalysis::ACTION_EXTRACT,
+				FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
 				                             $extract_filename=fname]);
 				extracting = T;
 				++extract_count;
@@ -129,116 +133,79 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
 			}
 
 		if ( c$smtp$current_entity$calc_md5 )
-			FileAnalysis::add_action(info$file_id,
+			FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
-			                         [$act=FileAnalysis::ACTION_MD5]);
 		}
 	}
 
-function check_extract_by_type(info: FileAnalysis::Info)
+function check_extract_by_type(f: fa_file)
 	{
-	if ( extract_file_types !in info$mime_type ) return;
+	if ( extract_file_types !in f$mime_type ) return;
 
-	for ( act in info$actions )
+	if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
-		if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
+		return;
 
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
+	local fname: string = get_extraction_name(f);
-	                          extract_count);
+	FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
-	++extract_count;
-	FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
 	                             $extract_filename=fname]);
 
-	if ( ! info?$conns ) return;
+	if ( ! f?$conns ) return;
 
-	for ( cid in info$conns )
+	for ( cid in f$conns )
 		{
-		local c: connection = info$conns[cid];
+		local c: connection = f$conns[cid];
-
 		if ( ! c?$smtp ) next;
-
 		c$smtp$current_entity$extraction_file = fname;
 		}
 	}
 
-function check_md5_by_type(info: FileAnalysis::Info)
+function check_md5_by_type(f: fa_file)
 	{
 	if ( never_calc_md5 ) return;
-	if ( generate_md5 !in info$mime_type ) return;
+	if ( generate_md5 !in f$mime_type ) return;
 
-	FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]);
+	FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
 	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_new(f: fa_file) &priority=5
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
+	if ( ! f?$source ) return;
-	if ( ! info?$mime_type ) return;
+	if ( f$source != "SMTP" ) return;
-	if ( ! info?$source ) return;
+	if ( ! f?$mime_type ) return;
-	if ( info$source != "SMTP" ) return;
 
-	if ( info?$conns )
+	if ( f?$conns )
-		for ( cid in info$conns )
+		for ( cid in f$conns )
 			{
-			local c: connection = info$conns[cid];
+			local c: connection = f$conns[cid];
 
 			if ( ! c?$smtp ) next;
 			if ( ! c$smtp?$current_entity ) next;
 
-			c$smtp$current_entity$mime_type = info$mime_type;
+			c$smtp$current_entity$mime_type = f$mime_type;
 			}
 
-	check_extract_by_type(info);
+	check_extract_by_type(f);
-	check_md5_by_type(info);
+	check_md5_by_type(f);
 	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_state_remove(f: fa_file) &priority=4
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_GAP ) return;
+	if ( ! f?$source ) return;
-	if ( ! info?$source ) return;
+	if ( f$source != "SMTP" ) return;
-	if ( info$source != "SMTP" ) return;
+	if ( ! f?$conns ) return;
-	if ( ! info?$conns ) return;
 
-	for ( cid in info$conns )
+	for ( cid in f$conns )
 		{
-		local c: connection = info$conns[cid];
+		local c: connection = f$conns[cid];
 
 		if ( ! c?$smtp ) next;
 		if ( ! c$smtp?$current_entity ) next;
+		# Only log if there was some content.
+		if ( f$seen_bytes == 0 ) next;
 
-		FileAnalysis::remove_action(info$file_id,
+		if ( f?$info && f$info?$md5 )
-		                            [$act=FileAnalysis::ACTION_MD5]);
+			c$smtp$current_entity$md5 = f$info$md5;
-		}
-	}
-
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
-	&priority=5
-	{
-	if ( trig != FileAnalysis::TRIGGER_EOF &&
-	     trig != FileAnalysis::TRIGGER_DONE ) return;
-	if ( ! info?$source ) return;
-	if ( info$source != "SMTP" ) return;
-	if ( ! info?$conns ) return;
-
-	for ( cid in info$conns )
-		{
-		local c: connection = info$conns[cid];
-
-		if ( ! c?$smtp ) next;
-		if ( ! c$smtp?$current_entity ) next;
-		# Only log is there was some content.
-		if ( info$seen_bytes == 0 ) next;
-
-		local act: FileAnalysis::ActionArgs = [$act=FileAnalysis::ACTION_MD5];
-
-		if ( act in info$actions )
-			{
-			local result = info$actions[act];
-			if ( result?$md5 )
-				c$smtp$current_entity$md5 = result$md5;
-			}
-
-		c$smtp$current_entity$content_len = info$seen_bytes;
 
+		c$smtp$current_entity$content_len = f$seen_bytes;
 		Log::write(SMTP::ENTITIES_LOG, c$smtp$current_entity);
 		delete c$smtp$current_entity;
 		return;

scripts/base/protocols/smtp/file-analysis.bro

@@ -6,27 +6,21 @@
 module SMTP;
 
 export {
-	## Determines whether the default :bro:see:`get_file_handle` handler
-	## is used to return file handles to the file analysis framework.
-	## Redefine to true in order to provide a custom handler which overrides
-	## the default for SMTP.
-	const disable_default_file_handle_provider: bool = F &redef;
-
 	## Default file handle provider for SMTP.
-	function get_file_handle(c: connection, is_orig: bool): string
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
 	{
 	if ( ! c?$smtp ) return "";
-
 	return fmt("%s %s %s %s", ANALYZER_SMTP, c$start_time,
 	           c$smtp$trans_depth, c$smtp_state$mime_level);
 	}
-}
 
 module GLOBAL;
 
 event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
 	{
 	if ( tag != ANALYZER_SMTP ) return;
-	if ( SMTP::disable_default_file_handle_provider ) return;
+	set_file_handle(SMTP::get_file_handle(c, is_orig));
-	return_file_handle(SMTP::get_file_handle(c, is_orig));
 	}

scripts/policy/frameworks/intel/smtp-url-extraction.bro

@@ -3,13 +3,13 @@
 @load base/utils/urls
 @load ./where-locations
 
-event intel_mime_data(info: FileAnalysis::Info, data: string)
+event intel_mime_data(f: fa_file, data: string)
 	{
-	if ( ! info?$conns ) return;
+	if ( ! f?$conns ) return;
 
-	for ( cid in info$conns )
+	for ( cid in f$conns )
 		{
-		local c: connection = info$conns[cid];
+		local c: connection = f$conns[cid];
 		local urls = find_all_urls_without_scheme(data);
 		for ( url in urls )
 			{
@@ -21,14 +21,11 @@ event intel_mime_data(info: FileAnalysis::Info, data: string)
 		}
 	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_new(f: fa_file) &priority=5
-	&priority=5
 	{
-	if ( trig != FileAnalysis::TRIGGER_NEW ) return;
+	if ( ! f?$source ) return;
-	if ( ! info?$source ) return;
+	if ( f$source != "SMTP" ) return;
-	if ( info$source != "SMTP" ) return;
 
-	FileAnalysis::add_action(info$file_id,
+	FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_DATA_EVENT,
-	                         [$act=FileAnalysis::ACTION_DATA_EVENT,
 	                             $stream_event=intel_mime_data]);
 	}

src/CMakeLists.txt

@@ -454,9 +454,8 @@ set(bro_SRCS
     input/readers/Binary.cc
 
     file_analysis/Manager.cc
-    file_analysis/Info.cc
+    file_analysis/File.cc
-    file_analysis/InfoTimer.cc
+    file_analysis/FileTimer.cc
-    file_analysis/PendingFile.cc
     file_analysis/FileID.h
     file_analysis/Action.h
     file_analysis/ActionSet.cc

src/Event.cc

@@ -6,7 +6,6 @@
 #include "Func.h"
 #include "NetVar.h"
 #include "Trigger.h"
-#include "file_analysis/Manager.h"
 
 EventMgr mgr;
 
@@ -111,6 +110,9 @@ void EventMgr::Dispatch()
 
 void EventMgr::Drain()
 	{
+	if ( event_queue_flush_point )
+		QueueEvent(event_queue_flush_point, new val_list());
+
 	SegmentProfiler(segment_logger, "draining-events");
 
 	draining = true;
@@ -125,8 +127,6 @@ void EventMgr::Drain()
 	// processing, we ensure that it's done at a regular basis by checking
 	// them here.
 	Trigger::EvaluatePending();
-
-	file_mgr->EventDrainDone();
 	}
 
 void EventMgr::Describe(ODesc* d) const

src/NetVar.cc

@@ -9,6 +9,7 @@ RecordType* conn_id;
 RecordType* endpoint;
 RecordType* endpoint_stats;
 RecordType* connection_type;
+RecordType* fa_file_type;
 RecordType* icmp_conn;
 RecordType* icmp_context;
 RecordType* SYN_packet;
@@ -314,6 +315,7 @@ void init_net_var()
 	endpoint = internal_type("endpoint")->AsRecordType();
 	endpoint_stats = internal_type("endpoint_stats")->AsRecordType();
 	connection_type = internal_type("connection")->AsRecordType();
+	fa_file_type = internal_type("fa_file")->AsRecordType();
 	icmp_conn = internal_type("icmp_conn")->AsRecordType();
 	icmp_context = internal_type("icmp_context")->AsRecordType();
 	signature_state = internal_type("signature_state")->AsRecordType();

src/NetVar.h

@@ -12,6 +12,7 @@ extern RecordType* conn_id;
 extern RecordType* endpoint;
 extern RecordType* endpoint_stats;
 extern RecordType* connection_type;
+extern RecordType* fa_file_type;
 extern RecordType* icmp_conn;
 extern RecordType* icmp_context;
 extern RecordType* signature_state;

src/bro.bif

@@ -5538,22 +5538,6 @@ function match_signatures%(c: connection, pattern_type: int, s: string,
 	return new Val(1, TYPE_BOOL);
 	%}
 
-## For use within a :bro:see:`get_file_handle` handler to return a unique
-## identifier to associate with some buffered input to the file analysis
-## framework.  The buffered data will then immediately be allowed to pass
-## pass through the file analysis framework and execute any policy hooks
-## that are available.  If an empty string is returned, that signifies that
-## the buffered data will be discarded with no further action taken on it.
-##
-## handle: A string that uniquely identifies a file.
-##
-## .. bro:see:: get_file_handle FileAnalysis::policy
-function return_file_handle%(handle: string%): any
-	%{
-	file_mgr->ReceiveHandle(handle->CheckString());
-	return 0;
-	%}
-
 # ===========================================================================
 #
 #                            Deprecated Functions

src/event.bif

@@ -6981,12 +6981,15 @@ event reporter_error%(t: time, msg: string, location: string%) &error_handler;
 ##        recursively for each ``@load``.
 event bro_script_loaded%(path: string, level: count%);
 
+## Marks a point in the event stream at which the event queue started flushing.
+event event_queue_flush_point%(%);
+
 ## This event is handled to provide feedback to the file analysis framework
 ## about how to identify the logical "file" to which some data/input
 ## belongs.  All incoming data to the framework is buffered, and depends
 ## on a handler for this event to return a string value that uniquely
-## identifies a file.  Among all handlers of this event, exactly one must
+## identifies a file.  Among all handlers of this event, the last one to
-## call :bro:see:`return_file_handle`.
+## call :bro:see:`set_file_handle` will "win".
 ##
 ## tag: The analyzer which is carrying the file data.
 ##
@@ -6994,9 +6997,35 @@ event bro_script_loaded%(path: string, level: count%);
 ##
 ## is_orig: The direction the file data is flowing over the connection.
 ##
-## .. bro:see:: return_file_handle
+## .. bro:see:: set_file_handle
 event get_file_handle%(tag: count, c: connection, is_orig: bool%);
 
+# TODO: document
+event file_new%(f: fa_file%);
+# TODO: give the new connection
+event file_over_new_connection%(f: fa_file%);
+event file_timeout%(f: fa_file%);
+# TODO: give size of gap
+event file_gap%(f: fa_file%);
+
+## This event is generated each time file analysis is ending for a given file.
+##
+## f: The file.
+event file_state_remove%(f: fa_file%);
+
+## This event is generated each time file analysis generates a digest of the
+## file contents.
+##
+## f: The file.
+##
+## kind: The type of digest algorithm.
+##
+## hash: The result of the hashing.
+##
+## .. bro:see:: FileAnalysis::add_action FileAnalysis::ACTION_MD5
+##    FileAnalysis::ACTION_SHA1 FileAnalysis::ACTION_SHA256
+event file_hash%(f: fa_file, kind: string, hash: string%);
+
 ## Deprecated. Will be removed.
 event stp_create_endp%(c: connection, e: int, is_orig: bool%);
 

src/file_analysis.bif

@@ -6,75 +6,6 @@ module FileAnalysis;
 #include "file_analysis/Manager.h"
 %%}
 
-type Info: record;
-type ActionArgs: record;
-type ActionResults: record;
-
-## An enumeration of significant things that can occur over the course of
-## analyzing files.  The :bro:see:`FileAnalysis::policy` hook is called each
-## time a trigger occurs.
-enum Trigger %{
-
-	## Raised when any part of a new file is detected.
-	TRIGGER_NEW,
-
-	## Raised when file is detected being transported over a new network
-	## connection (other than the first).
-	TRIGGER_NEW_CONN,
-
-	## Raised when file analysis has likely seen a complete file.  That
-	## is when a number of bytes indicated by the *total_bytes* field of
-	## :bro:see:`FileAnalysis::Info` have been processed.
-	TRIGGER_DONE,
-
-	## Raised when file analysis for a given file is aborted due
-	## to not seeing any data for it recently.  Note that this doesn't
-	## necessarily mean the full file wasn't seen (e.g. if the
-	## :bro:see:`FileAnalysis::Info` record indicates the file *total_bytes*
-	## isn't known).  Use :bro:see:`FileAnalysis::postpone_timeout`
-	## during a :bro:see:`FileAnalysis::policy` handler for this trigger to
-	## defer the timeout until later.
-	TRIGGER_TIMEOUT,
-
-	## Raised when the beginning of a file is detected.
-	TRIGGER_BOF,
-
-	## Raised when the beginning of a file is available in the *bof_buffer*
-	## field of :bro:see:`FileAnalysis::Info` and that beginning
-	## is at least the number of bytes indicated by the *bof_buffer_size* field.
-	TRIGGER_BOF_BUFFER,
-
-	## Raised when an initial guess at the file/mime type of a file is matched.
-	TRIGGER_TYPE,
-
-	## Raised to signal that no more file data is incoming and it couldn't be
-	## determined whether the full file was actually seen and analyzed.
-	TRIGGER_EOF,
-
-	## Raised when there's a missing chunk of data in the file stream.
-	TRIGGER_GAP,
-%}
-
-## An enumeration of various file analysis actions that can be taken.
-enum Action %{
-
-	## Extract a file to local filesystem
-	ACTION_EXTRACT,
-
-	## Calculate an MD5 digest of the file's contents.
-	ACTION_MD5,
-
-	## Calculate an SHA1 digest of the file's contents.
-	ACTION_SHA1,
-
-	## Calculate an SHA256 digest of the file's contents.
-	ACTION_SHA256,
-
-	## Deliver the file contents to the script-layer in an event.
-	ACTION_DATA_EVENT,
-	ACTION_PE_ANALYZER,
-%}
-
 ## :bro:see:`FileAnalysis::postpone_timeout`.
 function FileAnalysis::__postpone_timeout%(file_id: string%): bool
 	%{
@@ -149,6 +80,24 @@ function FileAnalysis::__eof%(source: string%): any
 	return 0;
 	%}
 
+module GLOBAL;
+
+## For use within a :bro:see:`get_file_handle` handler to return a unique
+## identifier to associate with some buffered input to the file analysis
+## framework.  The buffered data will then immediately be allowed to pass
+## pass through the file analysis framework and execute any policy hooks
+## that are available.  If an empty string is returned, that signifies that
+## the buffered data will be discarded with no further action taken on it.
+##
+## handle: A string that uniquely identifies a file.
+##
+## .. bro:see:: get_file_handle FileAnalysis::policy
+function set_file_handle%(handle: string%): any
+	%{
+	file_mgr->SetHandle(handle->CheckString());
+	return 0;
+	%}
+
 # Define file analysis framework events.
 
 #event FileAnalysis::windows_pe_dosstub%(fi: FileAnalysis::Info, sig: string, checksum: count%);

src/file_analysis/Action.h

@@ -8,10 +8,10 @@ namespace file_analysis {
 
 typedef BifEnum::FileAnalysis::Action ActionTag;
 
-class Info;
+class File;
 
 /**
- * Base class for actions that can be attached to a file_analysis::Info object.
+ * Base class for actions that can be attached to a file_analysis::File object.
  */
 class Action {
 public:
@@ -67,9 +67,9 @@ public:
 	RecordVal* Args() const { return args; }
 
 	/**
-	 * @return the file_analysis::Info object to which the action is attached.
+	 * @return the file_analysis::File object to which the action is attached.
 	 */
-	Info* GetInfo() const { return info; }
+	File* GetFile() const { return file; }
 
 	/**
 	 * @return the action tag equivalent of the 'act' field from the ActionArgs
@@ -84,17 +84,17 @@ public:
 
 protected:
 
-	Action(RecordVal* arg_args, Info* arg_info)
+	Action(RecordVal* arg_args, File* arg_file)
 	    : tag(Action::ArgsTag(arg_args)), args(arg_args->Ref()->AsRecordVal()),
-	      info(arg_info)
+	      file(arg_file)
 		{}
 
 	ActionTag tag;
 	RecordVal* args;
-	Info* info;
+	File* file;
 };
 
-typedef Action* (*ActionInstantiator)(RecordVal* args, Info* info);
+typedef Action* (*ActionInstantiator)(RecordVal* args, File* file);
 
 } // namespace file_analysis
 

src/file_analysis/ActionSet.cc

@@ -1,5 +1,5 @@
 #include "ActionSet.h"
-#include "Info.h"
+#include "File.h"
 #include "Action.h"
 #include "Extract.h"
 #include "DataEvent.h"
@@ -25,7 +25,7 @@ static void action_del_func(void* v)
 	delete (Action*) v;
 	}
 
-ActionSet::ActionSet(Info* arg_info) : info(arg_info)
+ActionSet::ActionSet(File* arg_file) : file(arg_file)
 	{
 	TypeList* t = new TypeList();
 	t->Append(BifType::Record::FileAnalysis::ActionArgs->Ref());
@@ -54,7 +54,7 @@ bool ActionSet::AddAction(RecordVal* args)
 		{
 		DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate action %d skipped for file id"
 		        " %s: already exists", Action::ArgsTag(args),
-		        info->GetFileID().c_str());
+		        file->GetID().c_str());
 		delete key;
 		return true;
 		}
@@ -94,7 +94,7 @@ bool ActionSet::Add::Perform(ActionSet* set)
 		{
 		DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d skipped for file id"
 		        " %s: already exists", act->Tag(),
-		        act->GetInfo()->GetFileID().c_str());
+		        act->GetFile()->GetID().c_str());
 		Abort();
 		return true;
 		}
@@ -116,12 +116,12 @@ bool ActionSet::RemoveAction(ActionTag tag, HashKey* key)
 	if ( ! act )
 		{
 		DBG_LOG(DBG_FILE_ANALYSIS, "Skip remove action %d for file id %s",
-		        tag, info->GetFileID().c_str());
+		        tag, file->GetID().c_str());
 		return false;
 		}
 
 	DBG_LOG(DBG_FILE_ANALYSIS, "Remove action %d for file id %s", act->Tag(),
-	        info->GetFileID().c_str());
+	        file->GetID().c_str());
 	delete act;
 	return true;
 	}
@@ -151,12 +151,12 @@ HashKey* ActionSet::GetKey(const RecordVal* args) const
 
 Action* ActionSet::InstantiateAction(RecordVal* args) const
 	{
-	Action* act = action_factory[Action::ArgsTag(args)](args, info);
+	Action* act = action_factory[Action::ArgsTag(args)](args, file);
 
 	if ( ! act )
 		{
 		DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate action %d failed for file id",
-		        " %s", Action::ArgsTag(args), info->GetFileID().c_str());
+		        " %s", Action::ArgsTag(args), file->GetID().c_str());
 		return 0;
 		}
 
@@ -166,11 +166,9 @@ Action* ActionSet::InstantiateAction(RecordVal* args) const
 void ActionSet::InsertAction(Action* act, HashKey* key)
 	{
 	DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d for file id %s", act->Tag(),
-	        info->GetFileID().c_str());
+	        file->GetID().c_str());
 	action_map.Insert(key, act);
 	delete key;
-	info->GetVal()->Lookup(Info::actions_idx)->AsTableVal()->Assign(act->Args(),
-	        new RecordVal(BifType::Record::FileAnalysis::ActionResults));
 	}
 
 void ActionSet::DrainModifications()
@@ -178,7 +176,7 @@ void ActionSet::DrainModifications()
 	if ( mod_queue.empty() ) return;
 
 	DBG_LOG(DBG_FILE_ANALYSIS, "Start flushing action mod queue of file id %s",
-	        info->GetFileID().c_str());
+	        file->GetID().c_str());
 	do
 		{
 		Modification* mod = mod_queue.front();
@@ -187,5 +185,5 @@ void ActionSet::DrainModifications()
 		mod_queue.pop();
 		} while ( ! mod_queue.empty() );
 	DBG_LOG(DBG_FILE_ANALYSIS, "End flushing action mod queue of file id %s",
-	        info->GetFileID().c_str());
+	        file->GetID().c_str());
 	}

src/file_analysis/ActionSet.h

@@ -10,7 +10,7 @@
 
 namespace file_analysis {
 
-class Info;
+class File;
 declare(PDict,Action);
 
 /**
@@ -21,7 +21,7 @@ declare(PDict,Action);
 class ActionSet {
 public:
 
-	ActionSet(Info* arg_info);
+	ActionSet(File* arg_file);
 
 	~ActionSet();
 
@@ -63,7 +63,7 @@ protected:
 	void InsertAction(Action* act, HashKey* key);
 	bool RemoveAction(ActionTag tag, HashKey* key);
 
-	Info* info;
+	File* file;
 	CompositeHash* action_hash; /**< ActionArgs hashes Action map lookup. */
 	PDict(Action) action_map;   /**< Actions indexed by ActionArgs. */
 

src/file_analysis/DataEvent.cc

@@ -7,13 +7,13 @@
 
 using namespace file_analysis;
 
-DataEvent::DataEvent(RecordVal* args, Info* info,
+DataEvent::DataEvent(RecordVal* args, File* file,
                      EventHandlerPtr ce, EventHandlerPtr se)
-    : Action(args, info), chunk_event(ce), stream_event(se)
+    : Action(args, file), chunk_event(ce), stream_event(se)
 	{
 	}
 
-Action* DataEvent::Instantiate(RecordVal* args, Info* info)
+Action* DataEvent::Instantiate(RecordVal* args, File* file)
 	{
 	using BifType::Record::FileAnalysis::ActionArgs;
 
@@ -36,7 +36,7 @@ Action* DataEvent::Instantiate(RecordVal* args, Info* info)
 	if ( stream_val )
 		stream = event_registry->Lookup(stream_val->AsFunc()->Name());
 
-	return new DataEvent(args, info, chunk, stream);
+	return new DataEvent(args, file, chunk, stream);
 	}
 
 bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
@@ -44,9 +44,10 @@ bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
 	if ( ! chunk_event ) return true;
 
 	val_list* args = new val_list;
-	args->append(info->GetVal()->Ref());
+	args->append(file->GetVal()->Ref());
 	args->append(new StringVal(new BroString(data, len, 0)));
 	args->append(new Val(offset, TYPE_COUNT));
+
 	mgr.QueueEvent(chunk_event, args);
 
 	return true;
@@ -57,8 +58,9 @@ bool DataEvent::DeliverStream(const u_char* data, uint64 len)
 	if ( ! stream_event ) return true;
 
 	val_list* args = new val_list;
-	args->append(info->GetVal()->Ref());
+	args->append(file->GetVal()->Ref());
 	args->append(new StringVal(new BroString(data, len, 0)));
+
 	mgr.QueueEvent(stream_event, args);
 
 	return true;

src/file_analysis/DataEvent.h

@@ -4,7 +4,7 @@
 #include <string>
 
 #include "Val.h"
-#include "Info.h"
+#include "File.h"
 #include "Action.h"
 
 namespace file_analysis {
@@ -15,7 +15,7 @@ namespace file_analysis {
 class DataEvent : public Action {
 public:
 
-	static Action* Instantiate(RecordVal* args, Info* info);
+	static Action* Instantiate(RecordVal* args, File* file);
 
 	virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);
 
@@ -23,7 +23,7 @@ public:
 
 protected:
 
-	DataEvent(RecordVal* args, Info* info,
+	DataEvent(RecordVal* args, File* file,
 	          EventHandlerPtr ce, EventHandlerPtr se);
 
 	EventHandlerPtr chunk_event;

src/file_analysis/Extract.cc

@@ -5,8 +5,8 @@
 
 using namespace file_analysis;
 
-Extract::Extract(RecordVal* args, Info* info, const string& arg_filename)
+Extract::Extract(RecordVal* args, File* file, const string& arg_filename)
-    : Action(args, info), filename(arg_filename)
+    : Action(args, file), filename(arg_filename)
 	{
 	fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
 
@@ -25,7 +25,7 @@ Extract::~Extract()
 		safe_close(fd);
 	}
 
-Action* Extract::Instantiate(RecordVal* args, Info* info)
+Action* Extract::Instantiate(RecordVal* args, File* file)
 	{
 	using BifType::Record::FileAnalysis::ActionArgs;
 	const char* field = "extract_filename";
@@ -33,7 +33,7 @@ Action* Extract::Instantiate(RecordVal* args, Info* info)
 
 	if ( ! v ) return 0;
 
-	return new Extract(args, info, v->AsString()->CheckString());
+	return new Extract(args, file, v->AsString()->CheckString());
 	}
 
 bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)

src/file_analysis/Extract.h

@@ -4,7 +4,7 @@
 #include <string>
 
 #include "Val.h"
-#include "Info.h"
+#include "File.h"
 #include "Action.h"
 
 namespace file_analysis {
@@ -15,7 +15,7 @@ namespace file_analysis {
 class Extract : public Action {
 public:
 
-	static Action* Instantiate(RecordVal* args, Info* info);
+	static Action* Instantiate(RecordVal* args, File* file);
 
 	virtual ~Extract();
 
@@ -23,7 +23,7 @@ public:
 
 protected:
 
-	Extract(RecordVal* args, Info* info, const string& arg_filename);
+	Extract(RecordVal* args, File* file, const string& arg_filename);
 
 	string filename;
 	int fd;

src/file_analysis/File.cc

@@ -1,14 +1,15 @@
 #include <string>
 #include <openssl/md5.h>
 
-#include "Info.h"
+#include "File.h"
-#include "InfoTimer.h"
+#include "FileTimer.h"
 #include "FileID.h"
 #include "Manager.h"
 #include "Reporter.h"
 #include "Val.h"
 #include "Type.h"
 #include "Analyzer.h"
+#include "Event.h"
 
 using namespace file_analysis;
 
@@ -32,33 +33,32 @@ static RecordVal* get_conn_id_val(const Connection* conn)
 	return v;
 	}
 
-int Info::file_id_idx = -1;
+int File::id_idx = -1;
-int Info::parent_file_id_idx = -1;
+int File::parent_id_idx = -1;
-int Info::source_idx = -1;
+int File::source_idx = -1;
-int Info::conns_idx = -1;
+int File::conns_idx = -1;
-int Info::last_active_idx = -1;
+int File::last_active_idx = -1;
-int Info::seen_bytes_idx = -1;
+int File::seen_bytes_idx = -1;
-int Info::total_bytes_idx = -1;
+int File::total_bytes_idx = -1;
-int Info::missing_bytes_idx = -1;
+int File::missing_bytes_idx = -1;
-int Info::overflow_bytes_idx = -1;
+int File::overflow_bytes_idx = -1;
-int Info::timeout_interval_idx = -1;
+int File::timeout_interval_idx = -1;
-int Info::bof_buffer_size_idx = -1;
+int File::bof_buffer_size_idx = -1;
-int Info::bof_buffer_idx = -1;
+int File::bof_buffer_idx = -1;
-int Info::file_type_idx = -1;
+int File::file_type_idx = -1;
-int Info::mime_type_idx = -1;
+int File::mime_type_idx = -1;
-int Info::actions_idx = -1;
 
-magic_t Info::magic = 0;
+magic_t File::magic = 0;
-magic_t Info::magic_mime = 0;
+magic_t File::magic_mime = 0;
 
-string Info::salt;
+string File::salt;
 
-void Info::StaticInit()
+void File::StaticInit()
 	{
-	if ( file_id_idx != -1 ) return;
+	if ( id_idx != -1 ) return;
 
-	file_id_idx = Idx("file_id");
+	id_idx = Idx("id");
-	parent_file_id_idx = Idx("parent_file_id");
+	parent_id_idx = Idx("parent_id");
 	source_idx = Idx("source");
 	conns_idx = Idx("conns");
 	last_active_idx = Idx("last_active");
@@ -71,7 +71,6 @@ void Info::StaticInit()
 	bof_buffer_idx = Idx("bof_buffer");
 	file_type_idx = Idx("file_type");
 	mime_type_idx = Idx("mime_type");
-	actions_idx = Idx("actions");
 
 	bro_init_magic(&magic, MAGIC_NONE);
 	bro_init_magic(&magic_mime, MAGIC_MIME);
@@ -79,26 +78,26 @@ void Info::StaticInit()
 	salt = BifConst::FileAnalysis::salt->CheckString();
 	}
 
-Info::Info(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
+File::File(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
-    : file_id(""), unique(unique), val(0), postpone_timeout(false),
+    : id(""), unique(unique), val(0), postpone_timeout(false),
-      first_chunk(true), need_type(false), need_reassembly(false), done(false),
+      first_chunk(true), missed_bof(false), need_reassembly(false), done(false),
       actions(this)
 	{
 	StaticInit();
 
-	char id[20];
+	char tmp[20];
 	uint64 hash[2];
 	string msg(unique + salt);
 	MD5(reinterpret_cast<const u_char*>(msg.data()), msg.size(),
 	    reinterpret_cast<u_char*>(hash));
-	uitoa_n(hash[0], id, sizeof(id), 62);
+	uitoa_n(hash[0], tmp, sizeof(tmp), 62);
 
-	DBG_LOG(DBG_FILE_ANALYSIS, "Creating new Info object %s (%s)", id,
+	DBG_LOG(DBG_FILE_ANALYSIS, "Creating new File object %s (%s)", tmp,
 	        unique.c_str());
 
-	val = new RecordVal(BifType::Record::FileAnalysis::Info);
+	val = new RecordVal(fa_file_type);
-	val->Assign(file_id_idx, new StringVal(id));
+	val->Assign(id_idx, new StringVal(tmp));
-	file_id = FileID(id);
+	id = FileID(tmp);
 
 	if ( conn )
 		{
@@ -113,23 +112,23 @@ Info::Info(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
 	UpdateLastActivityTime();
 	}
 
-Info::~Info()
+File::~File()
 	{
-	DBG_LOG(DBG_FILE_ANALYSIS, "Destroying Info object %s", file_id.c_str());
+	DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str());
 	Unref(val);
 	}
 
-void Info::UpdateLastActivityTime()
+void File::UpdateLastActivityTime()
 	{
 	val->Assign(last_active_idx, new Val(network_time, TYPE_TIME));
 	}
 
-double Info::GetLastActivityTime() const
+double File::GetLastActivityTime() const
 	{
 	return val->Lookup(last_active_idx)->AsTime();
 	}
 
-void Info::UpdateConnectionFields(Connection* conn)
+void File::UpdateConnectionFields(Connection* conn)
 	{
 	if ( ! conn ) return;
 
@@ -148,14 +147,13 @@ void Info::UpdateConnectionFields(Connection* conn)
 		{
 		conns->AsTableVal()->Assign(idx, conn->BuildConnVal());
 		if ( ! is_first )
-			file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW_CONN,
+			file_mgr->FileEvent(file_over_new_connection, this);
-			                         this);
 		}
 
 	Unref(idx);
 	}
 
-uint64 Info::LookupFieldDefaultCount(int idx) const
+uint64 File::LookupFieldDefaultCount(int idx) const
 	{
 	Val* v = val->LookupWithDefault(idx);
 	uint64 rval = v->AsCount();
@@ -163,7 +161,7 @@ uint64 Info::LookupFieldDefaultCount(int idx) const
 	return rval;
 	}
 
-double Info::LookupFieldDefaultInterval(int idx) const
+double File::LookupFieldDefaultInterval(int idx) const
 	{
 	Val* v = val->LookupWithDefault(idx);
 	double rval = v->AsInterval();
@@ -171,46 +169,31 @@ double Info::LookupFieldDefaultInterval(int idx) const
 	return rval;
 	}
 
-int Info::Idx(const string& field)
+int File::Idx(const string& field)
 	{
-	int rval = BifType::Record::FileAnalysis::Info->FieldOffset(field.c_str());
+	int rval = fa_file_type->FieldOffset(field.c_str());
 	if ( rval < 0 )
-		reporter->InternalError("Unknown FileAnalysis::Info field: %s",
+		reporter->InternalError("Unknown fa_file field: %s", field.c_str());
-		                        field.c_str());
 	return rval;
 	}
 
-double Info::GetTimeoutInterval() const
+double File::GetTimeoutInterval() const
 	{
 	return LookupFieldDefaultInterval(timeout_interval_idx);
 	}
 
-RecordVal* Info::GetResults(RecordVal* args) const
+void File::IncrementByteCount(uint64 size, int field_idx)
-	{
-	TableVal* actions_table = val->Lookup(actions_idx)->AsTableVal();
-	RecordVal* rval = actions_table->Lookup(args)->AsRecordVal();
-
-	if ( ! rval )
-		{
-		rval = new RecordVal(BifType::Record::FileAnalysis::ActionResults);
-		actions_table->Assign(args, rval);
-		}
-
-	return rval;
-	}
-
-void Info::IncrementByteCount(uint64 size, int field_idx)
 	{
 	uint64 old = LookupFieldDefaultCount(field_idx);
 	val->Assign(field_idx, new Val(old + size, TYPE_COUNT));
 	}
 
-void Info::SetTotalBytes(uint64 size)
+void File::SetTotalBytes(uint64 size)
 	{
 	val->Assign(total_bytes_idx, new Val(size, TYPE_COUNT));
 	}
 
-bool Info::IsComplete() const
+bool File::IsComplete() const
 	{
 	Val* total = val->Lookup(total_bytes_idx);
 	if ( ! total ) return false;
@@ -219,42 +202,27 @@ bool Info::IsComplete() const
 	return false;
 	}
 
-void Info::ScheduleInactivityTimer() const
+void File::ScheduleInactivityTimer() const
 	{
-	timer_mgr->Add(new InfoTimer(network_time, file_id, GetTimeoutInterval()));
+	timer_mgr->Add(new FileTimer(network_time, id, GetTimeoutInterval()));
 	}
 
-bool Info::AddAction(RecordVal* args)
+bool File::AddAction(RecordVal* args)
 	{
 	return done ? false : actions.QueueAddAction(args);
 	}
 
-bool Info::RemoveAction(const RecordVal* args)
+bool File::RemoveAction(const RecordVal* args)
 	{
 	return done ? false : actions.QueueRemoveAction(args);
 	}
 
-bool Info::BufferBOF(const u_char* data, uint64 len)
+bool File::BufferBOF(const u_char* data, uint64 len)
 	{
 	if ( bof_buffer.full || bof_buffer.replayed ) return false;
 
-	if ( bof_buffer.chunks.size() == 0 )
-		file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_BOF, this);
-
 	uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx);
 
-	/* Leaving out this optimization (I think) for now to keep things simpler.
-	// If first chunk satisfies desired size, do everything now without copying.
-	if ( bof_buffer.chunks.empty() && len >= desired_size )
-		{
-		bof_buffer.full = bof_buffer.replayed = true;
-		val->Assign(bof_buffer_idx, new StringVal(new BroString(data, len, 0)));
-		file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
-		// TODO: libmagic stuff
-		return false;
-		}
-	*/
-
 	bof_buffer.chunks.push_back(new BroString(data, len, 0));
 	bof_buffer.size += len;
 
@@ -267,7 +235,7 @@ bool Info::BufferBOF(const u_char* data, uint64 len)
 	return true;
 	}
 
-bool Info::DetectTypes(const u_char* data, uint64 len)
+bool File::DetectTypes(const u_char* data, uint64 len)
 	{
 	const char* desc = bro_magic_buffer(magic, data, len);
 	const char* mime = bro_magic_buffer(magic_mime, data, len);
@@ -281,7 +249,7 @@ bool Info::DetectTypes(const u_char* data, uint64 len)
 	return desc || mime;
 	}
 
-void Info::ReplayBOF()
+void File::ReplayBOF()
 	{
 	if ( bof_buffer.replayed ) return;
 	bof_buffer.replayed = true;
@@ -289,36 +257,33 @@ void Info::ReplayBOF()
 	if ( bof_buffer.chunks.empty() )
 		{
 		// Since we missed the beginning, try file type detect on next data in.
-		need_type = true;
+		missed_bof = true;
 		return;
 		}
 
 	BroString* bs = concatenate(bof_buffer.chunks);
 	val->Assign(bof_buffer_idx, new StringVal(bs));
-	bool have_type = DetectTypes(bs->Bytes(), bs->Len());
 
-	using BifEnum::FileAnalysis::TRIGGER_BOF_BUFFER;
+	DetectTypes(bs->Bytes(), bs->Len());
-	file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
 
-	if ( have_type )
+	file_mgr->FileEvent(file_new, this);
-		file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
+	mgr.Drain(); // need immediate feedback about actions to add
 
 	for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i )
 		DataIn(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len());
 	}
 
-void Info::DataIn(const u_char* data, uint64 len, uint64 offset)
+void File::DataIn(const u_char* data, uint64 len, uint64 offset)
 	{
 	actions.DrainModifications();
 
 	if ( first_chunk )
 		{
-		if ( DetectTypes(data, len) )
+		// TODO: this should all really be delayed until we attempt reassembly
-			{
+		DetectTypes(data, len);
-			file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
+		file_mgr->FileEvent(file_new, this);
+		mgr.Drain(); // need immediate feedback about actions to add
 		actions.DrainModifications();
-			}
-
 		first_chunk = false;
 		}
 
@@ -344,21 +309,19 @@ void Info::DataIn(const u_char* data, uint64 len, uint64 offset)
 	IncrementByteCount(len, seen_bytes_idx);
 	}
 
-void Info::DataIn(const u_char* data, uint64 len)
+void File::DataIn(const u_char* data, uint64 len)
 	{
 	actions.DrainModifications();
 
 	if ( BufferBOF(data, len) ) return;
 
-	if ( need_type )
+	if ( missed_bof )
 		{
-		if ( DetectTypes(data, len) )
+		DetectTypes(data, len);
-			{
+		file_mgr->FileEvent(file_new, this);
-			file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
+		mgr.Drain(); // need immediate feedback about actions to add
 		actions.DrainModifications();
-			}
+		missed_bof = false;
-
-		need_type = false;
 		}
 
 	Action* act = 0;
@@ -383,7 +346,7 @@ void Info::DataIn(const u_char* data, uint64 len)
 	IncrementByteCount(len, seen_bytes_idx);
 	}
 
-void Info::EndOfFile()
+void File::EndOfFile()
 	{
 	if ( done ) return;
 
@@ -403,15 +366,12 @@ void Info::EndOfFile()
 			actions.QueueRemoveAction(act->Args());
 		}
 
-	if ( IsComplete() )
+	file_mgr->FileEvent(file_state_remove, this);
-		file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, this);
-	else
-		file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_EOF, this);
 
 	actions.DrainModifications();
 	}
 
-void Info::Gap(uint64 offset, uint64 len)
+void File::Gap(uint64 offset, uint64 len)
 	{
 	actions.DrainModifications();
 
@@ -428,7 +388,7 @@ void Info::Gap(uint64 offset, uint64 len)
 			actions.QueueRemoveAction(act->Args());
 		}
 
-	file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_GAP, this);
+	file_mgr->FileEvent(file_gap, this);
 
 	actions.DrainModifications();
 	IncrementByteCount(len, missing_bytes_idx);

src/file_analysis/File.h

@@ -1,5 +1,5 @@
-#ifndef FILE_ANALYSIS_INFO_H
+#ifndef FILE_ANALYSIS_FILE_H
-#define FILE_ANALYSIS_INFO_H
+#define FILE_ANALYSIS_FILE_H
 
 #include <string>
 #include <vector>
@@ -15,12 +15,14 @@
 namespace file_analysis {
 
 /**
- * Wrapper class around \c FileAnalysis::Info record values from script layer.
+ * Wrapper class around \c fa_file record values from script layer.
  */
-class Info {
+class File {
+friend class Manager;
+
 public:
 
-	~Info();
+	~File();
 
 	/**
 	 * @return the #val record.
@@ -33,16 +35,9 @@ public:
 	double GetTimeoutInterval() const;
 
 	/**
-	 * @return value of the "file_id" field from #val record.
+	 * @return value of the "id" field from #val record.
 	 */
-	FileID GetFileID() const { return file_id; }
+	FileID GetID() const { return id; }
-
-	/**
-	 * @return looks up the value of the "actions" field in the #val record at
-	 *         the index corresponding to \a args.  If there was no value at
-	 *         the index, it is created.
-	 */
-	RecordVal* GetResults(RecordVal* args) const;
 
 	/**
 	 * @return the string which uniquely identifies the file.
@@ -113,12 +108,10 @@ public:
 
 protected:
 
-	friend class Manager;
-
 	/**
 	 * Constructor; only file_analysis::Manager should be creating these.
 	 */
-	Info(const string& unique, Connection* conn = 0,
+	File(const string& unique, Connection* conn = 0,
 	     AnalyzerTag::Tag tag = AnalyzerTag::Error);
 
 	/**
@@ -162,12 +155,12 @@ protected:
 	 */
 	bool DetectTypes(const u_char* data, uint64 len);
 
-	FileID file_id;            /**< A pretty hash that likely identifies file*/
+	FileID id;                 /**< A pretty hash that likely identifies file */
 	string unique;             /**< A string that uniquely identifies file */
-	RecordVal* val;            /**< \c FileAnalysis::Info from script layer. */
+	RecordVal* val;            /**< \c fa_file from script layer. */
 	bool postpone_timeout;     /**< Whether postponing timeout is requested. */
 	bool first_chunk;          /**< Track first non-linear chunk. */
-	bool need_type;            /**< Flags next data input to be magic typed. */
+	bool missed_bof;           /**< Flags that we missed start of file. */
 	bool need_reassembly;      /**< Whether file stream reassembly is needed. */
 	bool done;                 /**< If this object is about to be deleted. */
 	ActionSet actions;
@@ -198,9 +191,8 @@ protected:
 
 	static string salt;
 
-public:
+	static int id_idx;
-	static int file_id_idx;
+	static int parent_id_idx;
-	static int parent_file_id_idx;
 	static int source_idx;
 	static int conns_idx;
 	static int last_active_idx;
@@ -213,7 +205,6 @@ public:
 	static int bof_buffer_idx;
 	static int file_type_idx;
 	static int mime_type_idx;
-	static int actions_idx;
 };
 
 } // namespace file_analysis

src/file_analysis/FileTimer.cc

@@ -1,23 +1,23 @@
 #include "Manager.h"
-#include "Info.h"
+#include "File.h"
 
 using namespace file_analysis;
 
 
-InfoTimer::InfoTimer(double t, const FileID& id, double interval)
+FileTimer::FileTimer(double t, const FileID& id, double interval)
     : Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id)
 	{
 	DBG_LOG(DBG_FILE_ANALYSIS, "New %f second timeout timer for %s",
 	        file_id.c_str(), interval);
 	}
 
-void InfoTimer::Dispatch(double t, int is_expire)
+void FileTimer::Dispatch(double t, int is_expire)
 	{
-	Info* info = file_mgr->Lookup(file_id);
+	File* file = file_mgr->Lookup(file_id);
 
-	if ( ! info ) return;
+	if ( ! file ) return;
 
-	double last_active = info->GetLastActivityTime();
+	double last_active = file->GetLastActivityTime();
 	double inactive_time = t > last_active ? t - last_active : 0.0;
 
 	DBG_LOG(DBG_FILE_ANALYSIS, "Checking inactivity for %s, last active at %f, "
@@ -26,13 +26,13 @@ void InfoTimer::Dispatch(double t, int is_expire)
 	if ( last_active == 0.0 )
 		{
 		// was created when network_time was zero, so re-schedule w/ valid time
-		info->UpdateLastActivityTime();
+		file->UpdateLastActivityTime();
-		info->ScheduleInactivityTimer();
+		file->ScheduleInactivityTimer();
 		return;
 		}
 
-	if ( inactive_time >= info->GetTimeoutInterval() )
+	if ( inactive_time >= file->GetTimeoutInterval() )
 		file_mgr->Timeout(file_id);
 	else if ( ! is_expire )
-		info->ScheduleInactivityTimer();
+		file->ScheduleInactivityTimer();
 	}

src/file_analysis/FileTimer.h

@@ -1,5 +1,5 @@
-#ifndef FILE_ANALYSIS_INFOTIMER_H
+#ifndef FILE_ANALYSIS_FILETIMER_H
-#define FILE_ANALYSIS_INFOTIMER_H
+#define FILE_ANALYSIS_FILETIMER_H
 
 #include <string>
 #include "Timer.h"
@@ -10,13 +10,13 @@ namespace file_analysis {
 /**
  * Timer to periodically check if file analysis for a given file is inactive.
  */
-class InfoTimer : public Timer {
+class FileTimer : public Timer {
 public:
 
-	InfoTimer(double t, const FileID& id, double interval);
+	FileTimer(double t, const FileID& id, double interval);
 
 	/**
-	 * Check inactivity of file_analysis::Info corresponding to #file_id,
+	 * Check inactivity of file_analysis::File corresponding to #file_id,
 	 * reschedule if active, else call file_analysis::Manager::Timeout.
 	 */
 	void Dispatch(double t, int is_expire);

src/file_analysis/Hash.cc

@@ -2,15 +2,13 @@
 
 #include "Hash.h"
 #include "util.h"
+#include "Event.h"
 
 using namespace file_analysis;
 
-Hash::Hash(RecordVal* args, Info* info, HashVal* hv, const char* field)
+Hash::Hash(RecordVal* args, File* file, HashVal* hv, const char* arg_kind)
-	: Action(args, info), hash(hv), fed(false)
+	: Action(args, file), hash(hv), fed(false), kind(arg_kind)
 	{
-	using BifType::Record::FileAnalysis::ActionResults;
-	if ( (result_field_idx = ActionResults->FieldOffset(field)) < 0 )
-		reporter->InternalError("Missing ActionResults field: %s", field);
 	hash->Init();
 	}
 
@@ -45,6 +43,10 @@ void Hash::Finalize()
 	{
 	if ( ! hash->IsValid() || ! fed ) return;
 
-	StringVal* sv = hash->Get();
+	val_list* vl = new val_list();
-	info->GetResults(args)->Assign(result_field_idx, sv);
+	vl->append(file->GetVal()->Ref());
+	vl->append(new StringVal(kind));
+	vl->append(hash->Get());
+
+	mgr.QueueEvent(file_hash, vl);
 	}

src/file_analysis/Hash.h

@@ -5,7 +5,7 @@
 
 #include "Val.h"
 #include "OpaqueVal.h"
-#include "Info.h"
+#include "File.h"
 #include "Action.h"
 
 namespace file_analysis {
@@ -26,51 +26,51 @@ public:
 
 protected:
 
-	Hash(RecordVal* args, Info* info, HashVal* hv, const char* field);
+	Hash(RecordVal* args, File* file, HashVal* hv, const char* kind);
 
 	void Finalize();
 
 	HashVal* hash;
 	bool fed;
-	int result_field_idx;
+	const char* kind;
 };
 
 class MD5 : public Hash {
 public:
 
-	static Action* Instantiate(RecordVal* args, Info* info)
+	static Action* Instantiate(RecordVal* args, File* file)
-		{ return new MD5(args, info); }
+		{ return file_hash ? new MD5(args, file) : 0; }
 
 protected:
 
-	MD5(RecordVal* args, Info* info)
+	MD5(RecordVal* args, File* file)
-		: Hash(args, info, new MD5Val(), "md5")
+		: Hash(args, file, new MD5Val(), "md5")
 		{}
 };
 
 class SHA1 : public Hash {
 public:
 
-	static Action* Instantiate(RecordVal* args, Info* info)
+	static Action* Instantiate(RecordVal* args, File* file)
-		{ return new SHA1(args, info); }
+		{ return file_hash ? new SHA1(args, file) : 0; }
 
 protected:
 
-	SHA1(RecordVal* args, Info* info)
+	SHA1(RecordVal* args, File* file)
-		: Hash(args, info, new SHA1Val(), "sha1")
+		: Hash(args, file, new SHA1Val(), "sha1")
 		{}
 };
 
 class SHA256 : public Hash {
 public:
 
-	static Action* Instantiate(RecordVal* args, Info* info)
+	static Action* Instantiate(RecordVal* args, File* file)
-		{ return new SHA256(args, info); }
+		{ return file_hash ? new SHA256(args, file) : 0; }
 
 protected:
 
-	SHA256(RecordVal* args, Info* info)
+	SHA256(RecordVal* args, File* file)
-		: Hash(args, info, new SHA256Val(), "sha256")
+		: Hash(args, file, new SHA256Val(), "sha256")
 		{}
 };
 

src/file_analysis/Manager.cc

@@ -2,7 +2,7 @@
 #include <string>
 
 #include "Manager.h"
-#include "Info.h"
+#include "File.h"
 #include "Action.h"
 #include "Var.h"
 #include "Event.h"
@@ -29,78 +29,58 @@ void Manager::Terminate()
 		Timeout(keys[i], true);
 	}
 
-void Manager::ReceiveHandle(const string& handle)
+void Manager::SetHandle(const string& handle)
 	{
-	if ( pending.empty() )
+	current_handle = handle;
-		reporter->InternalError("File analysis underflow");
-
-	PendingFile* pf = pending.front();
-	if ( ! handle.empty() )
-		pf->Finish(handle);
-	delete pf;
-	pending.pop();
-	}
-
-void Manager::EventDrainDone()
-	{
-	if ( pending.empty() ) return;
-
-	reporter->Error("Too few return_file_handle() calls, discarding pending"
-	                " file analysis input.");
-
-	while ( ! pending.empty() )
-		{
-		delete pending.front();
-		pending.pop();
-		}
 	}
 
 void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
                      AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
 	{
 	if ( IsDisabled(tag) ) return;
-	if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
+
-	pending.push(new PendingDataInChunk(data, len, offset, tag, conn));
+	GetFileHandle(tag, conn, is_orig);
+	DataIn(data, len, offset, GetFile(current_handle, conn, tag));
 	}
 
 void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
                      const string& unique)
 	{
-	DataIn(data, len, offset, GetInfo(unique));
+	DataIn(data, len, offset, GetFile(unique));
 	}
 
 void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
-                     Info* info)
+                     File* file)
 	{
-	if ( ! info ) return;
+	if ( ! file ) return;
 
-	info->DataIn(data, len, offset);
+	file->DataIn(data, len, offset);
 
-	if ( info->IsComplete() )
+	if ( file->IsComplete() )
-		RemoveFile(info->GetUnique());
+		RemoveFile(file->GetUnique());
 	}
 
 void Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
                      Connection* conn, bool is_orig)
 	{
 	if ( IsDisabled(tag) ) return;
-	if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
+	GetFileHandle(tag, conn, is_orig);
-	pending.push(new PendingDataInStream(data, len, tag, conn));
+	DataIn(data, len, GetFile(current_handle, conn, tag));
 	}
 
 void Manager::DataIn(const u_char* data, uint64 len, const string& unique)
 	{
-	DataIn(data, len, GetInfo(unique));
+	DataIn(data, len, GetFile(unique));
 	}
 
-void Manager::DataIn(const u_char* data, uint64 len, Info* info)
+void Manager::DataIn(const u_char* data, uint64 len, File* file)
 	{
-	if ( ! info ) return;
+	if ( ! file ) return;
 
-	info->DataIn(data, len);
+	file->DataIn(data, len);
 
-	if ( info->IsComplete() )
+	if ( file->IsComplete() )
-		RemoveFile(info->GetUnique());
+		RemoveFile(file->GetUnique());
 	}
 
 void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn)
@@ -112,8 +92,9 @@ void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn)
 void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
 	{
 	if ( IsDisabled(tag) ) return;
-	if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
+
-	pending.push(new PendingEOF(tag, conn));
+	GetFileHandle(tag, conn, is_orig);
+	EndOfFile(current_handle);
 	}
 
 void Manager::EndOfFile(const string& unique)
@@ -125,102 +106,98 @@ void Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag,
                   Connection* conn, bool is_orig)
 	{
 	if ( IsDisabled(tag) ) return;
-	if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
+
-	pending.push(new PendingGap(offset, len, tag, conn));
+	GetFileHandle(tag, conn, is_orig);
+	Gap(offset, len, GetFile(current_handle, conn, tag));
 	}
 
 void Manager::Gap(uint64 offset, uint64 len, const string& unique)
 	{
-	Gap(offset, len, GetInfo(unique));
+	Gap(offset, len, GetFile(unique));
 	}
 
-void Manager::Gap(uint64 offset, uint64 len, Info* info)
+void Manager::Gap(uint64 offset, uint64 len, File* file)
 	{
-	if ( ! info ) return;
+	if ( ! file ) return;
 
-	info->Gap(offset, len);
+	file->Gap(offset, len);
 	}
 
 void Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
                       bool is_orig)
 	{
 	if ( IsDisabled(tag) ) return;
-	if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
+
-	pending.push(new PendingSize(size, tag, conn));
+	GetFileHandle(tag, conn, is_orig);
+	SetSize(size, GetFile(current_handle, conn, tag));
 	}
 
 void Manager::SetSize(uint64 size, const string& unique)
 	{
-	SetSize(size, GetInfo(unique));
+	SetSize(size, GetFile(unique));
 	}
 
-void Manager::SetSize(uint64 size, Info* info)
+void Manager::SetSize(uint64 size, File* file)
 	{
-	if ( ! info ) return;
+	if ( ! file ) return;
 
-	info->SetTotalBytes(size);
+	file->SetTotalBytes(size);
 
-	if ( info->IsComplete() )
+	if ( file->IsComplete() )
-		RemoveFile(info->GetUnique());
+		RemoveFile(file->GetUnique());
 	}
 
-void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info)
+void Manager::FileEvent(EventHandlerPtr h, File* file)
 	{
-	if ( IsIgnored(info->GetUnique()) ) return;
+	if ( ! h ) return;
+	if ( IsIgnored(file->GetUnique()) ) return;
 
-	const ID* id = global_scope()->Lookup("FileAnalysis::policy");
+	val_list * vl = new val_list();
-	assert(id);
+	vl->append(file->GetVal()->Ref());
-	const Func* hook = id->ID_Val()->AsFunc();
 
-	val_list vl(2);
+	mgr.QueueEvent(h, vl);
-	vl.append(new EnumVal(t, BifType::Enum::FileAnalysis::Trigger));
-	vl.append(info->val->Ref());
-
-	info->postpone_timeout = false;
-
-	Val* result = hook->Call(&vl);
-	Unref(result);
 	}
 
 bool Manager::PostponeTimeout(const FileID& file_id) const
 	{
-	Info* info = Lookup(file_id);
+	File* file = Lookup(file_id);
 
-	if ( ! info ) return false;
+	if ( ! file ) return false;
 
-	info->postpone_timeout = true;
+	file->postpone_timeout = true;
 	return true;
 	}
 
 bool Manager::AddAction(const FileID& file_id, RecordVal* args) const
 	{
-	Info* info = Lookup(file_id);
+	File* file = Lookup(file_id);
 
-	if ( ! info ) return false;
+	if ( ! file ) return false;
 
-	return info->AddAction(args);
+	return file->AddAction(args);
 	}
 
 bool Manager::RemoveAction(const FileID& file_id, const RecordVal* args) const
 	{
-	Info* info = Lookup(file_id);
+	File* file = Lookup(file_id);
 
-	if ( ! info ) return false;
+	if ( ! file ) return false;
 
-	return info->RemoveAction(args);
+	return file->RemoveAction(args);
 	}
 
-Info* Manager::GetInfo(const string& unique, Connection* conn,
+File* Manager::GetFile(const string& unique, Connection* conn,
                        AnalyzerTag::Tag tag)
 	{
+	if ( unique.empty() ) return 0;
 	if ( IsIgnored(unique) ) return 0;
 
-	Info* rval = str_map[unique];
+	File* rval = str_map[unique];
 
 	if ( ! rval )
 		{
-		rval = str_map[unique] = new Info(unique, conn, tag);
+		rval = str_map[unique] = new File(unique, conn, tag);
-		FileID id = rval->GetFileID();
+		FileID id = rval->GetID();
 
 		if ( id_map[id] )
 			{
@@ -229,7 +206,6 @@ Info* Manager::GetInfo(const string& unique, Connection* conn,
 			}
 
 		id_map[id] = rval;
-		file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, rval);
 		rval->ScheduleInactivityTimer();
 		if ( IsIgnored(unique) ) return 0;
 		}
@@ -242,7 +218,7 @@ Info* Manager::GetInfo(const string& unique, Connection* conn,
 	return rval;
 	}
 
-Info* Manager::Lookup(const FileID& file_id) const
+File* Manager::Lookup(const FileID& file_id) const
 	{
 	IDMap::const_iterator it = id_map.find(file_id);
 
@@ -253,25 +229,28 @@ Info* Manager::Lookup(const FileID& file_id) const
 
 void Manager::Timeout(const FileID& file_id, bool is_terminating)
 	{
-	Info* info = Lookup(file_id);
+	File* file = Lookup(file_id);
 
-	if ( ! info ) return;
+	if ( ! file ) return;
 
-	file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info);
+	file->postpone_timeout = false;
 
-	if ( info->postpone_timeout && ! is_terminating )
+	FileEvent(file_timeout, file);
+	mgr.Drain(); // need immediate feedback about whether to postpone
+
+	if ( file->postpone_timeout && ! is_terminating )
 		{
 		DBG_LOG(DBG_FILE_ANALYSIS, "Postpone file analysis timeout for %s",
-		        info->GetFileID().c_str());
+		        file->GetID().c_str());
-		info->UpdateLastActivityTime();
+		file->UpdateLastActivityTime();
-		info->ScheduleInactivityTimer();
+		file->ScheduleInactivityTimer();
 		return;
 		}
 
 	DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s",
-	        info->GetFileID().c_str());
+	        file->GetID().c_str());
 
-	RemoveFile(info->GetUnique());
+	RemoveFile(file->GetUnique());
 	}
 
 bool Manager::IgnoreFile(const FileID& file_id)
@@ -295,7 +274,7 @@ bool Manager::RemoveFile(const string& unique)
 
 	it->second->EndOfFile();
 
-	FileID id = it->second->GetFileID();
+	FileID id = it->second->GetID();
 
 	DBG_LOG(DBG_FILE_ANALYSIS, "Remove FileID %s", id.c_str());
 
@@ -313,6 +292,21 @@ bool Manager::IsIgnored(const string& unique)
 	return ignored.find(unique) != ignored.end();
 	}
 
+void Manager::GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig)
+	{
+	current_handle.clear();
+
+	if ( ! get_file_handle ) return;
+
+	val_list* vl = new val_list();
+	vl->append(new Val(tag, TYPE_COUNT));
+	vl->append(c->BuildConnVal());
+	vl->append(new Val(is_orig, TYPE_BOOL));
+
+	mgr.QueueEvent(get_file_handle, vl);
+	mgr.Drain(); // need file handle immediately so we don't have to buffer data
+	}
+
 bool Manager::IsDisabled(AnalyzerTag::Tag tag)
 	{
 	if ( ! disabled )
@@ -329,17 +323,3 @@ bool Manager::IsDisabled(AnalyzerTag::Tag tag)
 
 	return rval;
 	}
-
-bool Manager::QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
-                               bool is_orig)
-	{
-	if ( ! get_file_handle ) return false;
-
-	val_list* vl = new val_list();
-	vl->append(new Val(tag, TYPE_COUNT));
-	vl->append(conn->BuildConnVal());
-	vl->append(new Val(is_orig, TYPE_BOOL));
-
-	mgr.QueueEvent(get_file_handle, vl);
-	return true;
-	}

src/file_analysis/Manager.h

@@ -12,11 +12,11 @@
 #include "Val.h"
 #include "Analyzer.h"
 #include "Timer.h"
+#include "EventHandler.h"
 
-#include "Info.h"
+#include "File.h"
-#include "InfoTimer.h"
+#include "FileTimer.h"
 #include "FileID.h"
-#include "PendingFile.h"
 
 namespace file_analysis {
 
@@ -24,6 +24,8 @@ namespace file_analysis {
  * Main entry point for interacting with file analysis.
  */
 class Manager {
+friend class FileTimer;
+
 public:
 
 	Manager();
@@ -36,17 +38,9 @@ public:
 	void Terminate();
 
 	/**
-	 * Associates a handle with the next element in the #pending queue, which
+	 * Take in a unique file handle string to identifiy incoming file data.
-	 * will immediately push that element all the way through the file analysis
-	 * framework, possibly evaluating any policy hooks.
 	 */
-	void ReceiveHandle(const string& handle);
+	void SetHandle(const string& handle);
-
-	/**
-	 * Called when all events have been drained from the event queue.
-	 * There should be no pending file input/data at this point.
-	 */
-	void EventDrainDone();
 
 	/**
 	 * Pass in non-sequential file data.
@@ -56,7 +50,7 @@ public:
     void DataIn(const u_char* data, uint64 len, uint64 offset,
                 const string& unique);
     void DataIn(const u_char* data, uint64 len, uint64 offset,
-                Info* info);
+                File* file);
 
 	/**
 	 * Pass in sequential file data.
@@ -64,7 +58,7 @@ public:
 	void DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
 	            Connection* conn, bool is_orig);
 	void DataIn(const u_char* data, uint64 len, const string& unique);
-	void DataIn(const u_char* data, uint64 len, Info* info);
+	void DataIn(const u_char* data, uint64 len, File* file);
 
 	/**
 	 * Signal the end of file data.
@@ -79,7 +73,7 @@ public:
 	void Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, Connection* conn,
 	         bool is_orig);
 	void Gap(uint64 offset, uint64 len, const string& unique);
-	void Gap(uint64 offset, uint64 len, Info* info);
+	void Gap(uint64 offset, uint64 len, File* file);
 
 	/**
 	 * Provide the expected number of bytes that comprise a file.
@@ -87,7 +81,7 @@ public:
 	void SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
 	             bool is_orig);
 	void SetSize(uint64 size, const string& unique);
-	void SetSize(uint64 size, Info* info);
+	void SetSize(uint64 size, File* file);
 
 	/**
 	 * Starts ignoring a file, which will finally be removed from internal
@@ -97,8 +91,8 @@ public:
 	bool IgnoreFile(const FileID& file_id);
 
 	/**
-	 * If called during \c FileAnalysis::policy evaluation for a
+	 * If called during a \c file_timeout event handler, requests deferral of
-	 * \c FileAnalysis::TRIGGER_TIMEOUT, requests deferral of analysis timeout.
+	 * analysis timeout.
 	 */
 	bool PostponeTimeout(const FileID& file_id) const;
 
@@ -117,44 +111,40 @@ public:
 	bool RemoveAction(const FileID& file_id, const RecordVal* args) const;
 
 	/**
-	 * Calls the \c FileAnalysis::policy hook.
+	 * Queues an event related to the file's life-cycle.
 	 */
-	void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info);
+	 void FileEvent(EventHandlerPtr h, File* file);
 
 protected:
 
-	friend class InfoTimer;
+	typedef map<string, File*> StrMap;
-	friend class PendingFile;
-
-	typedef map<string, Info*> StrMap;
 	typedef set<string> StrSet;
-	typedef map<FileID, Info*> IDMap;
+	typedef map<FileID, File*> IDMap;
-	typedef queue<PendingFile*> PendingQueue;
 
 	/**
-	 * @return the Info object mapped to \a unique or a null pointer if analysis
+	 * @return the File object mapped to \a unique or a null pointer if analysis
-	 *         is being ignored for the associated file.  An Info object may be
+	 *         is being ignored for the associated file.  An File object may be
 	 *         created if a mapping doesn't exist, and if it did exist, the
 	 *         activity time is refreshed along with any connection-related
 	 *         fields.
 	 */
-	Info* GetInfo(const string& unique, Connection* conn = 0,
+	File* GetFile(const string& unique, Connection* conn = 0,
 	              AnalyzerTag::Tag tag = AnalyzerTag::Error);
 
 	/**
-	 * @return the Info object mapped to \a file_id, or a null pointer if no
+	 * @return the File object mapped to \a file_id, or a null pointer if no
 	 *         mapping exists.
 	 */
-	Info* Lookup(const FileID& file_id) const;
+	File* Lookup(const FileID& file_id) const;
 
 	/**
-	 * Evaluate timeout policy for a file and remove the Info object mapped to
+	 * Evaluate timeout policy for a file and remove the File object mapped to
 	 * \a file_id if needed.
 	 */
 	void Timeout(const FileID& file_id, bool is_terminating = ::terminating);
 
 	/**
-	 * Immediately remove file_analysis::Info object associated with \a unique.
+	 * Immediately remove file_analysis::File object associated with \a unique.
 	 * @return false if file string did not map to anything, else true.
 	 */
 	bool RemoveFile(const string& unique);
@@ -164,22 +154,22 @@ protected:
 	 */
 	bool IsIgnored(const string& unique);
 
+	/**
+	 * Sets #current_handle to a unique file handle string based on what the
+	 * \c get_file_handle event derives from the connection params.  The
+	 * event queue is flushed so that we can get the handle value immediately.
+	 */
+	void GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig);
+
 	/**
 	 * @return whether file analysis is disabled for the given analyzer.
 	 */
 	static bool IsDisabled(AnalyzerTag::Tag tag);
 
-	/**
+	StrMap str_map; /**< Map unique string to file_analysis::File. */
-	 * Queues \c get_file_handle event in order to retrieve unique file handle.
+	IDMap id_map;   /**< Map file ID to file_analysis::File records. */
-	 * @return true if there is a handler for the event, else false.
-	 */
-	static bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
-	                             bool is_orig);
-
-	StrMap str_map; /**< Map unique strings to \c FileAnalysis::Info records. */
-	IDMap id_map;   /**< Map file IDs to \c FileAnalysis::Info records. */
 	StrSet ignored; /**< Ignored files.  Will be finally removed on EOF. */
-	PendingQueue pending; /**< Files awaiting a unique handle. */
+	string current_handle; /**< Last file handle set by get_file_handle event.*/
 
 	static TableVal* disabled; /**< Table of disabled analyzers. */
 };

src/file_analysis/PendingFile.cc

@@ -1,111 +0,0 @@
-#include "PendingFile.h"
-#include "Manager.h"
-
-using namespace file_analysis;
-
-static void copy_data(const u_char** dst, const u_char* src, uint64 len)
-	{
-	u_char* tmp = new u_char[len];
-	memcpy(tmp, src, len);
-	*dst = tmp;
-	}
-
-static string conn_str(Connection* c)
-	{
-	char op[256], rp[256];
-	modp_ulitoa10(ntohs(c->OrigPort()), op);
-	modp_ulitoa10(ntohs(c->RespPort()), rp);
-	string rval = c->OrigAddr().AsString() + ":" + op + "->" +
-	              c->RespAddr().AsString() + ":" + rp;
-	return rval;
-	}
-
-PendingFile::PendingFile(Connection* arg_conn, AnalyzerTag::Tag arg_tag)
-	: conn(arg_conn), tag(arg_tag)
-	{
-	Ref(conn);
-	DBG_LOG(DBG_FILE_ANALYSIS, "New pending file: %s", conn_str(conn).c_str());
-	}
-
-PendingFile::~PendingFile()
-	{
-	Unref(conn);
-	DBG_LOG(DBG_FILE_ANALYSIS, "Delete pending file: %s",
-	        conn_str(conn).c_str());
-	}
-
-Info* PendingFile::GetInfo(const string& handle) const
-	{
-	return file_mgr->GetInfo(handle, conn, tag);
-	}
-
-PendingDataInChunk::PendingDataInChunk(const u_char* arg_data, uint64 arg_len,
-                                       uint64 arg_offset,
-                                       AnalyzerTag::Tag arg_tag,
-                                       Connection* arg_conn)
-	: PendingFile(arg_conn, arg_tag), len(arg_len),
-	  offset(arg_offset)
-	{
-	copy_data(&data, arg_data, len);
-	}
-
-void PendingDataInChunk::Finish(const string& handle) const
-	{
-	file_mgr->DataIn(data, len, offset, GetInfo(handle));
-	}
-
-PendingDataInChunk::~PendingDataInChunk()
-	{
-	delete [] data;
-	}
-
-PendingDataInStream::PendingDataInStream(const u_char* arg_data, uint64 arg_len,
-                                         AnalyzerTag::Tag arg_tag,
-                                         Connection* arg_conn)
-	: PendingFile(arg_conn, arg_tag), len(arg_len)
-	{
-	copy_data(&data, arg_data, len);
-	}
-
-void PendingDataInStream::Finish(const string& handle) const
-	{
-	file_mgr->DataIn(data, len, GetInfo(handle));
-	}
-
-PendingDataInStream::~PendingDataInStream()
-	{
-	delete [] data;
-	}
-
-PendingGap::PendingGap(uint64 arg_offset, uint64 arg_len,
-                       AnalyzerTag::Tag arg_tag, Connection* arg_conn)
-	: PendingFile(arg_conn, arg_tag), offset(arg_offset),
-	  len(arg_len)
-	{
-	}
-
-void PendingGap::Finish(const string& handle) const
-	{
-	file_mgr->Gap(offset, len, GetInfo(handle));
-	}
-
-PendingEOF::PendingEOF(AnalyzerTag::Tag arg_tag, Connection* arg_conn)
-	: PendingFile(arg_conn, arg_tag)
-	{
-	}
-
-void PendingEOF::Finish(const string& handle) const
-	{
-	file_mgr->EndOfFile(handle);
-	}
-
-PendingSize::PendingSize(uint64 arg_size, AnalyzerTag::Tag arg_tag,
-                         Connection* arg_conn)
-	: PendingFile(arg_conn, arg_tag), size(arg_size)
-	{
-	}
-
-void PendingSize::Finish(const string& handle) const
-	{
-	file_mgr->SetSize(size, GetInfo(handle));
-	}

src/file_analysis/PendingFile.h

@@ -1,103 +0,0 @@
-#ifndef FILE_ANALYSIS_PENDINGFILE_H
-#define FILE_ANALYSIS_PENDINGFILE_H
-
-#include "AnalyzerTags.h"
-#include "Conn.h"
-#include "Info.h"
-
-namespace file_analysis {
-
-/**
- * Provides buffering for file contents until the script-layer is able to
- * return a unique file handle for it.
- */
-class PendingFile {
-public:
-
-	virtual ~PendingFile();
-
-	virtual void Finish(const string& handle) const = 0;
-
-protected:
-
-	PendingFile(Connection* arg_conn,
-	            AnalyzerTag::Tag arg_tag = AnalyzerTag::Error);
-
-	Info* GetInfo(const string& handle) const;
-
-	Connection* conn;
-	AnalyzerTag::Tag tag;
-};
-
-class PendingDataInChunk : public PendingFile {
-public:
-
-	PendingDataInChunk(const u_char* arg_data, uint64 arg_len,
-	                   uint64 arg_offset, AnalyzerTag::Tag arg_tag,
-	                   Connection* arg_conn);
-
-	virtual ~PendingDataInChunk();
-
-	virtual void Finish(const string& handle) const;
-
-protected:
-
-	const u_char* data;
-	uint64 len;
-	uint64 offset;
-};
-
-class PendingDataInStream : public PendingFile {
-public:
-
-	PendingDataInStream(const u_char* arg_data, uint64 arg_len,
-	                    AnalyzerTag::Tag arg_tag, Connection* arg_conn);
-
-	virtual ~PendingDataInStream();
-
-	virtual void Finish(const string& handle) const;
-
-protected:
-
-	const u_char* data;
-	uint64 len;
-};
-
-class PendingGap : public PendingFile {
-public:
-
-	PendingGap(uint64 arg_offset, uint64 arg_len, AnalyzerTag::Tag arg_tag,
-	           Connection* arg_conn);
-
-	virtual void Finish(const string& handle) const;
-
-protected:
-
-	uint64 offset;
-	uint64 len;
-};
-
-class PendingEOF : public PendingFile {
-public:
-
-	PendingEOF(AnalyzerTag::Tag arg_tag, Connection* arg_conn);
-
-	virtual void Finish(const string& handle) const;
-};
-
-class PendingSize : public PendingFile {
-public:
-
-	PendingSize(uint64 arg_size, AnalyzerTag::Tag arg_tag,
-	            Connection* arg_conn);
-
-	virtual void Finish(const string& handle) const;
-
-protected:
-
-	uint64 size;
-};
-
-} // namespace file_analysis
-
-#endif

src/types.bif

@@ -226,3 +226,28 @@ type gtp_rai: record;
 type gtp_qos_profile: record;
 type gtp_private_extension: record;
 type gtp_gsn_addr: record;
+
+module FileAnalysis;
+
+type ActionArgs: record;
+
+## An enumeration of various file analysis actions that can be taken.
+enum Action %{
+
+	## Extract a file to local filesystem
+	ACTION_EXTRACT,
+
+	## Calculate an MD5 digest of the file's contents.
+	ACTION_MD5,
+
+	## Calculate an SHA1 digest of the file's contents.
+	ACTION_SHA1,
+
+	## Calculate an SHA256 digest of the file's contents.
+	ACTION_SHA256,
+
+	## Deliver the file contents to the script-layer in an event.
+	ACTION_DATA_EVENT,
+%}
+
+module GLOBAL;

testing/btest/Baseline/istate.events-ssl/sender.http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-21-05-55
+#open	2013-04-10-15-49-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
-1363986354.505533	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-
+1365608977.146651	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-
-#close	2013-03-22-21-05-56
+#close	2013-04-10-15-49-38

testing/btest/Baseline/istate.events/sender.http.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#open	2013-03-22-21-03-17
+#open	2013-04-10-15-48-08
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	string
-1363986197.076696	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-
+1365608887.935644	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	-	-	-
-#close	2013-03-22-21-03-18
+#close	2013-04-10-15-48-09

testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out

@@ -1,9 +1,8 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 Cx92a0ym5R8, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 ^J0.26 | 201
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
 file_stream, Cx92a0ym5R8, 1500, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J  * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J  * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J  * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J  * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J  * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J  * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J  * Tweaks to release scripts, plus a new one for signing files.^J    (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J  * Tweaks for OpenBSD support. (Jon Siwek)^J^J  * bro-cut extensions and fixes.  (Robin Sommer)^J    ^J    - If no field names are given on the command line, we now pass through^J      all fields. Adresses #657.^J^J    - Removing some GNUism from awk script. Addresses #653.^J^J    - Added option for time output in UTC. Addresses #668.^J^J    - Added output field separator option -F. Addresses #649.^J^J    - Fixing option -c: only some header lines were passed through^J      rather than all. (Robin Sommer)^J^J  * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J  * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J  * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J  * Little fix for make-relea
@@ -14,15 +13,15 @@ file_stream, Cx92a0ym5R8, 476, ormat for RFC2253^J    compliance. (Jon Siwek)^J^
 file_chunk, Cx92a0ym5R8, 476, 2524, ormat for RFC2253^J    compliance. (Jon Siwek)^J^J  * New tool devel-tools/check-release to run before making releases.^J    (Robin Sommer)^J^J  * devel-tools/update-changes gets a new option -a to amend to^J    previous commit if possible. Default is now not to (used to be the^J    opposite). (Robin Sommer)^J^J  * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)^J^J  * Change distclean to only remove build dir. (Jon Siwek)^J^J  * Make dist now cleans the 
 file_stream, Cx92a0ym5R8, 1024, copied source (Jon Siwek)^J^J  * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J  * Fix to not let updates scripts loose their executable permissions.^J    (Robin Sommer)^J^J  * devel-tools/update-changes now looks for a 'release' tag to^J    idenfify the stable version, and 'beta' for the beta versions.^J    (Robin Sommer).^J^J  * Distribution cleanup. (Robin Sommer)^J^J  * New script devel-tools/make-release to create source tar balls.^J    (Robin Sommer)^J^J  * Removing bdcat. With the new log format, this isn't very useful^J    anymore. (Robin Sommer)^J^J  * Adding script that shows all pending git fastpath commits. (Robin^J    Sommer)^J^J  * Script to measure CPU time by loading an increasing set of^J    scripts. (Robin Sommer)^J^J  * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J  * Tiny update to output a valid CA list file for SSL cert^J    validation. (Seth Hall)^J^J  * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J  * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J  * FindPCAP
 file_chunk, Cx92a0ym5R8, 1024, 3000, copied source (Jon Siwek)^J^J  * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J  * Fix to not let updates scripts loose their executable permissions.^J    (Robin Sommer)^J^J  * devel-tools/update-changes now looks for a 'release' tag to^J    idenfify the stable version, and 'beta' for the beta versions.^J    (Robin Sommer).^J^J  * Distribution cleanup. (Robin Sommer)^J^J  * New script devel-tools/make-release to create source tar balls.^J    (Robin Sommer)^J^J  * Removing bdcat. With the new log format, this isn't very useful^J    anymore. (Robin Sommer)^J^J  * Adding script that shows all pending git fastpath commits. (Robin^J    Sommer)^J^J  * Script to measure CPU time by loading an increasing set of^J    scripts. (Robin Sommer)^J^J  * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J  * Tiny update to output a valid CA list file for SSL cert^J    validation. (Seth Hall)^J^J  * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J  * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J  * FindPCAP
-FileAnalysis::TRIGGER_DONE
-Cx92a0ym5R8, 4705, 0
-[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
-total bytes: 4705
-source: HTTP
-SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
-MD5: 397168fd09991a0e712254df7bc639ac
-SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
 file_stream, Cx92a0ym5R8, 476,  now links against thread library when necessary (e.g.^J    PF_RING's libpcap) (Jon Siwek)^J^J  * Install binaries with an RPATH (Jon Siwek)^J^J  * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J  * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J  * Add a script for generating Mozilla's CA list for the SSL analyzer.^J    (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J  * Converting build process to CMake. (Jon Siwek)^J
 file_chunk, Cx92a0ym5R8, 476, 4024,  now links against thread library when necessary (e.g.^J    PF_RING's libpcap) (Jon Siwek)^J^J  * Install binaries with an RPATH (Jon Siwek)^J^J  * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J  * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J  * Add a script for generating Mozilla's CA list for the SSL analyzer.^J    (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J  * Converting build process to CMake. (Jon Siwek)^J
 file_stream, Cx92a0ym5R8, 205, ^J  * Removing cf/hf/ca-* from distribution. The README has a note where^J    to find them now. (Robin Sommer)^J^J  * General cleanup. (Robin Sommer)^J^J  * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J
 file_chunk, Cx92a0ym5R8, 205, 4500, ^J  * Removing cf/hf/ca-* from distribution. The README has a note where^J    to find them now. (Robin Sommer)^J^J  * General cleanup. (Robin Sommer)^J^J  * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J
+FILE_STATE_REMOVE
+Cx92a0ym5R8, 4705, 0
+[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
+total bytes: 4705
+source: HTTP
+MD5: 397168fd09991a0e712254df7bc639ac
+SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
+SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18

testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout

@@ -1,21 +1,21 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 oDwT1BbzjM1, 0, 0
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 oDwT1BbzjM1, 1022920, 0
 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
 total bytes: 1022920
 source: HTTP
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 oDwT1BbzjM1, 0, 0
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_TIMEOUT
+FILE_TIMEOUT
-FileAnalysis::TRIGGER_TIMEOUT
+FILE_TIMEOUT
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 oDwT1BbzjM1, 206024, 0
 [orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp]
 total bytes: 1022920

testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out

@@ -1,12 +1,11 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 Cx92a0ym5R8, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 ^J0.26 | 201
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 Cx92a0ym5R8, 4705, 0
 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
 total bytes: 4705

testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out

@@ -1,2 +1,7 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 Cx92a0ym5R8, 0, 0
+FILE_BOF_BUFFER
+^J0.26 | 201
+FILE_TYPE
+file type is set
+mime type is set

testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out

@@ -1,15 +1,14 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 sidhzrR4IT8, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 The Nationa
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 sidhzrR4IT8, 16557, 0
 [orig_h=141.142.228.5, orig_p=50737/tcp, resp_h=141.142.192.162, resp_p=38141/tcp]
 source: FTP_DATA
-SHA1: 44586aed07cfe19cad25076af98f535585cd5797
 MD5: 7192a8075196267203adb3dfaa5c908d
+SHA1: 44586aed07cfe19cad25076af98f535585cd5797
 SHA256: 202674eba48e832690a4475113acf8b16a3f6c82c04c94b36bb2c7ce457ac8d2

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out

@@ -1,15 +1,14 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 kg59rqyYxN, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 {^J  "origin
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 kg59rqyYxN, 197, 0
 [orig_h=141.142.228.5, orig_p=50153/tcp, resp_h=54.243.118.187, resp_p=80/tcp]
 source: HTTP
-SHA1: e351b8c693c3353716787c02e2923f4d12ebbb31
 MD5: 5baba7eea57bc8a42a92c817ed566d72
+SHA1: e351b8c693c3353716787c02e2923f4d12ebbb31
 SHA256: 202b775be087f5af98e95120e42769a9b3488f84c5aa79c4f4c1093d348f849c

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out

@@ -1,16 +1,15 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 Cx92a0ym5R8, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 ^J0.26 | 201
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 Cx92a0ym5R8, 4705, 0
 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
 total bytes: 4705
 source: HTTP
-SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
 MD5: 397168fd09991a0e712254df7bc639ac
+SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
 SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out

@@ -1,10 +1,10 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 7gZBKVUgy4l, 0, 0
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_NEW_CONN
+FILE_OVER_NEW_CONNECTION
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 7gZBKVUgy4l, 555523, 0
 [orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp]
 [orig_h=10.101.84.70, orig_p=10977/tcp, resp_h=129.174.93.161, resp_p=80/tcp]

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out

@@ -1,20 +1,20 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 oDwT1BbzjM1, 0, 0
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 oDwT1BbzjM1, 1022920, 0
 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
 total bytes: 1022920
 source: HTTP
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 oDwT1BbzjM1, 0, 0
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_TIMEOUT
+FILE_TIMEOUT
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 oDwT1BbzjM1, 206024, 0
 [orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp]
 total bytes: 1022920

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out

@@ -1,10 +1,10 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 uHS14uhRKGe, 0, 0
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_NEW_CONN
+FILE_OVER_NEW_CONNECTION
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 uHS14uhRKGe, 498702, 0
 [orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp]
 [orig_h=10.45.179.94, orig_p=19953/tcp, resp_h=129.174.93.170, resp_p=80/tcp]

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out

@@ -1,78 +1,73 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 aFQKI8SPOL2, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 /*^J********
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 aFQKI8SPOL2, 2675, 0
 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
 source: HTTP
-SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706
 MD5: b932c3310ce47e158d1a5a42e0b01279
+SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706
 SHA256: 5b037a2c5e36f56e63a3012c73e46a04b27741d8ff8f8b62c832fb681fc60f42
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 CCU3vUEr06l, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 //-- Google
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 CCU3vUEr06l, 21421, 0
 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
 source: HTTP
-SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983
 MD5: e732f7bf1d7cb4eedcb1661697d7bc8c
+SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983
 SHA256: 6a509fd05aa7c8fa05080198894bb19e638554ffcee0e0b3d7bc8ff54afee1da
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 HCzA0dVwDPj, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 GIF89a^D\0^D\0\xb3
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 HCzA0dVwDPj, 94, 0
 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
 total bytes: 94
 source: HTTP
-SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e
 MD5: d903de7e30db1691d3130ba5eae6b9a7
+SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e
 SHA256: 6fb22aa9d780ea63bd7a2e12b92b16fcbf1c4874f1d3e11309a5ba984433c315
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 a1Zu1fteVEf, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 \x89PNG^M^J^Z^J\0\0\0
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 a1Zu1fteVEf, 2349, 0
 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
 total bytes: 2349
 source: HTTP
-SHA1: 560eab5a0177246827a94042dd103916d8765ac7
 MD5: e0029eea80812e9a8e57b8d05d52938a
+SHA1: 560eab5a0177246827a94042dd103916d8765ac7
 SHA256: e0b4500c1fd1d675da4137461cbe64d3c8489f4180d194e47683b20e7fb876f4
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 xXlF7wFdsR, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 \x89PNG^M^J^Z^J\0\0\0
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 xXlF7wFdsR, 27579, 0
 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
 total bytes: 27579
 source: HTTP
-SHA1: ee2b41bdef85de14ef332da14fc392f110b84249
 MD5: 30aa926344f58019d047e85ba049ca1e
+SHA1: ee2b41bdef85de14ef332da14fc392f110b84249
 SHA256: eb482bda230a215b90aedbfe1eee72b8193608df76a319aaf11fb85511579a1e

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out

@@ -1,32 +1,30 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 v5HLI7MxPQh, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 hello world
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 v5HLI7MxPQh, 11, 0
 [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp]
 total bytes: 11
 source: HTTP
-SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
 MD5: 5eb63bbbe01eeed093cb22bb8f5acdc3
+SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
 SHA256: b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 PZS1XGHkIf1, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 {^J  "origin
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_DONE
+FILE_STATE_REMOVE
 PZS1XGHkIf1, 366, 0
 [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp]
 total bytes: 366
 source: HTTP
-SHA1: 6a1582672c203210c6d18d700322060b676365e7
 MD5: c9337794df612aeaa901dcf9fa446bca
+SHA1: 6a1582672c203210c6d18d700322060b676365e7
 SHA256: 8eb24c16df7cb45cb6a1790b0d26ad2571f754228d0ac111b3ac59adbfecbeb8

testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout

@@ -1,14 +1,13 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 nYgPNGLrZf9, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 #separator 
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 nYgPNGLrZf9, 311, 0
 source: ../input.log
-SHA1: 0a0f20de89c86d7bce1301af6548d6e9ae87b0f1
 MD5: bf4dfa6169b74146da5236e918743599
+SHA1: 0a0f20de89c86d7bce1301af6548d6e9ae87b0f1
 SHA256: 4e573192c5ea75da72494812fe24dae53a577837b2079df012fd464903d68a6f

testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out

@@ -1,15 +1,14 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 wqKMAamJVSb, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 PK^C^D^T\0\0\0^H\0\xae
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 wqKMAamJVSb, 42208, 0
 [orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp]
 source: IRC_DATA
-SHA1: 8abe0239263fd7326eb803d4465cf494f8bea218
 MD5: 8c0803242f549c2780cb88b9a9215c65
+SHA1: 8abe0239263fd7326eb803d4465cf494f8bea218
 SHA256: e4f0b0b9d7580e7a22dc1093c8db4df7d0115a4f3b03cc2875cc69705f0d0204

testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	file_analysis
-#open	2013-03-29-18-28-57
+#open	2013-04-04-21-22-26
-#fields	file_id	parent_file_id	source	last_active	seen_bytes	total_bytes	missing_bytes	overflow_bytes	timeout_interval	bof_buffer_size	file_type	mime_type	timedout	conn_uids	actions_taken	extracted_files	md5	sha1	sha256
+#fields	id	parent_id	source	last_active	seen_bytes	total_bytes	missing_bytes	overflow_bytes	timeout_interval	bof_buffer_size	file_type	mime_type	timedout	conn_uids	actions_taken	extracted_files	md5	sha1	sha256
 #types	string	string	string	time	count	count	count	count	interval	count	string	string	bool	table[string]	table[enum]	table[string]	string	string	string
-Cx92a0ym5R8	-	HTTP	1362692527.009775	4705	4705	0	0	120.000000	1024	set	set	F	UWkUyAuUGXf	FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256	Cx92a0ym5R8-file	397168fd09991a0e712254df7bc639ac	1dd7ac0398df6cbc0696445a91ec681facf4dc47	4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
+Cx92a0ym5R8	-	HTTP	1362692527.009775	4705	4705	0	0	120.000000	1024	set	set	F	UWkUyAuUGXf	FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256	Cx92a0ym5R8-file	397168fd09991a0e712254df7bc639ac	1dd7ac0398df6cbc0696445a91ec681facf4dc47	4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
-#close	2013-03-29-18-28-57
+#close	2013-04-04-21-22-26

testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out

@@ -1,45 +1,42 @@
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 cwR7l6Zctxb, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 Hello^M^J^M^J ^M
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 cwR7l6Zctxb, 79, 0
 [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]
 source: SMTP
-SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13
 MD5: 92bca2e6cdcde73647125da7dccbdd07
+SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13
 SHA256: 785a8a044d1454ec88837108f443bbb30cc4f529393ffd57118261036bfe59f5
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 ZAOEQmRyxv1, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 <html xmlns
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 ZAOEQmRyxv1, 1918, 0
 [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]
 source: SMTP
-SHA1: e54af6c6616525611364b80bd6557a7ea21dae94
 MD5: d194c6359c85bb88b54caee18b1e9b44
+SHA1: e54af6c6616525611364b80bd6557a7ea21dae94
 SHA256: b9556e92ddbe52379b64804136f830d111cafe7fcd78e54817fe40f3bc24268d
-FileAnalysis::TRIGGER_NEW
+FILE_NEW
 Ltd7QO7jEv3, 0, 0
-FileAnalysis::TRIGGER_BOF
+FILE_BOF_BUFFER
-FileAnalysis::TRIGGER_BOF_BUFFER
 Version 4.9
-FileAnalysis::TRIGGER_TYPE
+FILE_TYPE
 file type is set
 mime type is set
-FileAnalysis::TRIGGER_EOF
+FILE_STATE_REMOVE
 Ltd7QO7jEv3, 10823, 0
 [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]
 source: SMTP
-SHA1: 43bf1cea1cd4b7d15243e15611859aa49d515665
 MD5: a968bb0f9f9d95835b2e74c845877e87
+SHA1: 43bf1cea1cd4b7d15243e15611859aa49d515665
 SHA256: d5c4e7248840932b9d74ea2f3b3ae142c723a863abf5fd0599f9dd1171697e12

testing/btest/core/leaks/file-analysis-http-get.bro

@@ -8,7 +8,7 @@
 
 redef test_file_analysis_source = "HTTP";
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
 	{
-	return fmt("%s-file", info$file_id);
+	return fmt("%s-file", f$id);
 	};

testing/btest/istate/events-ssl.bro

@@ -41,12 +41,16 @@ redef ssl_ca_certificate = "../ca_cert.pem";
 redef ssl_private_key = "../bro.pem";
 redef ssl_passphrase = "my-password";
 
-# File analysis that populates fields in the http.log would make the sender's
+# File-analysis fields in http.log won't get set on receiver side correctly,
-# log differ from the receiver's since hooks don't get sent to peers.
+# one problem is with the way serialization may send a unique ID in place
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+# of a full value and expect the remote side to associate that unique ID with
-	&priority=10
+# a value it received at an earlier time.  So sometimes modifications the sender
+# makes to the value aren't seen on the receiver (in this case, the mime_type
+# field).
+event file_new(f: fa_file) &priority=10
 	{
-	FileAnalysis::stop(info$file_id);
+	delete f$mime_type;
+	FileAnalysis::stop(f);
 	}
 
 @TEST-END-FILE
@@ -63,7 +67,7 @@ event bro_init()
 redef peer_description = "events-rcv";
 
 redef Communication::nodes += {
-    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $ssl=T, $retry=1sec]
+    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match|file_.*/, $connect=T, $ssl=T, $retry=1sec]
 };
 
 redef ssl_ca_certificate = "../ca_cert.pem";

testing/btest/istate/events.bro

@@ -36,12 +36,15 @@ redef peer_description = "events-send";
 # it gets propagated but that's ok.)
 redef tcp_close_delay = 0secs;
 
-# File analysis that populates fields in the http.log would make the sender's
+# File-analysis fields in http.log won't get set on receiver side correctly,
-# log differ from the receiver's since hooks don't get sent to peers.
+# one problem is with the way serialization may send a unique ID in place
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+# of a full value and expect the remote side to associate that unique ID with
-	&priority=10
+# a value it received at an earlier time.  So sometimes modifications the sender# makes to the value aren't seen on the receiver (in this case, the mime_type
+# field).
+event file_new(f: fa_file) &priority=10
 	{
-	FileAnalysis::stop(info$file_id);
+	delete f$mime_type;
+	FileAnalysis::stop(f);
 	}
 
 @TEST-END-FILE
@@ -58,7 +61,7 @@ event bro_init()
 redef peer_description = "events-rcv";
 
 redef Communication::nodes += {
-    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $retry=1sec]
+    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match|file_.*/, $connect=T, $retry=1sec]
 };
 
 event remote_connection_closed(p: event_peer)

testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro

@@ -5,25 +5,22 @@
 global cnt: count = 0;
 global timeout_cnt: count = 0;
 
-redef FileAnalysis::default_timeout_interval=2sec;
-
 redef test_file_analysis_source = "HTTP";
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
 	{
-	local rval: string = fmt("%s-file%d", info$file_id, cnt);
+	local rval: string = fmt("%s-file%d", f$id, cnt);
 	++cnt;
 	return rval;
 	};
 
 redef exit_only_after_terminate = T;
+redef default_file_timeout_interval = 2sec;
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_timeout(f: fa_file)
 	{
-	if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
-
 	if ( timeout_cnt < 1 )
-		FileAnalysis::postpone_timeout(info$file_id);
+		FileAnalysis::postpone_timeout(f);
 	else
 		terminate();
 	++timeout_cnt;

testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro

@@ -3,18 +3,16 @@
 
 redef test_file_analysis_source = "HTTP";
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
 	{
-	return fmt("%s-file", info$file_id);
+	return fmt("%s-file", f$id);
 	};
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_new(f: fa_file) &priority=-10
 	{
-	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
 	for ( act in test_file_actions )
-		FileAnalysis::remove_action(info$file_id, act);
+		FileAnalysis::remove_action(f, act);
-	local filename = test_get_file_name(info);
+	local filename = test_get_file_name(f);
-	FileAnalysis::remove_action(info$file_id,
+	FileAnalysis::remove_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
-	                             [$act=FileAnalysis::ACTION_EXTRACT,
 	                                $extract_filename=filename]);
 	}

testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro

@@ -2,8 +2,7 @@
 # @TEST-EXEC: btest-diff get.out
 # @TEST-EXEC: test ! -s Cx92a0ym5R8-file
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_new(f: fa_file)
 	{
-	if ( trig != FileAnalysis::TRIGGER_NEW ) return;
+	FileAnalysis::stop(f);
-	FileAnalysis::stop(info$file_id);
 	}

testing/btest/scripts/base/frameworks/file-analysis/ftp.bro

@@ -4,7 +4,7 @@
 
 redef test_file_analysis_source = "FTP_DATA";
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
 	{
 	return "thefile";
 	};

testing/btest/scripts/base/frameworks/file-analysis/http/get.bro

@@ -7,7 +7,7 @@
 
 redef test_file_analysis_source = "HTTP";
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
 	{
-	return fmt("%s-file", info$file_id);
+	return fmt("%s-file", f$id);
 	};

testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro

@@ -17,9 +17,9 @@ global cnt: count = 0;
 
 redef test_file_analysis_source = "HTTP";
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
 	{
-	local rval: string = fmt("%s-file%d", info$file_id, cnt);
+	local rval: string = fmt("%s-file%d", f$id, cnt);
 	++cnt;
 	return rval;
 	};

testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro

@@ -8,7 +8,7 @@
 
 redef test_file_analysis_source = "HTTP";
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
 	{
-	return fmt("%s-file", info$file_id);
+	return fmt("%s-file", f$id);
 	};

testing/btest/scripts/base/frameworks/file-analysis/http/post.bro

@@ -5,7 +5,7 @@
 
 redef test_file_analysis_source = "HTTP";
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
 	{
-	return fmt("%s-file", info$file_id);
+	return fmt("%s-file", f$id);
 	};

testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro

@@ -5,9 +5,9 @@
 
 redef exit_only_after_terminate = T;
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
     {
-    return fmt("%s-file", info$file_id);
+    return fmt("%s-file", f$id);
     };
 
 @TEST-START-FILE input.log
@@ -42,10 +42,7 @@ event bro_init()
 	Input::remove("input");
 	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_state_remove(f: fa_file) &priority=-10
-	&priority=-10
 	{
-	if ( trig != FileAnalysis::TRIGGER_EOF ) return;
 	terminate();
 	}
-

testing/btest/scripts/base/frameworks/file-analysis/irc.bro

@@ -4,7 +4,7 @@
 
 redef test_file_analysis_source = "IRC_DATA";
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
 	{
 	return "thefile";
 	};

testing/btest/scripts/base/frameworks/file-analysis/logging.bro

@@ -3,7 +3,7 @@
 
 redef test_file_analysis_source = "HTTP";
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
     {
-    return fmt("%s-file", info$file_id);
+    return fmt("%s-file", f$id);
     };

testing/btest/scripts/base/frameworks/file-analysis/smtp.bro

@@ -8,7 +8,7 @@ redef test_file_analysis_source = "SMTP";
 
 global mycnt: count = 0;
 
-redef test_get_file_name = function(info: FileAnalysis::Info): string
+redef test_get_file_name = function(f: fa_file): string
 	{
 	local rval: string = fmt("thefile%d", mycnt);
 	++mycnt;

testing/external/subdir-btest.cfg

@@ -17,4 +17,4 @@ TRACES=%(testbase)s/Traces
 SCRIPTS=%(testbase)s/../scripts
 DIST=%(testbase)s/../../..
 BUILD=%(testbase)s/../../../build
-BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX
+#BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX

testing/scripts/file-analysis-test.bro

@@ -3,105 +3,101 @@ global test_file_analysis_source: string = "" &redef;
 
 global test_file_actions: set[FileAnalysis::ActionArgs];
 
-global test_get_file_name: function(info: FileAnalysis::Info): string =
+global test_get_file_name: function(f: fa_file): string =
-	function(info: FileAnalysis::Info): string { return ""; } &redef;
+	function(f: fa_file): string { return ""; } &redef;
 
 global test_print_file_data_events: bool = F &redef;
 
-event file_chunk(info: FileAnalysis::Info, data: string, off: count)
+event file_chunk(f: fa_file, data: string, off: count)
 	{
 	if ( test_print_file_data_events )
-		print "file_chunk", info$file_id, |data|, off, data;
+		print "file_chunk", f$id, |data|, off, data;
 	}
 
-event file_stream(info: FileAnalysis::Info, data: string)
+event file_stream(f: fa_file, data: string)
 	{
 	if ( test_print_file_data_events )
-		print "file_stream", info$file_id, |data|, data;
+		print "file_stream", f$id, |data|, data;
 	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+event file_new(f: fa_file)
 	{
-	print trig;
+	print "FILE_NEW";
 
-	switch ( trig ) {
+	print f$id, f$seen_bytes, f$missing_bytes;
-	case FileAnalysis::TRIGGER_NEW:
-		print info$file_id, info$seen_bytes, info$missing_bytes;
 
 	if ( test_file_analysis_source == "" ||
-		     info$source == test_file_analysis_source )
+	     f$source == test_file_analysis_source )
 		{
 		for ( act in test_file_actions )
-				FileAnalysis::add_action(info$file_id, act);
+			FileAnalysis::add_action(f, act);
 
-			local filename: string = test_get_file_name(info);
+		local filename: string = test_get_file_name(f);
 		if ( filename != "" )
-				FileAnalysis::add_action(info$file_id,
+			FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
-				                         [$act=FileAnalysis::ACTION_EXTRACT,
 			                             $extract_filename=filename]);
-			FileAnalysis::add_action(info$file_id,
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_DATA_EVENT,
-			                         [$act=FileAnalysis::ACTION_DATA_EVENT,
 		                             $chunk_event=file_chunk,
 		                             $stream_event=file_stream]);
-
 		}
-		break;
 
-	case FileAnalysis::TRIGGER_BOF_BUFFER:
+	if ( f?$bof_buffer )
-		if ( info?$bof_buffer )
+		{
-			print info$bof_buffer[0:10];
+		print "FILE_BOF_BUFFER";
-		break;
+		print f$bof_buffer[0:10];
+		}
 
-	case FileAnalysis::TRIGGER_TYPE:
+	if ( f?$file_type || f?$mime_type )
+		print "FILE_TYPE";
 	# not actually printing the values due to libmagic variances
-		if ( info?$file_type )
+	if ( f?$file_type )
+		{
 		print "file type is set";
-		if ( info?$mime_type )
+		f$file_type = "set";
+		}
+	if ( f?$mime_type )
+		{
 		print "mime type is set";
-		break;
+		f$mime_type = "set";
+		}
+	}
 
-	case FileAnalysis::TRIGGER_EOF:
+event file_over_new_connection(f: fa_file)
-		fallthrough;
+	{
-	case FileAnalysis::TRIGGER_DONE:
+	print "FILE_OVER_NEW_CONNECTION";
-		print info$file_id, info$seen_bytes, info$missing_bytes;
+	}
-		if ( info?$conns )
+
-			for ( cid in info$conns )
+event file_timeout(f: fa_file)
+	{
+	print "FILE_TIMEOUT";
+	}
+
+event file_gap(f: fa_file)
+	{
+	print "FILE_GAP";
+	}
+
+event file_state_remove(f: fa_file)
+	{
+	print "FILE_STATE_REMOVE";
+	print f$id, f$seen_bytes, f$missing_bytes;
+	if ( f?$conns )
+		for ( cid in f$conns )
 			print cid;
 
-		if ( info?$total_bytes )
+	if ( f?$total_bytes )
-			print "total bytes: " + fmt("%s", info$total_bytes);
+		print "total bytes: " + fmt("%s", f$total_bytes);
-		if ( info?$source )
+	if ( f?$source )
-			print "source: " + info$source;
+		print "source: " + f$source;
 
-		for ( act in info$actions )
+	if ( ! f?$info ) return;
-			switch ( act$act ) {
-			case FileAnalysis::ACTION_MD5:
-				if ( info$actions[act]?$md5 )
-					print fmt("MD5: %s", info$actions[act]$md5);
-				break;
-			case FileAnalysis::ACTION_SHA1:
-				if ( info$actions[act]?$sha1 )
-					print fmt("SHA1: %s", info$actions[act]$sha1);
-				break;
-			case FileAnalysis::ACTION_SHA256:
-				if ( info$actions[act]?$sha256 )
-					print fmt("SHA256: %s", info$actions[act]$sha256);
-				break;
-			}
-		break;
-	}
-	}
 
-hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+	if ( f$info?$md5 )
-	&priority=-5
+		print fmt("MD5: %s", f$info$md5);
-	{
+	if ( f$info?$sha1 )
-	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
+		print fmt("SHA1: %s", f$info$sha1);
-
+	if ( f$info?$sha256 )
-	# avoids libmagic variances across systems
+		print fmt("SHA256: %s", f$info$sha256);
-	if ( info?$mime_type )
-		info$mime_type = "set";
-	if ( info?$file_type )
-		info$file_type = "set";
 	}
 
 event bro_init()