FileAnalysis: remove some file events.

The file_new event now takes over the function of file_type, file_bof,
and file_bof_buffer.

scripts/base/init-bare.bro

@@ -316,6 +316,9 @@ type connection: record {
 	tunnel: EncapsulatingConnVector &optional;
 };
 
+const default_file_timeout_interval: interval = 2 mins &redef;
+const default_file_bof_buffer_size: count = 1024 &redef;
+
 ## A file that Bro is analyzing.  This is Bro's type for describing the basic
 ## internal metadata collected about a "file", which is essentially just a
 ## byte stream that is e.g. pulled from a network connection or possibly
@@ -356,11 +359,11 @@ type fa_file: record {
 
 	## The amount of time between receiving new data for this file that
 	## the analysis engine will wait before giving up on it.
-	timeout_interval: interval &default=2mins;
+	timeout_interval: interval &default=default_file_timeout_interval;
 
 	## The number of bytes at the beginning of a file to save for later
 	## inspection in *bof_buffer* field.
-	bof_buffer_size: count &default=1024;
+	bof_buffer_size: count &default=default_file_bof_buffer_size;
 
 	## The content of the beginning of a file up to *bof_buffer_size* bytes.
 	## This is also the buffer that's used for file/mime type detection.

scripts/base/protocols/ftp/file-extract.bro

@@ -24,15 +24,26 @@ redef record Info += {
 	extract_file:          bool &default=F;
 };
 
+function get_extraction_name(f: fa_file): string
+	{
+	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
+	++extract_count;
+	return r;
+	}
+
 event file_new(f: fa_file) &priority=5
 	{
 	if ( ! f?$source ) return;
 	if ( f$source != "FTP_DATA" ) return;
-	if ( ! f?$conns ) return;
 
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
-	                          extract_count);
-	local extracting: bool = F;
+	if ( f?$mime_type && extract_file_types in f$mime_type )
+		{
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
+		                             $extract_filename=get_extraction_name(f)]);
+		return;
+		}
+
+	if ( ! f?$conns ) return;
 
 	for ( cid in f$conns )
 		{
@@ -44,31 +55,10 @@ event file_new(f: fa_file) &priority=5
 
 		if ( ! s$extract_file ) next;
 
-		if ( ! extracting )
-			{
 		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
-			                             $extract_filename=fname]);
-			extracting = T;
-			++extract_count;
-			}
-		}
-	}
-
-event file_type(f: fa_file) &priority=5
-	{
-	if ( ! f?$mime_type ) return;
-	if ( ! f?$source ) return;
-	if ( f$source != "FTP_DATA" ) return;
-	if ( extract_file_types !in f$mime_type ) return;
-
-	if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
+		                             $extract_filename=get_extraction_name(f)]);
 		return;
-
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
-	                          extract_count);
-	++extract_count;
-	FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
-	                             $extract_filename=fname]);
+		}
 	}
 
 event file_state_remove(f: fa_file) &priority=4

scripts/base/protocols/http/file-extract.bro

@@ -25,32 +25,11 @@ export {
 
 global extract_count: count = 0;
 
-event file_type(f: fa_file) &priority=5
+function get_extraction_name(f: fa_file): string
 	{
-	if ( ! f?$mime_type ) return;
-	if ( ! f?$source ) return;
-	if ( f$source != "HTTP" ) return;
-	if ( extract_file_types !in f$mime_type ) return;
-
-	if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
-		return;
-
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
-	                          extract_count);
+	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
 	++extract_count;
-	FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
-	                             $extract_filename=fname]);
-
-	if ( ! f?$conns ) return;
-
-	for ( cid in f$conns )
-		{
-		local c: connection = f$conns[cid];
-
-		if ( ! c?$http ) next;
-
-		c$http$extraction_file = fname;
-		}
+	return r;
 	}
 
 event file_new(f: fa_file) &priority=5
@@ -59,27 +38,47 @@ event file_new(f: fa_file) &priority=5
 	if ( f$source != "HTTP" ) return;
 	if ( ! f?$conns ) return;
 
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
-	                          extract_count);
+	local fname: string;
+	local c: connection;
+
+	if ( f?$mime_type && extract_file_types in f$mime_type )
+		{
+		fname = get_extraction_name(f);
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
+		                             $extract_filename=fname]);
+
+		for ( cid in f$conns )
+			{
+			c = f$conns[cid];
+			if ( ! c?$http ) next;
+			c$http$extraction_file = fname;
+			}
+
+		return;
+		}
+
 	local extracting: bool = F;
 
 	for ( cid in f$conns )
 		{
-		local c: connection = f$conns[cid];
+		c = f$conns[cid];
 
 		if ( ! c?$http ) next;
 
-		if ( c$http$extract_file )
-			{
-			if ( ! extracting )
-				{
+		if ( ! c$http$extract_file ) next;
+
+		fname = get_extraction_name(f);
 		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
 		                             $extract_filename=fname]);
 		extracting = T;
-				++extract_count;
+		break;
 		}
 
+	if ( extracting )
+		for ( cid in f$conns )
+			{
+			c = f$conns[cid];
+			if ( ! c?$http ) next;
 			c$http$extraction_file = fname;
 			}
 	}
-	}

scripts/base/protocols/http/file-hash.bro

@@ -23,30 +23,31 @@ export {
 	                   &redef;
 }
 
-event file_type(f: fa_file) &priority=5
+event file_new(f: fa_file) &priority=5
 	{
-	if ( ! f?$mime_type ) return;
 	if ( ! f?$source ) return;
 	if ( f$source != "HTTP" ) return;
 
-	if ( generate_md5 in f$mime_type )
-		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
-	else if ( f?$conns )
+	if ( f?$mime_type && generate_md5 in f$mime_type )
 		{
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
+		return;
+		}
+
+	if ( ! f?$conns ) return;
+
 	for ( cid in f$conns )
 		{
 		local c: connection = f$conns[cid];
 
 		if ( ! c?$http ) next;
 
-			if ( c$http$calc_md5 )
-				{
+		if ( ! c$http$calc_md5 ) next;
+
 		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
 		return;
 		}
 	}
-		}
-	}
 
 event file_state_remove(f: fa_file) &priority=4
 	{

scripts/base/protocols/http/file-ident.bro

@@ -34,11 +34,11 @@ export {
 	const ignored_incorrect_file_type_urls = /^$/ &redef;
 }
 
-event file_type(f: fa_file) &priority=5
+event file_new(f: fa_file) &priority=5
 	{
-	if ( ! f?$mime_type ) return;
 	if ( ! f?$source ) return;
 	if ( f$source != "HTTP" ) return;
+	if ( ! f?$mime_type ) return;
 	if ( ! f?$conns ) return;
 
 	for ( cid in f$conns )
@@ -68,9 +68,9 @@ event file_type(f: fa_file) &priority=5
 
 event file_over_new_connection(f: fa_file) &priority=5
 	{
-	if ( ! f?$mime_type ) return;
 	if ( ! f?$source ) return;
 	if ( f$source != "HTTP" ) return;
+	if ( ! f?$mime_type ) return;
 	if ( ! f?$conns ) return;
 
 	# Spread the mime around (e.g. for partial content, file_type event only
@@ -80,9 +80,7 @@ event file_over_new_connection(f: fa_file) &priority=5
 	for ( cid in f$conns )
 		{
 		local c: connection = f$conns[cid];
-
 		if ( ! c?$http ) next;
-
 		c$http$mime_type = f$mime_type;
 		}
 	}

scripts/base/protocols/irc/dcc-send.bro

@@ -41,38 +41,6 @@ global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins;
 
 global extract_count: count = 0;
 
-event file_new(f: fa_file) &priority=5
-	{
-	if ( ! f?$source ) return;
-	if ( f$source != "IRC_DATA" ) return;
-	if ( ! f?$conns ) return;
-
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
-	                          extract_count);
-	local extracting: bool = F;
-
-	for ( cid in f$conns )
-		{
-		local c: connection = f$conns[cid];
-
-		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
-
-		local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
-
-		if ( ! s$extract_file ) next;
-
-		if ( ! extracting )
-			{
-			FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
-			                             $extract_filename=fname]);
-			extracting = T;
-			++extract_count;
-			}
-
-		s$extraction_file = fname;
-		}
-	}
-
 function set_dcc_mime(f: fa_file)
 	{
 	if ( ! f?$conns ) return;
@@ -105,6 +73,60 @@ function set_dcc_extraction_file(f: fa_file, filename: string)
 		}
 	}
 
+function get_extraction_name(f: fa_file): string
+	{
+	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
+	++extract_count;
+	return r;
+	}
+
+# this handler sets the IRC::Info mime type
+event file_new(f: fa_file) &priority=5
+	{
+	if ( ! f?$source ) return;
+	if ( f$source != "IRC_DATA" ) return;
+	if ( ! f?$mime_type ) return;
+
+	set_dcc_mime(f);
+	}
+
+# this handler check if file extraction is desired
+event file_new(f: fa_file) &priority=5
+	{
+	if ( ! f?$source ) return;
+	if ( f$source != "IRC_DATA" ) return;
+
+	local fname: string;
+
+	if ( f?$mime_type && extract_file_types in f$mime_type )
+		{
+		fname = get_extraction_name(f);
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
+		                             $extract_filename=fname]);
+		set_dcc_extraction_file(f, fname);
+		return;
+		}
+
+	if ( ! f?$conns ) return;
+
+	for ( cid in f$conns )
+		{
+		local c: connection = f$conns[cid];
+
+		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
+
+		local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
+
+		if ( ! s$extract_file ) next;
+
+		fname = get_extraction_name(f);
+		FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
+		                             $extract_filename=fname]);
+		s$extraction_file = fname;
+		return;
+		}
+	}
+
 function log_dcc(f: fa_file)
 	{
 	if ( ! f?$conns ) return;
@@ -134,28 +156,7 @@ function log_dcc(f: fa_file)
 		}
 	}
 
-event file_type(f: fa_file) &priority=5
-	{
-	if ( ! f?$mime_type ) return;
-	if ( ! f?$source ) return;
-	if ( f$source != "IRC_DATA" ) return;
-
-	set_dcc_mime(f);
-
-	if ( extract_file_types !in f$mime_type ) return;
-
-	if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
-		return;
-
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
-	                          extract_count);
-	++extract_count;
-	FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
-	                             $extract_filename=fname]);
-	set_dcc_extraction_file(f, fname);
-	}
-
-event file_type(f: fa_file) &priority=-5
+event file_new(f: fa_file) &priority=-5
 	{
 	if ( ! f?$source ) return;
 	if ( f$source != "IRC_DATA" ) return;

scripts/base/protocols/smtp/entities-excerpt.bro

@@ -12,7 +12,8 @@ export {
 	};
 	
 	## This is the default value for how much of the entity body should be
-	## included for all MIME entities.
+	## included for all MIME entities.  The lesser of this value and
+	## :bro:see:`default_file_bof_buffer_size` will be used.
 	const default_entity_excerpt_len = 0 &redef;
 }
 
@@ -20,16 +21,7 @@ event file_new(f: fa_file) &priority=5
 	{
 	if ( ! f?$source ) return;
 	if ( f$source != "SMTP" ) return;
-
-	if ( default_entity_excerpt_len > f$bof_buffer_size )
-		f$bof_buffer_size = default_entity_excerpt_len;
-	}
-
-event file_bof_buffer(f: fa_file) &priority=5
-	{
 	if ( ! f?$bof_buffer ) return;
-	if ( ! f?$source ) return;
-	if ( f$source != "SMTP" ) return;
 	if ( ! f?$conns ) return;
 
 	for ( cid in f$conns )

scripts/base/protocols/smtp/entities.bro

@@ -88,6 +88,13 @@ function set_session(c: connection, new_entity: bool)
 		}
 	}
 
+function get_extraction_name(f: fa_file): string
+	{
+	local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
+	++extract_count;
+	return r;
+	}
+
 event mime_begin_entity(c: connection) &priority=10
 	{
 	if ( ! c?$smtp ) return;
@@ -101,8 +108,7 @@ event file_new(f: fa_file) &priority=5
 	if ( f$source != "SMTP" ) return;
 	if ( ! f?$conns ) return;
 
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
-	                          extract_count);
+	local fname: string;
 	local extracting: bool = F;
 
 	for ( cid in f$conns )
@@ -116,6 +122,7 @@ event file_new(f: fa_file) &priority=5
 			{
 			if ( ! extracting )
 				{
+				fname = get_extraction_name(f);
 				FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
 				                             $extract_filename=fname]);
 				extracting = T;
@@ -137,9 +144,7 @@ function check_extract_by_type(f: fa_file)
 	if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
 		return;
 
-	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
-	                          extract_count);
-	++extract_count;
+	local fname: string = get_extraction_name(f);
 	FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
 	                             $extract_filename=fname]);
 
@@ -148,9 +153,7 @@ function check_extract_by_type(f: fa_file)
 	for ( cid in f$conns )
 		{
 		local c: connection = f$conns[cid];
-
 		if ( ! c?$smtp ) next;
-
 		c$smtp$current_entity$extraction_file = fname;
 		}
 	}
@@ -163,11 +166,11 @@ function check_md5_by_type(f: fa_file)
 	FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
 	}
 
-event file_type(f: fa_file) &priority=5
+event file_new(f: fa_file) &priority=5
 	{
-	if ( ! f?$mime_type ) return;
 	if ( ! f?$source ) return;
 	if ( f$source != "SMTP" ) return;
+	if ( ! f?$mime_type ) return;
 
 	if ( f?$conns )
 		for ( cid in f$conns )

src/event.bif

@@ -7002,11 +7002,6 @@ event file_new%(f: fa_file%);
 # TODO: give the new connection
 event file_over_new_connection%(f: fa_file%);
 event file_timeout%(f: fa_file%);
-event file_bof%(f: fa_file%);
-# TODO: give buffer? (unless we remove the event completely)
-event file_bof_buffer%(f: fa_file%);
-# TODO: give mime type? (unless we remove the event completely)
-event file_type%(f: fa_file%);
 # TODO: give size of gap
 event file_gap%(f: fa_file%);
 

src/file_analysis/File.cc

@@ -80,7 +80,7 @@ void File::StaticInit()
 
 File::File(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
     : id(""), unique(unique), val(0), postpone_timeout(false),
-      first_chunk(true), need_type(false), need_reassembly(false), done(false),
+      first_chunk(true), missed_bof(false), need_reassembly(false), done(false),
       actions(this)
 	{
 	StaticInit();
@@ -221,9 +221,6 @@ bool File::BufferBOF(const u_char* data, uint64 len)
 	{
 	if ( bof_buffer.full || bof_buffer.replayed ) return false;
 
-	if ( bof_buffer.chunks.size() == 0 )
-		file_mgr->FileEvent(file_bof, this);
-
 	uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx);
 
 	bof_buffer.chunks.push_back(new BroString(data, len, 0));
@@ -260,18 +257,17 @@ void File::ReplayBOF()
 	if ( bof_buffer.chunks.empty() )
 		{
 		// Since we missed the beginning, try file type detect on next data in.
-		need_type = true;
+		missed_bof = true;
 		return;
 		}
 
 	BroString* bs = concatenate(bof_buffer.chunks);
 	val->Assign(bof_buffer_idx, new StringVal(bs));
-	bool have_type = DetectTypes(bs->Bytes(), bs->Len());
 
-	file_mgr->FileEvent(file_bof_buffer, this);
+	DetectTypes(bs->Bytes(), bs->Len());
 
-	if ( have_type )
-		file_mgr->FileEvent(file_type, this);
+	file_mgr->FileEvent(file_new, this);
+	//mgr.Drain();
 
 	for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i )
 		DataIn(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len());
@@ -283,12 +279,11 @@ void File::DataIn(const u_char* data, uint64 len, uint64 offset)
 
 	if ( first_chunk )
 		{
-		if ( DetectTypes(data, len) )
-			{
-			file_mgr->FileEvent(file_type, this);
+		// TODO: this should all really be delayed until we attempt reassembly
+		DetectTypes(data, len);
+		file_mgr->FileEvent(file_new, this);
+		//mgr.Drain();
 		actions.DrainModifications();
-			}
-
 		first_chunk = false;
 		}
 
@@ -320,15 +315,13 @@ void File::DataIn(const u_char* data, uint64 len)
 
 	if ( BufferBOF(data, len) ) return;
 
-	if ( need_type )
+	if ( missed_bof )
 		{
-		if ( DetectTypes(data, len) )
-			{
-			file_mgr->FileEvent(file_type, this);
+		DetectTypes(data, len);
+		file_mgr->FileEvent(file_new, this);
+		//mgr.Drain();
 		actions.DrainModifications();
-			}
-
-		need_type = false;
+		missed_bof = false;
 		}
 
 	Action* act = 0;

src/file_analysis/File.h

@@ -155,12 +155,12 @@ protected:
 	 */
 	bool DetectTypes(const u_char* data, uint64 len);
 
-	FileID id;                 /**< A pretty hash that likely identifies file*/
+	FileID id;                 /**< A pretty hash that likely identifies file */
 	string unique;             /**< A string that uniquely identifies file */
 	RecordVal* val;            /**< \c fa_file from script layer. */
 	bool postpone_timeout;     /**< Whether postponing timeout is requested. */
 	bool first_chunk;          /**< Track first non-linear chunk. */
-	bool need_type;            /**< Flags next data input to be magic typed. */
+	bool missed_bof;           /**< Flags that we missed start of file. */
 	bool need_reassembly;      /**< Whether file stream reassembly is needed. */
 	bool done;                 /**< If this object is about to be deleted. */
 	ActionSet actions;

src/file_analysis/Manager.cc

@@ -221,7 +221,6 @@ File* Manager::GetFile(const string& unique, Connection* conn,
 			}
 
 		id_map[id] = rval;
-		FileEvent(file_new, rval);
 		rval->ScheduleInactivityTimer();
 		if ( IsIgnored(unique) ) return 0;
 		}

testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out

@@ -1,6 +1,5 @@
 FILE_NEW
 Cx92a0ym5R8, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 ^J0.26 | 201
 FILE_TYPE

testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out

@@ -1,6 +1,5 @@
 FILE_NEW
 Cx92a0ym5R8, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 ^J0.26 | 201
 FILE_TYPE

testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out

@@ -1,2 +1,7 @@
 FILE_NEW
 Cx92a0ym5R8, 0, 0
+FILE_BOF_BUFFER
+^J0.26 | 201
+FILE_TYPE
+file type is set
+mime type is set

testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out

@@ -1,6 +1,5 @@
 FILE_NEW
 sidhzrR4IT8, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 The Nationa
 FILE_TYPE

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out

@@ -1,6 +1,5 @@
 FILE_NEW
 kg59rqyYxN, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 {^J  "origin
 FILE_TYPE

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out

@@ -1,6 +1,5 @@
 FILE_NEW
 Cx92a0ym5R8, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 ^J0.26 | 201
 FILE_TYPE

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out

@@ -1,6 +1,5 @@
 FILE_NEW
 aFQKI8SPOL2, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 /*^J********
 FILE_TYPE
@@ -15,7 +14,6 @@ SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706
 SHA256: 5b037a2c5e36f56e63a3012c73e46a04b27741d8ff8f8b62c832fb681fc60f42
 FILE_NEW
 CCU3vUEr06l, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 //-- Google
 FILE_TYPE
@@ -30,7 +28,6 @@ SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983
 SHA256: 6a509fd05aa7c8fa05080198894bb19e638554ffcee0e0b3d7bc8ff54afee1da
 FILE_NEW
 HCzA0dVwDPj, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 GIF89a^D\0^D\0\xb3
 FILE_TYPE
@@ -46,7 +43,6 @@ SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e
 SHA256: 6fb22aa9d780ea63bd7a2e12b92b16fcbf1c4874f1d3e11309a5ba984433c315
 FILE_NEW
 a1Zu1fteVEf, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 \x89PNG^M^J^Z^J\0\0\0
 FILE_TYPE
@@ -62,7 +58,6 @@ SHA1: 560eab5a0177246827a94042dd103916d8765ac7
 SHA256: e0b4500c1fd1d675da4137461cbe64d3c8489f4180d194e47683b20e7fb876f4
 FILE_NEW
 xXlF7wFdsR, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 \x89PNG^M^J^Z^J\0\0\0
 FILE_TYPE

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out

@@ -1,6 +1,5 @@
 FILE_NEW
 v5HLI7MxPQh, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 hello world
 FILE_TYPE
@@ -16,7 +15,6 @@ SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
 SHA256: b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9
 FILE_NEW
 PZS1XGHkIf1, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 {^J  "origin
 FILE_TYPE

testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout

@@ -1,6 +1,5 @@
 FILE_NEW
 nYgPNGLrZf9, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 #separator 
 FILE_TYPE

testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out

@@ -1,6 +1,5 @@
 FILE_NEW
 wqKMAamJVSb, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 PK^C^D^T\0\0\0^H\0\xae
 FILE_TYPE

testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out

@@ -1,6 +1,5 @@
 FILE_NEW
 cwR7l6Zctxb, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 Hello^M^J^M^J ^M
 FILE_TYPE
@@ -15,7 +14,6 @@ SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13
 SHA256: 785a8a044d1454ec88837108f443bbb30cc4f529393ffd57118261036bfe59f5
 FILE_NEW
 ZAOEQmRyxv1, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 <html xmlns
 FILE_TYPE
@@ -30,7 +28,6 @@ SHA1: e54af6c6616525611364b80bd6557a7ea21dae94
 SHA256: b9556e92ddbe52379b64804136f830d111cafe7fcd78e54817fe40f3bc24268d
 FILE_NEW
 Ltd7QO7jEv3, 0, 0
-FILE_BOF
 FILE_BOF_BUFFER
 Version 4.9
 FILE_TYPE

testing/btest/istate/events-ssl.bro

@@ -47,8 +47,9 @@ redef ssl_passphrase = "my-password";
 # a value it received at an earlier time.  So sometimes modifications the sender
 # makes to the value aren't seen on the receiver (in this case, the mime_type
 # field).
-event file_new(f: fa_file)
+event file_new(f: fa_file) &priority=10
 	{
+	delete f$mime_type;
 	FileAnalysis::stop(f);
 	}
 

testing/btest/istate/events.bro

@@ -41,8 +41,9 @@ redef tcp_close_delay = 0secs;
 # of a full value and expect the remote side to associate that unique ID with
 # a value it received at an earlier time.  So sometimes modifications the sender# makes to the value aren't seen on the receiver (in this case, the mime_type
 # field).
-event file_new(f: fa_file)
+event file_new(f: fa_file) &priority=10
 	{
+	delete f$mime_type;
 	FileAnalysis::stop(f);
 	}
 

testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro

@@ -15,11 +15,7 @@ redef test_get_file_name = function(f: fa_file): string
 	};
 
 redef exit_only_after_terminate = T;
-
-event file_new(f: fa_file)
-	{
-	f$timeout_interval = 2sec;
-	}
+redef default_file_timeout_interval = 2sec;
 
 event file_timeout(f: fa_file)
 	{

testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro

@@ -8,7 +8,7 @@ redef test_get_file_name = function(f: fa_file): string
 	return fmt("%s-file", f$id);
 	};
 
-event file_type(f: fa_file)
+event file_new(f: fa_file) &priority=-10
 	{
 	for ( act in test_file_actions )
 		FileAnalysis::remove_action(f, act);

testing/scripts/file-analysis-test.bro

@@ -40,36 +40,14 @@ event file_new(f: fa_file)
 		                             $chunk_event=file_chunk,
 		                             $stream_event=file_stream]);
 		}
-	}
 
-event file_over_new_connection(f: fa_file)
-	{
-	print "FILE_OVER_NEW_CONNECTION";
-	}
-
-event file_timeout(f: fa_file)
-	{
-	print "FILE_TIMEOUT";
-	}
-
-event file_bof(f: fa_file)
-	{
-	print "FILE_BOF";
-	}
-
-event file_bof_buffer(f: fa_file)
+	if ( f?$bof_buffer )
 		{
 		print "FILE_BOF_BUFFER";
-	if ( f?$bof_buffer )
 		print f$bof_buffer[0:10];
 		}
 
-event file_type(f: fa_file) &priority=-5
-	{
-	}
-
-event file_type(f: fa_file)
-	{
+	if ( f?$file_type || f?$mime_type )
 		print "FILE_TYPE";
 	# not actually printing the values due to libmagic variances
 	if ( f?$file_type )
@@ -84,6 +62,16 @@ event file_type(f: fa_file)
 		}
 	}
 
+event file_over_new_connection(f: fa_file)
+	{
+	print "FILE_OVER_NEW_CONNECTION";
+	}
+
+event file_timeout(f: fa_file)
+	{
+	print "FILE_TIMEOUT";
+	}
+
 event file_gap(f: fa_file)
 	{
 	print "FILE_GAP";