Merge branch 'topic/seth/small-files-bof-handling-fix'

* topic/seth/small-files-bof-handling-fix:
  Fix a bug in the core files framework with handling the BOF buffer.

BIT-1310 #merged

CHANGES

@@ -1,4 +1,14 @@
 
+2.3-411 | 2015-02-05 10:05:48 -0600
+
+  * Fix file analysis of files with total size below the bof_buffer size
+    never delivering content to stream analyzers. (Seth Hall)
+
+  * Add/fix log fields in x509 diff canonifier. (Jon Siwek)
+
+  * "id" not defined for debug code when using -DPROFILE_BRO_FUNCTIONS
+    (Mike Smiley)
+
 2.3-406 | 2015-02-03 17:02:45 -0600
 
   * Add x509 canonifier to a unit test. (Jon Siwek)

VERSION

@@ -1 +1 @@
-2.3-406
+2.3-411

src/file_analysis/File.cc

@@ -492,18 +492,22 @@ void File::EndOfFile()
 	if ( done )
 		return;
 
-	if ( ! did_mime_type &&
-	     LookupFieldDefaultCount(missing_bytes_idx) == 0 )
-		DetectMIME();
-
-	analyzers.DrainModifications();
-
 	if ( file_reassembler )
 		{
 		file_reassembler->Flush();
-		analyzers.DrainModifications();
 		}
 
+	// Mark the bof_buffer as full in case it isn't yet
+	// so that the whole thing can be flushed out to
+	// any stream analyzers.
+	if ( ! bof_buffer.full )
+		{
+		bof_buffer.full = true;
+		DeliverStream((const u_char*) "", 0);
+		}
+
+	analyzers.DrainModifications();
+
 	done = true;
 
 	file_analysis::Analyzer* a = 0;

testing/btest/Baseline/scripts.base.frameworks.file-analysis.big-bof-buffer/files.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	files
+#open	2015-02-05-13-55-41
+#fields	ts	fuid	tx_hosts	rx_hosts	conn_uids	source	depth	analyzers	mime_type	filename	duration	local_orig	is_orig	seen_bytes	total_bytes	missing_bytes	overflow_bytes	timedout	parent_fuid	md5	sha1	sha256	extracted
+#types	time	string	set[addr]	set[addr]	set[string]	string	count	set[string]	string	string	interval	bool	bool	count	count	count	count	bool	string	string	string	string	string
+1362692527.009512	FakNcS1Jfe01uljb3	192.150.187.43	141.142.228.5	CXWv6p3arKYeMETxOg	HTTP	0	MD5,SHA1	text/plain	-	0.000263	-	F	4705	4705	0	0	F	-	397168fd09991a0e712254df7bc639ac	1dd7ac0398df6cbc0696445a91ec681facf4dc47	-	-
+#close	2015-02-05-13-55-41

testing/btest/scripts/base/frameworks/file-analysis/big-bof-buffer.bro

@@ -0,0 +1,6 @@
+# @TEST-EXEC: bro -r $TRACES/http/get.trace %INPUT
+# @TEST-EXEC: btest-diff files.log
+
+@load frameworks/files/hash-all-files
+
+redef default_file_bof_buffer_size=5000;