Various minor changes related to file mime type detection.

- Improve or just remove some file magic signatures ported from libmagic
  that were too general and matched incorrectly too often.

- Fix MHR script's use of fa_file$mime_type before checking if it's
  initialized.  It may be uninitialized if no signatures match.

- The "fa_file" record now contains a "mime_types" field that contains
  all magic signatures that matched the file content (where the
  "mime_type" field is just a shortcut for the strongest match).

src/file_analysis/File.cc

@@ -53,6 +53,7 @@ int File::timeout_interval_idx = -1;
 int File::bof_buffer_size_idx = -1;
 int File::bof_buffer_idx = -1;
 int File::mime_type_idx = -1;
+int File::mime_types_idx = -1;
 
 void File::StaticInit()
 	{
@@ -73,6 +74,7 @@ void File::StaticInit()
 	bof_buffer_size_idx = Idx("bof_buffer_size");
 	bof_buffer_idx = Idx("bof_buffer");
 	mime_type_idx = Idx("mime_type");
+	mime_types_idx = Idx("mime_types");
 	}
 
 File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
@@ -280,12 +282,15 @@ bool File::BufferBOF(const u_char* data, uint64 len)
 
 bool File::DetectMIME(const u_char* data, uint64 len)
 	{
-	string strongest_match = file_mgr->DetectMIME(data, len);
+	RuleMatcher::MIME_Matches matches;
+	file_mgr->DetectMIME(data, len, &matches);
 
-	if ( strongest_match.empty() )
+	if ( matches.empty() )
 		return false;
 
-	val->Assign(mime_type_idx, new StringVal(strongest_match));
+	val->Assign(mime_type_idx,
+	            new StringVal(*(matches.begin()->second.begin())));
+	val->Assign(mime_types_idx, file_analysis::GenMIMEMatchesVal(matches));
 	return true;
 	}
 

src/file_analysis/File.h

@@ -283,6 +283,7 @@ private:
 	static int bof_buffer_size_idx;
 	static int bof_buffer_idx;
 	static int mime_type_idx;
+	static int mime_types_idx;
 };
 
 } // namespace file_analysis

src/file_analysis/Manager.cc

@@ -425,3 +425,25 @@ string Manager::DetectMIME(const u_char* data, uint64 len) const
 
 	return *(matches.begin()->second.begin());
 	}
+
+VectorVal* file_analysis::GenMIMEMatchesVal(const RuleMatcher::MIME_Matches& m)
+	{
+	VectorVal* rval = new VectorVal(mime_matches);
+
+	for ( RuleMatcher::MIME_Matches::const_iterator it = m.begin();
+	      it != m.end(); ++it )
+		{
+		RecordVal* element = new RecordVal(mime_match);
+
+		for ( set<string>::const_iterator it2 = it->second.begin();
+		      it2 != it->second.end(); ++it2 )
+			{
+			element->Assign(0, new Val(it->first, TYPE_INT));
+			element->Assign(1, new StringVal(*it2));
+			}
+
+		rval->Assign(rval->Size(), element);
+		}
+
+	return rval;
+	}

src/file_analysis/Manager.h

@@ -285,6 +285,7 @@ public:
 	 */
 	std::string DetectMIME(const u_char* data, uint64 len) const;
 
+
 protected:
 	friend class FileTimer;
 
@@ -370,6 +371,12 @@ private:
 	static string salt; /**< A salt added to file handles before hashing. */
 };
 
+/**
+ * Returns a script-layer value corresponding to the \c mime_matches type.
+ * @param m The MIME match information with which to populate the value.
+ */
+VectorVal* GenMIMEMatchesVal(const RuleMatcher::MIME_Matches& m);
+
 } // namespace file_analysis
 
 extern file_analysis::Manager* file_mgr;