Merge remote-tracking branch 'origin/topic/seth/faf-updates'

* origin/topic/seth/faf-updates: (27 commits)
  Undoing the FTP tests I updated earlier.
  Update the last two btest FAF tests.
  File analysis fixes and test updates.
  Fix a bug with getting analyzer tags.
  A few test updates.
  Some tests work now (at least they all don't fail anymore!)
  Forgot a file.
  Added protocol description functions that provide a super compressed log representation.
  Fix a bug where orig file information in http wasn't working right.
  Added mime types to http.log
  Clean up queued but unused file_over_new_connections event args.
  Add jar files to the default MHR lookups.
  Adding CAB files for MHR checking.
  Improve malware hash registry script.
  Fix a small issue with finding smtp entities.
  Added support for files to the notice framework.
  Make the custom libmagic database a git submodule.
  Add an is_orig parameter to file_over_new_connection event.
  Make magic for emitting application/msword mime type less strict.
  Disable more libmagic builtin checks that override the magic database.
  ...

Conflicts:
	doc/scripts/DocSourcesList.cmake
	scripts/base/init-bare.bro
	scripts/test-all-policy.bro
	testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
	testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

src/file_analysis/Analyzer.h

@@ -96,7 +96,7 @@ public:
 	 */
 	static FA_Tag ArgsTag(const RecordVal* args)
 		{
-		using BifType::Record::FileAnalysis::AnalyzerArgs;
+		using BifType::Record::Files::AnalyzerArgs;
 		return args->Lookup(AnalyzerArgs->FieldOffset("tag"))->AsEnum();
 		}
 

src/file_analysis/AnalyzerSet.cc

@@ -15,7 +15,7 @@ static void analyzer_del_func(void* v)
 AnalyzerSet::AnalyzerSet(File* arg_file) : file(arg_file)
 	{
 	TypeList* t = new TypeList();
-	t->Append(BifType::Record::FileAnalysis::AnalyzerArgs->Ref());
+	t->Append(BifType::Record::Files::AnalyzerArgs->Ref());
 	analyzer_hash = new CompositeHash(t);
 	Unref(t);
 	analyzer_map.SetDeleteFunc(analyzer_del_func);

src/file_analysis/File.cc

@@ -75,7 +75,8 @@ void File::StaticInit()
 File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
            bool is_orig)
 	: id(file_id), val(0), postpone_timeout(false), first_chunk(true),
-	  missed_bof(false), need_reassembly(false), done(false), analyzers(this)
+	  missed_bof(false), need_reassembly(false), done(false),
+	  did_file_new_event(false), analyzers(this)
 	{
 	StaticInit();
 
@@ -89,7 +90,7 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
 		// add source, connection, is_orig fields
 		SetSource(analyzer_mgr->GetAnalyzerName(tag));
 		val->Assign(is_orig_idx, new Val(is_orig, TYPE_BOOL));
-		UpdateConnectionFields(conn);
+		UpdateConnectionFields(conn, is_orig);
 		}
 
 	UpdateLastActivityTime();
@@ -99,6 +100,13 @@ File::~File()
 	{
 	DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str());
 	Unref(val);
+
+	// Queue may not be empty in the case where only content gaps were seen.
+	while ( ! fonc_queue.empty() )
+		{
+		delete_vals(fonc_queue.front().second);
+		fonc_queue.pop();
+		}
 	}
 
 void File::UpdateLastActivityTime()
@@ -111,18 +119,15 @@ double File::GetLastActivityTime() const
 	return val->Lookup(last_active_idx)->AsTime();
 	}
 
-void File::UpdateConnectionFields(Connection* conn)
+void File::UpdateConnectionFields(Connection* conn, bool is_orig)
 	{
 	if ( ! conn )
 		return;
 
 	Val* conns = val->Lookup(conns_idx);
 
-	bool is_first = false;
-
 	if ( ! conns )
 		{
-		is_first = true;
 		conns = empty_connection_table();
 		val->Assign(conns_idx, conns);
 		}
@@ -133,12 +138,18 @@ void File::UpdateConnectionFields(Connection* conn)
 		Val* conn_val = conn->BuildConnVal();
 		conns->AsTableVal()->Assign(idx, conn_val);
 
-		if ( ! is_first && FileEventAvailable(file_over_new_connection) )
+		if ( FileEventAvailable(file_over_new_connection) )
 			{
 			val_list* vl = new val_list();
 			vl->append(val->Ref());
 			vl->append(conn_val->Ref());
-			FileEvent(file_over_new_connection, vl);
+			vl->append(new Val(is_orig, TYPE_BOOL));
+
+			if ( did_file_new_event )
+				FileEvent(file_over_new_connection, vl);
+			else
+				fonc_queue.push(pair<EventHandlerPtr, val_list*>(
+				        file_over_new_connection, vl));
 			}
 		}
 
@@ -435,6 +446,18 @@ void File::FileEvent(EventHandlerPtr h, val_list* vl)
 	{
 	mgr.QueueEvent(h, vl);
 
+	if ( h == file_new )
+		{
+		did_file_new_event = true;
+
+		while ( ! fonc_queue.empty() )
+			{
+			pair<EventHandlerPtr, val_list*> p = fonc_queue.front();
+			mgr.QueueEvent(p.first, p.second);
+			fonc_queue.pop();
+			}
+		}
+
 	if ( h == file_new || h == file_timeout )
 		{
 		// immediate feedback is required for these events.

src/file_analysis/File.h

@@ -3,7 +3,9 @@
 #ifndef FILE_ANALYSIS_FILE_H
 #define FILE_ANALYSIS_FILE_H
 
+#include <queue>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "Conn.h"
@@ -171,8 +173,9 @@ protected:
 	 * Updates the "conn_ids" and "conn_uids" fields in #val record with the
 	 * \c conn_id and UID taken from \a conn.
 	 * @param conn the connection over which a part of the file has been seen.
+	 * @param is_orig true if the connection originator is sending the file.
 	 */
-	void UpdateConnectionFields(Connection* conn);
+	void UpdateConnectionFields(Connection* conn, bool is_orig);
 
 	/**
 	 * Increment a byte count field of #val record by \a size.
@@ -239,7 +242,9 @@ private:
 	bool missed_bof;           /**< Flags that we missed start of file. */
 	bool need_reassembly;      /**< Whether file stream reassembly is needed. */
 	bool done;                 /**< If this object is about to be deleted. */
+	bool did_file_new_event;   /**< Whether the file_new event has been done. */
 	AnalyzerSet analyzers;     /**< A set of attached file analyzer. */
+	queue<pair<EventHandlerPtr, val_list*> > fonc_queue;
 
 	struct BOF_Buffer {
 		BOF_Buffer() : full(false), replayed(false), size(0) {}

src/file_analysis/Manager.cc

@@ -19,8 +19,8 @@ string Manager::salt;
 
 Manager::Manager()
 	{
-	tag_enum_type = new EnumType("FileAnalysis::Tag");
-	::ID* id = install_ID("Tag", "FileAnalysis", true, true);
+	tag_enum_type = new EnumType("Files::Tag");
+	::ID* id = install_ID("Tag", "Files", true, true);
 	add_type(id, tag_enum_type, 0, 0);
 	}
 
@@ -42,7 +42,7 @@ void Manager::RegisterAnalyzerComponent(Component* component)
 	{
 	const char* cname = component->CanonicalName();
 
-	if ( tag_enum_type->Lookup("FileAnalysis", cname) != -1 )
+	if ( tag_enum_type->Lookup("Files", cname) != -1 )
 		reporter->FatalError("File Analyzer %s defined more than once", cname);
 
 	DBG_LOG(DBG_FILE_ANALYSIS, "Registering analyzer %s (tag %s)",
@@ -54,7 +54,7 @@ void Manager::RegisterAnalyzerComponent(Component* component)
 	        component->Tag().AsEnumVal()->InternalInt(), component));
 
 	string id = fmt("ANALYZER_%s", cname);
-	tag_enum_type->AddName("FileAnalysis", id.c_str(),
+	tag_enum_type->AddName("Files", id.c_str(),
 						   component->Tag().AsEnumVal()->InternalInt(), true);
 	}
 
@@ -76,7 +76,7 @@ void Manager::Terminate()
 string Manager::HashHandle(const string& handle) const
 	{
 	if ( salt.empty() )
-		salt = BifConst::FileAnalysis::salt->CheckString();
+		salt = BifConst::Files::salt->CheckString();
 
 	char tmp[20];
 	uint64 hash[2];
@@ -249,7 +249,7 @@ File* Manager::GetFile(const string& file_id, Connection* conn,
 		rval->UpdateLastActivityTime();
 
 		if ( update_conn )
-			rval->UpdateConnectionFields(conn);
+			rval->UpdateConnectionFields(conn, is_orig);
 		}
 
 	return rval;
@@ -351,7 +351,7 @@ void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig)
 bool Manager::IsDisabled(analyzer::Tag tag)
 	{
 	if ( ! disabled )
-		disabled = internal_const_val("FileAnalysis::disable")->AsTableVal();
+		disabled = internal_const_val("Files::disable")->AsTableVal();
 
 	Val* index = new Val(tag, TYPE_COUNT);
 	Val* yield = disabled->Lookup(index);

src/file_analysis/analyzer/data_event/DataEvent.cc

@@ -17,7 +17,7 @@ DataEvent::DataEvent(RecordVal* args, File* file,
 
 file_analysis::Analyzer* DataEvent::Instantiate(RecordVal* args, File* file)
 	{
-	using BifType::Record::FileAnalysis::AnalyzerArgs;
+	using BifType::Record::Files::AnalyzerArgs;
 
 	int chunk_off = AnalyzerArgs->FieldOffset("chunk_event");
 	int stream_off = AnalyzerArgs->FieldOffset("stream_event");

src/file_analysis/analyzer/extract/Extract.cc

@@ -29,7 +29,7 @@ Extract::~Extract()
 
 file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file)
 	{
-	using BifType::Record::FileAnalysis::AnalyzerArgs;
+	using BifType::Record::Files::AnalyzerArgs;
 	Val* v = args->Lookup(AnalyzerArgs->FieldOffset("extract_filename"));
 
 	if ( ! v )

src/file_analysis/analyzer/hash/events.bif

@@ -7,6 +7,6 @@
 ##
 ## hash: The result of the hashing.
 ##
-## .. bro:see:: FileAnalysis::add_analyzer FileAnalysis::ANALYZER_MD5
-##    FileAnalysis::ANALYZER_SHA1 FileAnalysis::ANALYZER_SHA256
+## .. bro:see:: Files::add_analyzer Files::ANALYZER_MD5
+##    Files::ANALYZER_SHA1 Files::ANALYZER_SHA256
 event file_hash%(f: fa_file, kind: string, hash: string%);

src/file_analysis/file_analysis.bif

@@ -1,6 +1,6 @@
 ##! Internal functions and types used by the file analysis framework.
 
-module FileAnalysis;
+module Files;
 
 %%{
 #include "file_analysis/Manager.h"
@@ -8,40 +8,46 @@ module FileAnalysis;
 
 type AnalyzerArgs: record;
 
-## :bro:see:`FileAnalysis::set_timeout_interval`.
-function FileAnalysis::__set_timeout_interval%(file_id: string, t: interval%): bool
+## :bro:see:`Files::set_timeout_interval`.
+function Files::__set_timeout_interval%(file_id: string, t: interval%): bool
 	%{
 	bool result = file_mgr->SetTimeoutInterval(file_id->CheckString(), t);
 	return new Val(result, TYPE_BOOL);
 	%}
 
-## :bro:see:`FileAnalysis::add_analyzer`.
-function FileAnalysis::__add_analyzer%(file_id: string, args: any%): bool
+## :bro:see:`Files::add_analyzer`.
+function Files::__add_analyzer%(file_id: string, args: any%): bool
 	%{
-	using BifType::Record::FileAnalysis::AnalyzerArgs;
+	using BifType::Record::Files::AnalyzerArgs;
 	RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
 	bool result = file_mgr->AddAnalyzer(file_id->CheckString(), rv);
 	Unref(rv);
 	return new Val(result, TYPE_BOOL);
 	%}
 
-## :bro:see:`FileAnalysis::remove_analyzer`.
-function FileAnalysis::__remove_analyzer%(file_id: string, args: any%): bool
+## :bro:see:`Files::remove_analyzer`.
+function Files::__remove_analyzer%(file_id: string, args: any%): bool
 	%{
-	using BifType::Record::FileAnalysis::AnalyzerArgs;
+	using BifType::Record::Files::AnalyzerArgs;
 	RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
 	bool result = file_mgr->RemoveAnalyzer(file_id->CheckString(), rv);
 	Unref(rv);
 	return new Val(result, TYPE_BOOL);
 	%}
 
-## :bro:see:`FileAnalysis::stop`.
-function FileAnalysis::__stop%(file_id: string%): bool
+## :bro:see:`Files::stop`.
+function Files::__stop%(file_id: string%): bool
 	%{
 	bool result = file_mgr->IgnoreFile(file_id->CheckString());
 	return new Val(result, TYPE_BOOL);
 	%}
 
+## :bro:see:`Files::analyzer_name`.
+function Files::__analyzer_name%(tag: Files::Tag%) : string
+	%{
+	return new StringVal(file_mgr->GetAnalyzerName(tag->InternalInt()));
+	%}
+
 module GLOBAL;
 
 ## For use within a :bro:see:`get_file_handle` handler to set a unique
@@ -58,4 +64,4 @@ function set_file_handle%(handle: string%): any
 	return 0;
 	%}
 
-const FileAnalysis::salt: string;
+const Files::salt: string;