Merge remote-tracking branch 'origin/master' into topic/seth/faf-updates

Conflicts:
	scripts/base/frameworks/files/main.bro
	scripts/base/init-bare.bro
	scripts/base/protocols/ftp/file-analysis.bro
	scripts/base/protocols/http/file-analysis.bro
	scripts/base/protocols/irc/file-analysis.bro
	scripts/base/protocols/smtp/file-analysis.bro
	src/const.bif
	src/event.bif
	src/file_analysis/Analyzer.h
	src/file_analysis/file_analysis.bif

src/file_analysis/Analyzer.h

@@ -5,10 +5,13 @@
 
 #include "Val.h"
 #include "NetVar.h"
+#include "analyzer/Tag.h"
+
+#include "file_analysis/file_analysis.bif.h"
 
 namespace file_analysis {
 
-typedef BifEnum::Files::Analyzer FA_Tag;
+typedef int FA_Tag;
 
 class File;
 
@@ -94,8 +97,7 @@ public:
 	static FA_Tag ArgsTag(const RecordVal* args)
 		{
 		using BifType::Record::Files::AnalyzerArgs;
-		return static_cast<FA_Tag>(
-		              args->Lookup(AnalyzerArgs->FieldOffset("tag"))->AsEnum());
+		return args->Lookup(AnalyzerArgs->FieldOffset("tag"))->AsEnum();
 		}
 
 protected:
@@ -114,14 +116,11 @@ protected:
 
 private:
 
-	FA_Tag tag; /**< The particular analyzer type of the analyzer instance. */
-	RecordVal* args; /**< \c AnalyzerArgs val gives tunable analyzer params. */
-	File* file; /**< The file to which the analyzer is attached. */
+	FA_Tag tag;	/**< The particular analyzer type of the analyzer instance. */
+	RecordVal* args;	/**< \c AnalyzerArgs val gives tunable analyzer params. */
+	File* file;	/**< The file to which the analyzer is attached. */
 };
 
-typedef file_analysis::Analyzer* (*AnalyzerInstantiator)(RecordVal* args,
-                                                         File* file);
-
 } // namespace file_analysis
 
 #endif

src/file_analysis/AnalyzerSet.cc

@@ -3,21 +3,10 @@
 #include "AnalyzerSet.h"
 #include "File.h"
 #include "Analyzer.h"
-#include "Extract.h"
-#include "DataEvent.h"
-#include "Hash.h"
+#include "Manager.h"
 
 using namespace file_analysis;
 
-// keep in order w/ declared enum values in file_analysis.bif
-static AnalyzerInstantiator analyzer_factory[] = {
-	file_analysis::Extract::Instantiate,
-	file_analysis::MD5::Instantiate,
-	file_analysis::SHA1::Instantiate,
-	file_analysis::SHA256::Instantiate,
-	file_analysis::DataEvent::Instantiate,
-};
-
 static void analyzer_del_func(void* v)
 	{
 	delete (file_analysis::Analyzer*) v;
@@ -154,14 +143,13 @@ HashKey* AnalyzerSet::GetKey(const RecordVal* args) const
 
 file_analysis::Analyzer* AnalyzerSet::InstantiateAnalyzer(RecordVal* args) const
 	{
-	file_analysis::Analyzer* a =
-	    analyzer_factory[file_analysis::Analyzer::ArgsTag(args)](args, file);
+	FA_Tag tag = file_analysis::Analyzer::ArgsTag(args);
+	file_analysis::Analyzer* a = file_mgr->InstantiateAnalyzer(tag, args, file);
 
 	if ( ! a )
 		{
-		DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %d failed for file id",
-		        " %s", file_analysis::Analyzer::ArgsTag(args),
-		        file->GetID().c_str());
+		reporter->Error("Failed file analyzer %s instantiation for file id %s",
+		                file_mgr->GetAnalyzerName(tag), file->GetID().c_str());
 		return 0;
 		}
 

src/file_analysis/AnalyzerSet.h

@@ -187,7 +187,7 @@ private:
 	};
 
 	typedef queue<Modification*> ModQueue;
-	ModQueue mod_queue; /**< A queue of analyzer additions/removals requests. */
+	ModQueue mod_queue;	/**< A queue of analyzer additions/removals requests. */
 };
 
 } // namespace file_analysiss

src/file_analysis/CMakeLists.txt

@@ -0,0 +1,22 @@
+include(BroSubdir)
+
+include_directories(BEFORE
+                    ${CMAKE_CURRENT_SOURCE_DIR}
+                    ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+add_subdirectory(analyzer)
+
+set(file_analysis_SRCS
+    Manager.cc
+    File.cc
+    FileTimer.cc
+    Analyzer.h
+    AnalyzerSet.cc
+    Component.cc
+)
+
+bif_target(file_analysis.bif)
+
+bro_add_subdir_library(file_analysis ${file_analysis_SRCS} ${BIF_OUTPUT_CC})
+add_dependencies(bro_file_analysis generate_outputs)

src/file_analysis/Component.cc

@@ -0,0 +1,69 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Component.h"
+#include "Manager.h"
+
+#include "../Desc.h"
+#include "../util.h"
+
+using namespace file_analysis;
+
+analyzer::Tag::type_t Component::type_counter = 0;
+
+Component::Component(const char* arg_name, factory_callback arg_factory,
+                     analyzer::Tag::subtype_t arg_subtype)
+	: plugin::Component(plugin::component::FILE_ANALYZER)
+	{
+	name = copy_string(arg_name);
+	canon_name = canonify_name(arg_name);
+	factory = arg_factory;
+
+	tag = analyzer::Tag(++type_counter, arg_subtype);
+	}
+
+Component::Component(const Component& other)
+	: plugin::Component(Type())
+	{
+	name = copy_string(other.name);
+	canon_name = copy_string(other.canon_name);
+	factory = other.factory;
+	tag = other.tag;
+	}
+
+Component::~Component()
+	{
+	delete [] name;
+	delete [] canon_name;
+	}
+
+analyzer::Tag Component::Tag() const
+	{
+	return tag;
+	}
+
+void Component::Describe(ODesc* d)
+	{
+	plugin::Component::Describe(d);
+	d->Add(name);
+	d->Add(" (");
+
+	if ( factory )
+		{
+		d->Add("ANALYZER_");
+		d->Add(canon_name);
+		}
+
+	d->Add(")");
+	}
+
+Component& Component::operator=(const Component& other)
+	{
+	if ( &other != this )
+		{
+		name = copy_string(other.name);
+		factory = other.factory;
+		tag = other.tag;
+		}
+
+	return *this;
+	}

src/file_analysis/Component.h

@@ -0,0 +1,109 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef FILE_ANALYZER_PLUGIN_COMPONENT_H
+#define FILE_ANALYZER_PLUGIN_COMPONENT_H
+
+#include "analyzer/Tag.h"
+#include "plugin/Component.h"
+
+#include "Val.h"
+
+#include "../config.h"
+#include "../util.h"
+
+namespace file_analysis {
+
+class File;
+class Analyzer;
+
+/**
+ * Component description for plugins providing file analyzers.
+ *
+ * A plugin can provide a specific file analyzer by registering this
+ * analyzer component, describing the analyzer.
+ */
+class Component : public plugin::Component {
+public:
+	typedef Analyzer* (*factory_callback)(RecordVal* args, File* file);
+
+	/**
+	 * Constructor.
+	 *
+	 * @param name The name of the provided analyzer. This name is used
+	 * across the system to identify the analyzer, e.g., when calling
+	 * file_analysis::Manager::InstantiateAnalyzer with a name.
+	 *
+	 * @param factory A factory function to instantiate instances of the
+	 * analyzer's class, which must be derived directly or indirectly
+	 * from file_analysis::Analyzer. This is typically a static \c
+	 * Instatiate() method inside the class that just allocates and
+	 * returns a new instance.
+	 *
+	 * @param subtype A subtype associated with this component that
+	 * further distinguishes it. The subtype will be integrated into
+	 * the analyzer::Tag that the manager associates with this analyzer,
+	 * and analyzer instances can accordingly access it via analyzer::Tag().
+	 * If not used, leave at zero.
+	 */
+	Component(const char* name, factory_callback factory,
+	          analyzer::Tag::subtype_t subtype = 0);
+
+	/**
+	 * Copy constructor.
+	 */
+	Component(const Component& other);
+
+	/**
+	 * Destructor.
+	 */
+	~Component();
+
+	/**
+	 * Returns the name of the analyzer. This name is unique across all
+	 * analyzers and used to identify it. The returned name is derived
+	 * from what's passed to the constructor but upper-cased and
+	 * canonified to allow being part of a script-level ID.
+	 */
+	const char* Name() const	{ return name; }
+
+	/**
+	 * Returns a canonocalized version of the analyzer's name.  The
+	 * returned name is derived from what's passed to the constructor but
+	 * upper-cased and transformed to allow being part of a script-level
+	 * ID.
+	 */
+	const char* CanonicalName() const	{ return canon_name; }
+
+	/**
+	 * Returns the analyzer's factory function.
+	 */
+	factory_callback Factory() const	{ return factory; }
+
+	/**
+	 * Returns the analyzer's tag. Note that this is automatically
+	 * generated for each new Components, and hence unique across all of
+	 * them.
+	 */
+	analyzer::Tag Tag() const;
+
+	/**
+	 * Generates a human-readable description of the component's main
+	 * parameters. This goes into the output of \c "bro -NN".
+	 */
+	virtual void Describe(ODesc* d);
+
+	Component& operator=(const Component& other);
+
+private:
+	const char* name;	// The analyzer's name.
+	const char* canon_name;	// The analyzer's canonical name.
+	factory_callback factory;	// The analyzer's factory callback.
+	analyzer::Tag tag;	// The automatically assigned analyzer tag.
+
+	// Global counter used to generate unique tags.
+	static analyzer::Tag::type_t type_counter;
+};
+
+}
+
+#endif

src/file_analysis/File.cc

@@ -9,9 +9,11 @@
 #include "Reporter.h"
 #include "Val.h"
 #include "Type.h"
-#include "../Analyzer.h"
 #include "Event.h"
 
+#include "analyzer/Analyzer.h"
+#include "analyzer/Manager.h"
+
 using namespace file_analysis;
 
 static Val* empty_connection_table()
@@ -70,7 +72,7 @@ void File::StaticInit()
 	mime_type_idx = Idx("mime_type");
 	}
 
-File::File(const string& file_id, Connection* conn, AnalyzerTag::Tag tag,
+File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
            bool is_orig)
 	: id(file_id), val(0), postpone_timeout(false), first_chunk(true),
 	  missed_bof(false), need_reassembly(false), done(false), analyzers(this)
@@ -85,7 +87,7 @@ File::File(const string& file_id, Connection* conn, AnalyzerTag::Tag tag,
 	if ( conn )
 		{
 		// add source, connection, is_orig fields
-		SetSource(::Analyzer::GetTagName(tag));
+		SetSource(analyzer_mgr->GetAnalyzerName(tag));
 		val->Assign(is_orig_idx, new Val(is_orig, TYPE_BOOL));
 		UpdateConnectionFields(conn);
 		}

src/file_analysis/File.h

@@ -6,7 +6,6 @@
 #include <string>
 #include <vector>
 
-#include "AnalyzerTags.h"
 #include "Conn.h"
 #include "Val.h"
 #include "AnalyzerSet.h"
@@ -166,7 +165,7 @@ protected:
 	 *        direction.
 	 */
 	File(const string& file_id, Connection* conn = 0,
-	     AnalyzerTag::Tag tag = AnalyzerTag::Error, bool is_orig = false);
+	     analyzer::Tag tag = analyzer::Tag::Error, bool is_orig = false);
 
 	/**
 	 * Updates the "conn_ids" and "conn_uids" fields in #val record with the

src/file_analysis/Manager.cc

@@ -10,12 +10,18 @@
 #include "Var.h"
 #include "Event.h"
 
+#include "plugin/Manager.h"
+
 using namespace file_analysis;
 
 TableVal* Manager::disabled = 0;
+string Manager::salt;
 
 Manager::Manager()
 	{
+	tag_enum_type = new EnumType("FileAnalysis::Tag");
+	::ID* id = install_ID("Tag", "FileAnalysis", true, true);
+	add_type(id, tag_enum_type, 0, 0);
 	}
 
 Manager::~Manager()
@@ -23,9 +29,44 @@ Manager::~Manager()
 	Terminate();
 	}
 
+void Manager::InitPreScript()
+	{
+	std::list<Component*> analyzers = plugin_mgr->Components<Component>();
+
+	for ( std::list<Component*>::const_iterator i = analyzers.begin();
+	      i != analyzers.end(); ++i )
+	      RegisterAnalyzerComponent(*i);
+	}
+
+void Manager::RegisterAnalyzerComponent(Component* component)
+	{
+	const char* cname = component->CanonicalName();
+
+	if ( tag_enum_type->Lookup("FileAnalysis", cname) != -1 )
+		reporter->FatalError("File Analyzer %s defined more than once", cname);
+
+	DBG_LOG(DBG_FILE_ANALYSIS, "Registering analyzer %s (tag %s)",
+			component->Name(), component->Tag().AsString().c_str());
+
+	analyzers_by_name.insert(std::make_pair(cname, component));
+	analyzers_by_tag.insert(std::make_pair(component->Tag(), component));
+	analyzers_by_val.insert(std::make_pair(
+	        component->Tag().AsEnumVal()->InternalInt(), component));
+
+	string id = fmt("ANALYZER_%s", cname);
+	tag_enum_type->AddName("FileAnalysis", id.c_str(),
+						   component->Tag().AsEnumVal()->InternalInt(), true);
+	}
+
+void Manager::InitPostScript()
+	{
+	#include "file_analysis.bif.init.cc"
+	}
+
 void Manager::Terminate()
 	{
 	vector<string> keys;
+
 	for ( IDMap::iterator it = id_map.begin(); it != id_map.end(); ++it )
 		keys.push_back(it->first);
 
@@ -35,8 +76,6 @@ void Manager::Terminate()
 
 string Manager::HashHandle(const string& handle) const
 	{
-	static string salt;
-
 	if ( salt.empty() )
 		salt = BifConst::Files::salt->CheckString();
 
@@ -60,7 +99,7 @@ void Manager::SetHandle(const string& handle)
 	}
 
 void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
-                     AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
+                     analyzer::Tag tag, Connection* conn, bool is_orig)
 	{
 	GetFileHandle(tag, conn, is_orig);
 	File* file = GetFile(current_file_id, conn, tag, is_orig);
@@ -74,7 +113,7 @@ void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
 		RemoveFile(file->GetID());
 	}
 
-void Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
+void Manager::DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
                      Connection* conn, bool is_orig)
 	{
 	GetFileHandle(tag, conn, is_orig);
@@ -108,13 +147,13 @@ void Manager::DataIn(const u_char* data, uint64 len, const string& file_id,
 		RemoveFile(file->GetID());
 	}
 
-void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn)
+void Manager::EndOfFile(analyzer::Tag tag, Connection* conn)
 	{
 	EndOfFile(tag, conn, true);
 	EndOfFile(tag, conn, false);
 	}
 
-void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
+void Manager::EndOfFile(analyzer::Tag tag, Connection* conn, bool is_orig)
 	{
 	// Don't need to create a file if we're just going to remove it right away.
 	GetFileHandle(tag, conn, is_orig);
@@ -126,7 +165,7 @@ void Manager::EndOfFile(const string& file_id)
 	RemoveFile(file_id);
 	}
 
-void Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag,
+void Manager::Gap(uint64 offset, uint64 len, analyzer::Tag tag,
                   Connection* conn, bool is_orig)
 	{
 	GetFileHandle(tag, conn, is_orig);
@@ -138,7 +177,7 @@ void Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag,
 	file->Gap(offset, len);
 	}
 
-void Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
+void Manager::SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
                       bool is_orig)
 	{
 	GetFileHandle(tag, conn, is_orig);
@@ -188,7 +227,7 @@ bool Manager::RemoveAnalyzer(const string& file_id, const RecordVal* args) const
 	}
 
 File* Manager::GetFile(const string& file_id, Connection* conn,
-                       AnalyzerTag::Tag tag, bool is_orig, bool update_conn)
+                       analyzer::Tag tag, bool is_orig, bool update_conn)
 	{
 	if ( file_id.empty() )
 		return 0;
@@ -288,7 +327,7 @@ bool Manager::IsIgnored(const string& file_id)
 	return ignored.find(file_id) != ignored.end();
 	}
 
-void Manager::GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig)
+void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig)
 	{
 	current_file_id.clear();
 
@@ -298,8 +337,11 @@ void Manager::GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig)
 	if ( ! get_file_handle )
 		return;
 
+	EnumVal* tagval = tag.AsEnumVal();
+	Ref(tagval);
+
 	val_list* vl = new val_list();
-	vl->append(new Val(tag, TYPE_COUNT));
+	vl->append(tagval);
 	vl->append(c->BuildConnVal());
 	vl->append(new Val(is_orig, TYPE_BOOL));
 
@@ -307,7 +349,7 @@ void Manager::GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig)
 	mgr.Drain(); // need file handle immediately so we don't have to buffer data
 	}
 
-bool Manager::IsDisabled(AnalyzerTag::Tag tag)
+bool Manager::IsDisabled(analyzer::Tag tag)
 	{
 	if ( ! disabled )
 		disabled = internal_const_val("Files::disable")->AsTableVal();
@@ -324,3 +366,31 @@ bool Manager::IsDisabled(AnalyzerTag::Tag tag)
 
 	return rval;
 	}
+
+Analyzer* Manager::InstantiateAnalyzer(int tag, RecordVal* args, File* f) const
+	{
+	analyzer_map_by_val::const_iterator it = analyzers_by_val.find(tag);
+
+	if ( it == analyzers_by_val.end() )
+		reporter->InternalError("cannot instantiate unknown file analyzer: %d",
+		                        tag);
+
+	Component* c = it->second;
+
+	if ( ! c->Factory() )
+		reporter->InternalError("file analyzer %s cannot be instantiated "
+								"dynamically", c->CanonicalName());
+
+	return c->Factory()(args, f);
+	}
+
+const char* Manager::GetAnalyzerName(int tag) const
+	{
+	analyzer_map_by_val::const_iterator it = analyzers_by_val.find(tag);
+
+	if ( it == analyzers_by_val.end() )
+		reporter->InternalError("cannot get name of unknown file analyzer: %d",
+		                        tag);
+
+	return it->second->CanonicalName();
+	}

src/file_analysis/Manager.h

@@ -9,7 +9,6 @@
 #include <queue>
 
 #include "Net.h"
-#include "AnalyzerTags.h"
 #include "Conn.h"
 #include "Val.h"
 #include "Analyzer.h"
@@ -18,6 +17,11 @@
 
 #include "File.h"
 #include "FileTimer.h"
+#include "Component.h"
+
+#include "analyzer/Tag.h"
+
+#include "file_analysis/file_analysis.bif.h"
 
 namespace file_analysis {
 
@@ -37,6 +41,18 @@ public:
 	 */
 	~Manager();
 
+	/**
+	 * First-stage initializion of the manager. This is called early on
+	 * during Bro's initialization, before any scripts are processed.
+	 */
+	void InitPreScript();
+
+	/**
+	 * Second-stage initialization of the manager. This is called late
+	 * during Bro's initialization after any scripts are processed.
+	 */
+	void InitPostScript();
+
 	/**
 	 * Times out any active file analysis to prepare for shutdown.
 	 */
@@ -67,7 +83,7 @@ public:
 	 *        or false if is being sent in the opposite direction.
 	 */
 	void DataIn(const u_char* data, uint64 len, uint64 offset,
-		    AnalyzerTag::Tag tag, Connection* conn, bool is_orig);
+		    analyzer::Tag tag, Connection* conn, bool is_orig);
 
 	/**
 	 * Pass in sequential file data.
@@ -78,7 +94,7 @@ public:
 	 * @param is_orig true if the file is being sent from connection originator
 	 *        or false if is being sent in the opposite direction.
 	 */
-	void DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
+	void DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
 	            Connection* conn, bool is_orig);
 
 	/**
@@ -99,7 +115,7 @@ public:
 	 * @param tag network protocol over which the file data is transferred.
 	 * @param conn network connection over which the file data is transferred.
 	 */
-	void EndOfFile(AnalyzerTag::Tag tag, Connection* conn);
+	void EndOfFile(analyzer::Tag tag, Connection* conn);
 
 	/**
 	 * Signal the end of file data being transferred over a connection in
@@ -107,7 +123,7 @@ public:
 	 * @param tag network protocol over which the file data is transferred.
 	 * @param conn network connection over which the file data is transferred.
 	 */
-	void EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig);
+	void EndOfFile(analyzer::Tag tag, Connection* conn, bool is_orig);
 
 	/**
 	 * Signal the end of file data being transferred using the file identifier.
@@ -124,7 +140,7 @@ public:
 	 * @param is_orig true if the file is being sent from connection originator
 	 *        or false if is being sent in the opposite direction.
 	 */
-	void Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, Connection* conn,
+	void Gap(uint64 offset, uint64 len, analyzer::Tag tag, Connection* conn,
 	         bool is_orig);
 
 	/**
@@ -135,7 +151,7 @@ public:
 	 * @param is_orig true if the file is being sent from connection originator
 	 *        or false if is being sent in the opposite direction.
 	 */
-	void SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
+	void SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
 	             bool is_orig);
 
 	/**
@@ -181,6 +197,23 @@ public:
 	 */
 	bool IsIgnored(const string& file_id);
 
+	/**
+	 * Instantiates a new file analyzer instance for the file.
+	 * @param tag The file analyzer's tag.
+	 * @param args The file analzer argument/option values.
+	 * @param f The file analzer is to be associated with.
+	 * @return The new analyzer instance or null if tag is invalid.
+	 */
+	Analyzer* InstantiateAnalyzer(int tag, RecordVal* args, File* f) const;
+
+	/**
+	 * Translates a script-level file analyzer tag in to corresponding file
+	 * analyzer name.
+	 * @param tag The enum val of a file analyzer.
+	 * @return The human-readable name of the file analyzer.
+	 */
+	const char* GetAnalyzerName(int tag) const;
+
 protected:
 	friend class FileTimer;
 
@@ -205,7 +238,7 @@ protected:
 	 *         connection-related fields.
 	 */
 	File* GetFile(const string& file_id, Connection* conn = 0,
-	              AnalyzerTag::Tag tag = AnalyzerTag::Error,
+	              analyzer::Tag tag = analyzer::Tag::Error,
 	              bool is_orig = false, bool update_conn = true);
 
 	/**
@@ -241,7 +274,7 @@ protected:
 	 * @param is_orig true if the file is being sent from connection originator
 	 *        or false if is being sent in the opposite direction.
 	 */
-	void GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig);
+	void GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig);
 
 	/**
 	 * Check if analysis is available for files transferred over a given
@@ -251,14 +284,26 @@ protected:
 	 * @return whether file analysis is disabled for the analyzer given by
 	 *         \a tag.
 	 */
-	static bool IsDisabled(AnalyzerTag::Tag tag);
+	static bool IsDisabled(analyzer::Tag tag);
 
 private:
+	typedef map<string, Component*> analyzer_map_by_name;
+	typedef map<analyzer::Tag, Component*> analyzer_map_by_tag;
+	typedef map<int, Component*> analyzer_map_by_val;
+
+	void RegisterAnalyzerComponent(Component* component);
+
 	IDMap id_map;	/**< Map file ID to file_analysis::File records. */
 	IDSet ignored;	/**< Ignored files.  Will be finally removed on EOF. */
-	string current_file_id;	/**< Hash of what get_file_handle event sets.*/
+	string current_file_id;	/**< Hash of what get_file_handle event sets. */
+	EnumType* tag_enum_type;	/**< File analyzer tag type. */
+
+	analyzer_map_by_name analyzers_by_name;
+	analyzer_map_by_tag analyzers_by_tag;
+	analyzer_map_by_val analyzers_by_val;
 
 	static TableVal* disabled;	/**< Table of disabled analyzers. */
+	static string salt; /**< A salt added to file handles before hashing. */
 };
 
 } // namespace file_analysis

src/file_analysis/analyzer/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_subdirectory(data_event)
+add_subdirectory(extract)
+add_subdirectory(hash)

src/file_analysis/analyzer/data_event/CMakeLists.txt

@@ -0,0 +1,8 @@
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro FileDataEvent)
+bro_plugin_cc(DataEvent.cc Plugin.cc)
+bro_plugin_end()

src/file_analysis/analyzer/data_event/Plugin.cc

@@ -0,0 +1,26 @@
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
+
+#include "DataEvent.h"
+
+namespace plugin { namespace Bro_FileDataEvent {
+
+class Plugin : public plugin::Plugin {
+protected:
+	void InitPreScript()
+		{
+		SetName("Bro::FileDataEvent");
+		SetVersion(-1);
+		SetAPIVersion(BRO_PLUGIN_API_VERSION);
+		SetDynamicPlugin(false);
+
+		SetDescription("Delivers file content via events");
+
+		AddComponent(new ::file_analysis::Component("DATA_EVENT",
+		        ::file_analysis::DataEvent::Instantiate));
+		}
+};
+
+Plugin __plugin;
+
+} }

src/file_analysis/analyzer/extract/CMakeLists.txt

@@ -0,0 +1,8 @@
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro FileExtract)
+bro_plugin_cc(Extract.cc Plugin.cc)
+bro_plugin_end()

src/file_analysis/analyzer/extract/Plugin.cc

@@ -0,0 +1,26 @@
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
+
+#include "Extract.h"
+
+namespace plugin { namespace Bro_FileExtract {
+
+class Plugin : public plugin::Plugin {
+protected:
+	void InitPreScript()
+		{
+		SetName("Bro::FileExtract");
+		SetVersion(-1);
+		SetAPIVersion(BRO_PLUGIN_API_VERSION);
+		SetDynamicPlugin(false);
+
+		SetDescription("Extract file content to local file system");
+
+		AddComponent(new ::file_analysis::Component("EXTRACT",
+		        ::file_analysis::Extract::Instantiate));
+		}
+};
+
+Plugin __plugin;
+
+} }

src/file_analysis/analyzer/hash/CMakeLists.txt

@@ -0,0 +1,9 @@
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro FileHash)
+bro_plugin_cc(Hash.cc Plugin.cc)
+bro_plugin_bif(events.bif)
+bro_plugin_end()

src/file_analysis/analyzer/hash/Hash.h

@@ -10,6 +10,8 @@
 #include "File.h"
 #include "Analyzer.h"
 
+#include "events.bif.h"
+
 namespace file_analysis {
 
 /**

src/file_analysis/analyzer/hash/Plugin.cc

@@ -0,0 +1,33 @@
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
+
+#include "Hash.h"
+
+namespace plugin { namespace Bro_FileHash {
+
+class Plugin : public plugin::Plugin {
+protected:
+	void InitPreScript()
+		{
+		SetName("Bro::FileHash");
+		SetVersion(-1);
+		SetAPIVersion(BRO_PLUGIN_API_VERSION);
+		SetDynamicPlugin(false);
+
+		SetDescription("Hash file content");
+
+		AddComponent(new ::file_analysis::Component("MD5",
+		        ::file_analysis::MD5::Instantiate));
+		AddComponent(new ::file_analysis::Component("SHA1",
+		        ::file_analysis::SHA1::Instantiate));
+		AddComponent(new ::file_analysis::Component("SHA256",
+		        ::file_analysis::SHA256::Instantiate));
+
+		extern std::list<std::pair<const char*, int> > __bif_events_init();
+		AddBifInitFunction(&__bif_events_init);
+		}
+};
+
+Plugin __plugin;
+
+} }

src/file_analysis/analyzer/hash/events.bif

@@ -0,0 +1,12 @@
+## This event is generated each time file analysis generates a digest of the
+## file contents.
+##
+## f: The file.
+##
+## kind: The type of digest algorithm.
+##
+## hash: The result of the hashing.
+##
+## .. bro:see:: FileAnalysis::add_analyzer FileAnalysis::ANALYZER_MD5
+##    FileAnalysis::ANALYZER_SHA1 FileAnalysis::ANALYZER_SHA256
+event file_hash%(f: fa_file, kind: string, hash: string%);

src/file_analysis/file_analysis.bif

@@ -0,0 +1,61 @@
+##! Internal functions and types used by the file analysis framework.
+
+module Files;
+
+%%{
+#include "file_analysis/Manager.h"
+%%}
+
+type AnalyzerArgs: record;
+
+## :bro:see:`Files::set_timeout_interval`.
+function Files::__set_timeout_interval%(file_id: string, t: interval%): bool
+	%{
+	bool result = file_mgr->SetTimeoutInterval(file_id->CheckString(), t);
+	return new Val(result, TYPE_BOOL);
+	%}
+
+## :bro:see:`Files::add_analyzer`.
+function Files::__add_analyzer%(file_id: string, args: any%): bool
+	%{
+	using BifType::Record::Files::AnalyzerArgs;
+	RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
+	bool result = file_mgr->AddAnalyzer(file_id->CheckString(), rv);
+	Unref(rv);
+	return new Val(result, TYPE_BOOL);
+	%}
+
+## :bro:see:`Files::remove_analyzer`.
+function Files::__remove_analyzer%(file_id: string, args: any%): bool
+	%{
+	using BifType::Record::Files::AnalyzerArgs;
+	RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
+	bool result = file_mgr->RemoveAnalyzer(file_id->CheckString(), rv);
+	Unref(rv);
+	return new Val(result, TYPE_BOOL);
+	%}
+
+## :bro:see:`Files::stop`.
+function Files::__stop%(file_id: string%): bool
+	%{
+	bool result = file_mgr->IgnoreFile(file_id->CheckString());
+	return new Val(result, TYPE_BOOL);
+	%}
+
+module GLOBAL;
+
+## For use within a :bro:see:`get_file_handle` handler to set a unique
+## identifier to associate with the current input to the file analysis
+## framework.  Using an empty string for the handle signifies that the
+## input will be ignored/discarded.
+##
+## handle: A string that uniquely identifies a file.
+##
+## .. bro:see:: get_file_handle
+function set_file_handle%(handle: string%): any
+	%{
+	file_mgr->SetHandle(handle->CheckString());
+	return 0;
+	%}
+
+const Files::salt: string;