Merge remote-tracking branch 'origin/master' into topic/seth/faf-updates

Conflicts:
	scripts/base/frameworks/files/main.bro
	scripts/base/init-bare.bro
	scripts/base/protocols/ftp/file-analysis.bro
	scripts/base/protocols/http/file-analysis.bro
	scripts/base/protocols/irc/file-analysis.bro
	scripts/base/protocols/smtp/file-analysis.bro
	src/const.bif
	src/event.bif
	src/file_analysis/Analyzer.h
	src/file_analysis/file_analysis.bif

src/file_analysis/analyzer/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_subdirectory(data_event)
+add_subdirectory(extract)
+add_subdirectory(hash)

src/file_analysis/analyzer/data_event/CMakeLists.txt

@@ -0,0 +1,8 @@
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro FileDataEvent)
+bro_plugin_cc(DataEvent.cc Plugin.cc)
+bro_plugin_end()

src/file_analysis/analyzer/data_event/DataEvent.cc

@@ -0,0 +1,67 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <string>
+
+#include "DataEvent.h"
+#include "EventRegistry.h"
+#include "Event.h"
+#include "util.h"
+
+using namespace file_analysis;
+
+DataEvent::DataEvent(RecordVal* args, File* file,
+                     EventHandlerPtr ce, EventHandlerPtr se)
+    : file_analysis::Analyzer(args, file), chunk_event(ce), stream_event(se)
+	{
+	}
+
+file_analysis::Analyzer* DataEvent::Instantiate(RecordVal* args, File* file)
+	{
+	using BifType::Record::Files::AnalyzerArgs;
+
+	int chunk_off = AnalyzerArgs->FieldOffset("chunk_event");
+	int stream_off = AnalyzerArgs->FieldOffset("stream_event");
+
+	Val* chunk_val = args->Lookup(chunk_off);
+	Val* stream_val = args->Lookup(stream_off);
+
+	if ( ! chunk_val && ! stream_val ) return 0;
+
+	EventHandlerPtr chunk;
+	EventHandlerPtr stream;
+
+	if ( chunk_val )
+		chunk = event_registry->Lookup(chunk_val->AsFunc()->Name());
+
+	if ( stream_val )
+		stream = event_registry->Lookup(stream_val->AsFunc()->Name());
+
+	return new DataEvent(args, file, chunk, stream);
+	}
+
+bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
+	{
+	if ( ! chunk_event ) return true;
+
+	val_list* args = new val_list;
+	args->append(GetFile()->GetVal()->Ref());
+	args->append(new StringVal(new BroString(data, len, 0)));
+	args->append(new Val(offset, TYPE_COUNT));
+
+	mgr.QueueEvent(chunk_event, args);
+
+	return true;
+	}
+
+bool DataEvent::DeliverStream(const u_char* data, uint64 len)
+	{
+	if ( ! stream_event ) return true;
+
+	val_list* args = new val_list;
+	args->append(GetFile()->GetVal()->Ref());
+	args->append(new StringVal(new BroString(data, len, 0)));
+
+	mgr.QueueEvent(stream_event, args);
+
+	return true;
+	}

src/file_analysis/analyzer/data_event/DataEvent.h

@@ -0,0 +1,69 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef FILE_ANALYSIS_DATAEVENT_H
+#define FILE_ANALYSIS_DATAEVENT_H
+
+#include <string>
+
+#include "Val.h"
+#include "File.h"
+#include "Analyzer.h"
+
+namespace file_analysis {
+
+/**
+ * An analyzer to send file data to script-layer via events.
+ */
+class DataEvent : public file_analysis::Analyzer {
+public:
+
+	/**
+	 * Generates the event, if any, specified by the "chunk_event" field of this
+	 * analyzer's \c AnalyzerArgs.  This is for non-sequential file data input.
+	 * @param data pointer to start of file data chunk.
+	 * @param len number of bytes in the data chunk.
+	 * @param offset number of bytes from start of file at which chunk occurs.
+	 * @return always true
+	 */
+	virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);
+
+	/**
+	 * Generates the event, if any, specified by the "stream_event" field of
+	 * this analyzer's \c AnalyzerArgs.  This is for sequential file data input.
+	 * @param data pointer to start of file data chunk.
+	 * @param len number of bytes in the data chunk.
+	 * @return always true
+	 */
+	virtual bool DeliverStream(const u_char* data, uint64 len);
+
+	/**
+	 * Create a new instance of a DataEvent analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new DataEvent analyzer instance or a null pointer if
+	 *         no "chunk_event" or "stream_event" field was specfied in \a args.
+	 */
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
+
+protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @param ce pointer to event handler which will be called to receive
+	 *        non-sequential file data.
+	 * @param se pointer to event handler which will be called to receive
+	 *        sequential file data.
+	 */
+	DataEvent(RecordVal* args, File* file,
+	          EventHandlerPtr ce, EventHandlerPtr se);
+
+private:
+	EventHandlerPtr chunk_event;
+	EventHandlerPtr stream_event;
+};
+
+} // namespace file_analysis
+
+#endif

src/file_analysis/analyzer/data_event/Plugin.cc

@@ -0,0 +1,26 @@
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
+
+#include "DataEvent.h"
+
+namespace plugin { namespace Bro_FileDataEvent {
+
+class Plugin : public plugin::Plugin {
+protected:
+	void InitPreScript()
+		{
+		SetName("Bro::FileDataEvent");
+		SetVersion(-1);
+		SetAPIVersion(BRO_PLUGIN_API_VERSION);
+		SetDynamicPlugin(false);
+
+		SetDescription("Delivers file content via events");
+
+		AddComponent(new ::file_analysis::Component("DATA_EVENT",
+		        ::file_analysis::DataEvent::Instantiate));
+		}
+};
+
+Plugin __plugin;
+
+} }

src/file_analysis/analyzer/extract/CMakeLists.txt

@@ -0,0 +1,8 @@
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro FileExtract)
+bro_plugin_cc(Extract.cc Plugin.cc)
+bro_plugin_end()

src/file_analysis/analyzer/extract/Extract.cc

@@ -0,0 +1,48 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <string>
+
+#include "Extract.h"
+#include "util.h"
+
+using namespace file_analysis;
+
+Extract::Extract(RecordVal* args, File* file, const string& arg_filename)
+    : file_analysis::Analyzer(args, file), filename(arg_filename)
+	{
+	fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+
+	if ( fd < 0 )
+		{
+		fd = 0;
+		char buf[128];
+		strerror_r(errno, buf, sizeof(buf));
+		reporter->Error("cannot open %s: %s", filename.c_str(), buf);
+		}
+	}
+
+Extract::~Extract()
+	{
+	if ( fd )
+		safe_close(fd);
+	}
+
+file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file)
+	{
+	using BifType::Record::Files::AnalyzerArgs;
+	Val* v = args->Lookup(AnalyzerArgs->FieldOffset("extract_filename"));
+
+	if ( ! v )
+		return 0;
+
+	return new Extract(args, file, v->AsString()->CheckString());
+	}
+
+bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
+	{
+	if ( ! fd )
+		return false;
+
+	safe_pwrite(fd, data, len, offset);
+	return true;
+	}

src/file_analysis/analyzer/extract/Extract.h

@@ -0,0 +1,62 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef FILE_ANALYSIS_EXTRACT_H
+#define FILE_ANALYSIS_EXTRACT_H
+
+#include <string>
+
+#include "Val.h"
+#include "File.h"
+#include "Analyzer.h"
+
+namespace file_analysis {
+
+/**
+ * An analyzer to extract content of files to local disk.
+ */
+class Extract : public file_analysis::Analyzer {
+public:
+
+	/**
+	 * Destructor.  Will close the file that was used for data extraction.
+	 */
+	virtual ~Extract();
+
+	/**
+	 * Write a chunk of file data to the local extraction file.
+	 * @param data pointer to a chunk of file data.
+	 * @param len number of bytes in the data chunk.
+	 * @param offset number of bytes from start of file at which chunk starts.
+	 * @return false if there was no extraction file open and the data couldn't
+	 *         be written, else true.
+	 */
+	virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);
+
+	/**
+	 * Create a new instance of an Extract analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new Extract analyzer instance or a null pointer if the
+	 *         the "extraction_file" field of \a args wasn't set.
+	 */
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
+
+protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @param arg_filename a file system path which specifies the local file
+	 *        to which the contents of the file will be extracted/written.
+	 */
+	Extract(RecordVal* args, File* file, const string& arg_filename);
+
+private:
+	string filename;
+	int fd;
+};
+
+} // namespace file_analysis
+
+#endif

src/file_analysis/analyzer/extract/Plugin.cc

@@ -0,0 +1,26 @@
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
+
+#include "Extract.h"
+
+namespace plugin { namespace Bro_FileExtract {
+
+class Plugin : public plugin::Plugin {
+protected:
+	void InitPreScript()
+		{
+		SetName("Bro::FileExtract");
+		SetVersion(-1);
+		SetAPIVersion(BRO_PLUGIN_API_VERSION);
+		SetDynamicPlugin(false);
+
+		SetDescription("Extract file content to local file system");
+
+		AddComponent(new ::file_analysis::Component("EXTRACT",
+		        ::file_analysis::Extract::Instantiate));
+		}
+};
+
+Plugin __plugin;
+
+} }

src/file_analysis/analyzer/hash/CMakeLists.txt

@@ -0,0 +1,9 @@
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro FileHash)
+bro_plugin_cc(Hash.cc Plugin.cc)
+bro_plugin_bif(events.bif)
+bro_plugin_end()

src/file_analysis/analyzer/hash/Hash.cc

@@ -0,0 +1,56 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <string>
+
+#include "Hash.h"
+#include "util.h"
+#include "Event.h"
+
+using namespace file_analysis;
+
+Hash::Hash(RecordVal* args, File* file, HashVal* hv, const char* arg_kind)
+	: file_analysis::Analyzer(args, file), hash(hv), fed(false), kind(arg_kind)
+	{
+	hash->Init();
+	}
+
+Hash::~Hash()
+	{
+	Unref(hash);
+	}
+
+bool Hash::DeliverStream(const u_char* data, uint64 len)
+	{
+	if ( ! hash->IsValid() )
+		return false;
+
+	if ( ! fed )
+		fed = len > 0;
+
+	hash->Feed(data, len);
+	return true;
+	}
+
+bool Hash::EndOfFile()
+	{
+	Finalize();
+	return false;
+	}
+
+bool Hash::Undelivered(uint64 offset, uint64 len)
+	{
+	return false;
+	}
+
+void Hash::Finalize()
+	{
+	if ( ! hash->IsValid() || ! fed )
+		return;
+
+	val_list* vl = new val_list();
+	vl->append(GetFile()->GetVal()->Ref());
+	vl->append(new StringVal(kind));
+	vl->append(hash->Get());
+
+	mgr.QueueEvent(file_hash, vl);
+	}

src/file_analysis/analyzer/hash/Hash.h

@@ -0,0 +1,160 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef FILE_ANALYSIS_HASH_H
+#define FILE_ANALYSIS_HASH_H
+
+#include <string>
+
+#include "Val.h"
+#include "OpaqueVal.h"
+#include "File.h"
+#include "Analyzer.h"
+
+#include "events.bif.h"
+
+namespace file_analysis {
+
+/**
+ * An analyzer to produce a hash of file contents.
+ */
+class Hash : public file_analysis::Analyzer {
+public:
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~Hash();
+
+	/**
+	 * Incrementally hash next chunk of file contents.
+	 * @param data pointer to start of a chunk of a file data.
+	 * @param len number of bytes in the data chunk.
+	 * @return false if the digest is in an invalid state, else true.
+	 */
+	virtual bool DeliverStream(const u_char* data, uint64 len);
+
+	/**
+	 * Finalizes the hash and raises a "file_hash" event.
+	 * @return always false so analyze will be deteched from file.
+	 */
+	virtual bool EndOfFile();
+
+	/**
+	 * Missing data can't be handled, so just indicate the this analyzer should
+	 * be removed from receiving further data.  The hash will not be finalized.
+	 * @param offset byte offset in file at which missing chunk starts.
+	 * @param len number of missing bytes.
+	 * @return always false so analyzer will detach from file.
+	 */
+	virtual bool Undelivered(uint64 offset, uint64 len);
+
+protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @param hv specific hash calculator object.
+	 * @param kind human readable name of the hash algorithm to use.
+	 */
+	Hash(RecordVal* args, File* file, HashVal* hv, const char* kind);
+
+	/**
+	 * If some file contents have been seen, finalizes the hash of them and
+	 * raises the "file_hash" event with the results.
+	 */
+	void Finalize();
+
+private:
+	HashVal* hash;
+	bool fed;
+	const char* kind;
+};
+
+/**
+ * An analyzer to produce an MD5 hash of file contents.
+ */
+class MD5 : public Hash {
+public:
+
+	/**
+	 * Create a new instance of the MD5 hashing file analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new MD5 analyzer instance or a null pointer if there's no
+	 *         handler for the "file_hash" event.
+	 */
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
+		{ return file_hash ? new MD5(args, file) : 0; }
+
+protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 */
+	MD5(RecordVal* args, File* file)
+		: Hash(args, file, new MD5Val(), "md5")
+		{}
+};
+
+/**
+ * An analyzer to produce a SHA1 hash of file contents.
+ */
+class SHA1 : public Hash {
+public:
+
+	/**
+	 * Create a new instance of the SHA1 hashing file analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new MD5 analyzer instance or a null pointer if there's no
+	 *         handler for the "file_hash" event.
+	 */
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
+		{ return file_hash ? new SHA1(args, file) : 0; }
+
+protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 */
+	SHA1(RecordVal* args, File* file)
+		: Hash(args, file, new SHA1Val(), "sha1")
+		{}
+};
+
+/**
+ * An analyzer to produce a SHA256 hash of file contents.
+ */
+class SHA256 : public Hash {
+public:
+
+	/**
+	 * Create a new instance of the SHA256 hashing file analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new MD5 analyzer instance or a null pointer if there's no
+	 *         handler for the "file_hash" event.
+	 */
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
+		{ return file_hash ? new SHA256(args, file) : 0; }
+
+protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 */
+	SHA256(RecordVal* args, File* file)
+		: Hash(args, file, new SHA256Val(), "sha256")
+		{}
+};
+
+} // namespace file_analysis
+
+#endif

src/file_analysis/analyzer/hash/Plugin.cc

@@ -0,0 +1,33 @@
+#include "plugin/Plugin.h"
+#include "file_analysis/Component.h"
+
+#include "Hash.h"
+
+namespace plugin { namespace Bro_FileHash {
+
+class Plugin : public plugin::Plugin {
+protected:
+	void InitPreScript()
+		{
+		SetName("Bro::FileHash");
+		SetVersion(-1);
+		SetAPIVersion(BRO_PLUGIN_API_VERSION);
+		SetDynamicPlugin(false);
+
+		SetDescription("Hash file content");
+
+		AddComponent(new ::file_analysis::Component("MD5",
+		        ::file_analysis::MD5::Instantiate));
+		AddComponent(new ::file_analysis::Component("SHA1",
+		        ::file_analysis::SHA1::Instantiate));
+		AddComponent(new ::file_analysis::Component("SHA256",
+		        ::file_analysis::SHA256::Instantiate));
+
+		extern std::list<std::pair<const char*, int> > __bif_events_init();
+		AddBifInitFunction(&__bif_events_init);
+		}
+};
+
+Plugin __plugin;
+
+} }

src/file_analysis/analyzer/hash/events.bif

@@ -0,0 +1,12 @@
+## This event is generated each time file analysis generates a digest of the
+## file contents.
+##
+## f: The file.
+##
+## kind: The type of digest algorithm.
+##
+## hash: The result of the hashing.
+##
+## .. bro:see:: FileAnalysis::add_analyzer FileAnalysis::ANALYZER_MD5
+##    FileAnalysis::ANALYZER_SHA1 FileAnalysis::ANALYZER_SHA256
+event file_hash%(f: fa_file, kind: string, hash: string%);