Merge remote-tracking branch 'origin/master' into topic/johanna/ocsp

src/file_analysis/Analyzer.h

@@ -123,6 +123,21 @@ public:
 	void SetGotStreamDelivery()
 		{ got_stream_delivery = true; }
 
+	/**
+	 * Signals that the analyzer is to skip all further input
+	 * processsing. This won't have an immediate effect internally, but
+	 * the flag can be queried through Skipping().
+	 *
+	 * @param do_skip If true, further processing will be skipped.
+	 */
+	void SetSkip(bool do_skip)		{ skip = do_skip; }
+
+	/**
+	 * Returns true if the analyzer has been told to skip processing all
+	 * further input.
+	 */
+	bool Skipping() const			{ return skip; }
+
 protected:
 
 	/**
@@ -136,7 +151,8 @@ protected:
 	    : tag(arg_tag),
 	      args(arg_args->Ref()->AsRecordVal()),
 	      file(arg_file),
-	      got_stream_delivery(false)
+	      got_stream_delivery(false),
+	      skip(false)
 		{
 		id = ++id_counter;
 		}
@@ -154,7 +170,8 @@ protected:
 	    : tag(),
 	      args(arg_args->Ref()->AsRecordVal()),
 	      file(arg_file),
-	      got_stream_delivery(false)
+	      got_stream_delivery(false),
+	      skip(false)
 		{
 		id = ++id_counter;
 		}
@@ -166,6 +183,7 @@ private:
 	RecordVal* args;	/**< \c AnalyzerArgs val gives tunable analyzer params. */
 	File* file;	/**< The file to which the analyzer is attached. */
 	bool got_stream_delivery;
+	bool skip;
 
 	static ID id_counter;
 };

src/file_analysis/AnalyzerSet.cc

@@ -130,7 +130,11 @@ bool AnalyzerSet::Remove(file_analysis::Tag tag, HashKey* key)
 	        file_mgr->GetComponentName(tag).c_str());
 
 	a->Done();
-	delete a;
+
+	// We don't delete the analyzer object right here because the remove
+	// operation may execute at a time when it can still be accessed.
+	// Instead we let the file know to delete the analyzer later.
+	file->DoneWithAnalyzer(a);
 
 	return true;
 	}

src/file_analysis/Component.cc

@@ -13,7 +13,11 @@ Component::Component(const std::string& name, factory_callback arg_factory, Tag:
 	  plugin::TaggedComponent<file_analysis::Tag>(subtype)
 	{
 	factory = arg_factory;
+	}
 
+void Component::Initialize()
+	{
+	InitializeTag();
 	file_mgr->RegisterComponent(this, "ANALYZER_");
 	}
 

src/file_analysis/Component.h

@@ -54,6 +54,13 @@ public:
 	 */
 	~Component();
 
+	/**
+	 * Initialization function. This function has to be called before any
+	 * plugin component functionality is used; it is used to add the
+	 * plugin component to the list of components and to initialize tags
+	 */
+	void Initialize() override;
+
 	/**
 	 * Returns the analyzer's factory function.
 	 */
@@ -63,7 +70,7 @@ protected:
 	/**
 	  * Overriden from plugin::Component.
 	  */
-	virtual void DoDescribe(ODesc* d) const;
+	void DoDescribe(ODesc* d) const override;
 
 private:
 	factory_callback factory;	// The analyzer's factory callback.

src/file_analysis/File.cc

@@ -107,6 +107,9 @@ File::~File()
 	DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Destroying File object", id.c_str());
 	Unref(val);
 	delete file_reassembler;
+
+	for ( auto a : done_analyzers )
+		delete a;
 	}
 
 void File::UpdateLastActivityTime()
@@ -375,8 +378,10 @@ void File::DeliverStream(const u_char* data, uint64 len)
 
 	while ( (a = analyzers.NextEntry(c)) )
 		{
+		DBG_LOG(DBG_FILE_ANALYSIS, "stream delivery to analyzer %s", file_mgr->GetComponentName(a->Tag()).c_str());
 		if ( ! a->GotStreamDelivery() )
 			{
+			DBG_LOG(DBG_FILE_ANALYSIS, "skipping stream delivery to analyzer %s", file_mgr->GetComponentName(a->Tag()).c_str());
 			int num_bof_chunks_behind = bof_buffer.chunks.size();
 
 			if ( ! bof_was_full )
@@ -389,9 +394,15 @@ void File::DeliverStream(const u_char* data, uint64 len)
 			// Catch this analyzer up with the BOF buffer.
 			for ( int i = 0; i < num_bof_chunks_behind; ++i )
 				{
-				if ( ! a->DeliverStream(bof_buffer.chunks[i]->Bytes(),
-				                        bof_buffer.chunks[i]->Len()) )
-					analyzers.QueueRemove(a->Tag(), a->Args());
+				if ( ! a->Skipping() )
+					{
+					if ( ! a->DeliverStream(bof_buffer.chunks[i]->Bytes(),
+								bof_buffer.chunks[i]->Len()) )
+						{
+						a->SetSkip(true);
+						analyzers.QueueRemove(a->Tag(), a->Args());
+						}
+					}
 
 				bytes_delivered += bof_buffer.chunks[i]->Len();
 				}
@@ -401,8 +412,14 @@ void File::DeliverStream(const u_char* data, uint64 len)
 			// Analyzer should be fully caught up to stream_offset now.
 			}
 
-		if ( ! a->DeliverStream(data, len) )
-			analyzers.QueueRemove(a->Tag(), a->Args());
+		if ( ! a->Skipping() )
+			{
+			if ( ! a->DeliverStream(data, len) )
+				{
+				a->SetSkip(true);
+				analyzers.QueueRemove(a->Tag(), a->Args());
+				}
+			}
 		}
 
 	stream_offset += len;
@@ -465,9 +482,14 @@ void File::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
 
 	while ( (a = analyzers.NextEntry(c)) )
 		{
-		if ( ! a->DeliverChunk(data, len, offset) )
+		DBG_LOG(DBG_FILE_ANALYSIS, "chunk delivery to analyzer %s", file_mgr->GetComponentName(a->Tag()).c_str());
+		if ( ! a->Skipping() )
 			{
-			analyzers.QueueRemove(a->Tag(), a->Args());
+			if ( ! a->DeliverChunk(data, len, offset) )
+				{
+				a->SetSkip(true);
+				analyzers.QueueRemove(a->Tag(), a->Args());
+				}
 			}
 		}
 
@@ -475,6 +497,11 @@ void File::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
 		EndOfFile();
 	}
 
+void File::DoneWithAnalyzer(Analyzer* analyzer)
+	{
+	done_analyzers.push_back(analyzer);
+	}
+
 void File::DataIn(const u_char* data, uint64 len, uint64 offset)
 	{
 	analyzers.DrainModifications();
@@ -530,7 +557,7 @@ void File::EndOfFile()
 
 void File::Gap(uint64 offset, uint64 len)
 	{
-	DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Gap of size %" PRIu64 " at offset %," PRIu64,
+	DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Gap of size %" PRIu64 " at offset %" PRIu64,
 		id.c_str(), len, offset);
 
 	if ( file_reassembler && ! file_reassembler->IsCurrentlyFlushing() )

src/file_analysis/File.h

@@ -119,6 +119,11 @@ public:
 	 */
 	bool RemoveAnalyzer(file_analysis::Tag tag, RecordVal* args);
 
+	/**
+	 * Signal that this analyzer can be deleted once it's safe to do so.
+	 */
+	void DoneWithAnalyzer(Analyzer* analyzer);
+
 	/**
 	 * Pass in non-sequential data and deliver to attached analyzers.
 	 * @param data pointer to start of a chunk of file data.
@@ -287,6 +292,7 @@ protected:
 	bool postpone_timeout;     /**< Whether postponing timeout is requested. */
 	bool done;                 /**< If this object is about to be deleted. */
 	AnalyzerSet analyzers;     /**< A set of attached file analyzers. */
+	std::list<Analyzer *> done_analyzers; /**< Analyzers we're done with, remembered here until they can be safely deleted. */
 
 	struct BOF_Buffer {
 		BOF_Buffer() : full(false), size(0) {}

src/file_analysis/FileReassembler.cc

@@ -8,7 +8,7 @@ namespace file_analysis {
 class File;
 
 FileReassembler::FileReassembler(File *f, uint64 starting_offset)
-	: Reassembler(starting_offset), the_file(f), flushing(false)
+	: Reassembler(starting_offset, REASSEM_FILE), the_file(f), flushing(false)
 	{
 	}
 

src/file_analysis/FileReassembler.h

@@ -52,9 +52,9 @@ protected:
 
 	DECLARE_SERIAL(FileReassembler);
 
-	void Undelivered(uint64 up_to_seq);
-	void BlockInserted(DataBlock* b);
-	void Overlap(const u_char* b1, const u_char* b2, uint64 n);
+	void Undelivered(uint64 up_to_seq) override;
+	void BlockInserted(DataBlock* b) override;
+	void Overlap(const u_char* b1, const u_char* b2, uint64 n) override;
 
 	File* the_file;
 	bool flushing;

src/file_analysis/FileTimer.cc

@@ -9,7 +9,7 @@ FileTimer::FileTimer(double t, const string& id, double interval)
     : Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id)
 	{
 	DBG_LOG(DBG_FILE_ANALYSIS, "New %f second timeout timer for %s",
-	        file_id.c_str(), interval);
+	        interval, file_id.c_str());
 	}
 
 void FileTimer::Dispatch(double t, int is_expire)

src/file_analysis/Manager.h

@@ -302,6 +302,15 @@ public:
 	 */
 	std::string DetectMIME(const u_char* data, uint64 len) const;
 
+	uint64 CurrentFiles()
+		{ return id_map.Length(); }
+
+	uint64 MaxFiles()
+		{ return id_map.MaxLength(); }
+
+	uint64 CumulativeFiles()
+		{ return id_map.NumCumulativeInserts(); }
+
 protected:
 	friend class FileTimer;
 

src/file_analysis/analyzer/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_subdirectory(data_event)
+add_subdirectory(entropy)
 add_subdirectory(extract)
 add_subdirectory(hash)
 add_subdirectory(pe)

src/file_analysis/analyzer/entropy/CMakeLists.txt

@@ -0,0 +1,9 @@
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro FileEntropy)
+bro_plugin_cc(Entropy.cc Plugin.cc ../../Analyzer.cc)
+bro_plugin_bif(events.bif)
+bro_plugin_end()

src/file_analysis/analyzer/entropy/Entropy.cc

@@ -0,0 +1,72 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <string>
+
+#include "Entropy.h"
+#include "util.h"
+#include "Event.h"
+#include "file_analysis/Manager.h"
+
+using namespace file_analysis;
+
+Entropy::Entropy(RecordVal* args, File* file)
+    : file_analysis::Analyzer(file_mgr->GetComponentTag("ENTROPY"), args, file)
+	{
+	//entropy->Init();
+	entropy = new EntropyVal;
+	fed = false;
+	}
+
+Entropy::~Entropy()
+	{
+	Unref(entropy);
+	}
+
+file_analysis::Analyzer* Entropy::Instantiate(RecordVal* args, File* file)
+	{
+	return new Entropy(args, file);
+	}
+
+bool Entropy::DeliverStream(const u_char* data, uint64 len)
+	{
+	if ( ! fed )
+		fed = len > 0;
+
+	entropy->Feed(data, len);
+	return true;
+	}
+
+bool Entropy::EndOfFile()
+	{
+	Finalize();
+	return false;
+	}
+
+bool Entropy::Undelivered(uint64 offset, uint64 len)
+	{
+	return false;
+	}
+
+void Entropy::Finalize()
+	{
+	//if ( ! entropy->IsValid() || ! fed )
+	if ( ! fed )
+		return;
+
+	val_list* vl = new val_list();
+	vl->append(GetFile()->GetVal()->Ref());
+
+	double montepi, scc, ent, mean, chisq;
+	montepi = scc = ent = mean = chisq = 0.0;
+	entropy->Get(&ent, &chisq, &mean, &montepi, &scc);
+
+	RecordVal* ent_result = new RecordVal(entropy_test_result);
+	ent_result->Assign(0, new Val(ent,     TYPE_DOUBLE));
+	ent_result->Assign(1, new Val(chisq,   TYPE_DOUBLE));
+	ent_result->Assign(2, new Val(mean,    TYPE_DOUBLE));
+	ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE));
+	ent_result->Assign(4, new Val(scc,     TYPE_DOUBLE));
+
+	vl->append(ent_result);
+	mgr.QueueEvent(file_entropy, vl);
+	}

src/file_analysis/analyzer/entropy/Entropy.h

@@ -0,0 +1,84 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef FILE_ANALYSIS_ENTROPY_H
+#define FILE_ANALYSIS_ENTROPY_H
+
+#include <string>
+
+#include "Val.h"
+#include "OpaqueVal.h"
+#include "File.h"
+#include "Analyzer.h"
+
+#include "events.bif.h"
+
+namespace file_analysis {
+
+/**
+ * An analyzer to produce a hash of file contents.
+ */
+class Entropy : public file_analysis::Analyzer {
+public:
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~Entropy();
+
+	/**
+	 * Create a new instance of an Extract analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new Extract analyzer instance or a null pointer if the
+	 *         the "extraction_file" field of \a args wasn't set.
+	 */
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
+
+	/**
+	 * Incrementally hash next chunk of file contents.
+	 * @param data pointer to start of a chunk of a file data.
+	 * @param len number of bytes in the data chunk.
+	 * @return false if the digest is in an invalid state, else true.
+	 */
+	virtual bool DeliverStream(const u_char* data, uint64 len);
+
+	/**
+	 * Finalizes the hash and raises a "file_entropy_test" event.
+	 * @return always false so analyze will be deteched from file.
+	 */
+	virtual bool EndOfFile();
+
+	/**
+	 * Missing data can't be handled, so just indicate the this analyzer should
+	 * be removed from receiving further data.  The hash will not be finalized.
+	 * @param offset byte offset in file at which missing chunk starts.
+	 * @param len number of missing bytes.
+	 * @return always false so analyzer will detach from file.
+	 */
+	virtual bool Undelivered(uint64 offset, uint64 len);
+
+protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @param hv specific hash calculator object.
+	 * @param kind human readable name of the hash algorithm to use.
+	 */
+	Entropy(RecordVal* args, File* file);
+
+	/**
+	 * If some file contents have been seen, finalizes the hash of them and
+	 * raises the "file_hash" event with the results.
+	 */
+	void Finalize();
+
+private:
+	EntropyVal* entropy;
+	bool fed;
+};
+
+} // namespace file_analysis
+
+#endif

src/file_analysis/analyzer/entropy/Plugin.cc

@@ -0,0 +1,24 @@
+// See the file  in the main distribution directory for copyright.
+
+#include "plugin/Plugin.h"
+
+#include "Entropy.h"
+
+namespace plugin {
+namespace Bro_FileEntropy {
+
+class Plugin : public plugin::Plugin {
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::file_analysis::Component("ENTROPY", ::file_analysis::Entropy::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::FileEntropy";
+		config.description = "Entropy test file content";
+		return config;
+		}
+} plugin;
+
+}
+}

src/file_analysis/analyzer/entropy/events.bif

@@ -0,0 +1,8 @@
+## This event is generated each time file analysis performs
+## entropy testing on a file.
+##
+## f: The file.
+##
+## ent: The results of the entropy testing.
+##
+event file_entropy%(f: fa_file, ent: entropy_test_result%);

src/file_analysis/analyzer/extract/events.bif

@@ -14,4 +14,4 @@
 ## len: The length of the file chunk about to be written.
 ##
 ## .. bro:see:: Files::add_analyzer Files::ANALYZER_EXTRACT
-event file_extraction_limit%(f: fa_file, args: any, limit: count, len: count%);
+event file_extraction_limit%(f: fa_file, args: Files::AnalyzerArgs, limit: count, len: count%);

src/file_analysis/analyzer/x509/X509.cc

@@ -14,6 +14,7 @@
 #include <openssl/x509v3.h>
 #include <openssl/asn1.h>
 #include <openssl/opensslconf.h>
+#include <openssl/err.h>
 
 using namespace file_analysis;
 
@@ -543,7 +544,7 @@ double file_analysis::X509::GetTimeFromAsn1(const ASN1_TIME* atime, const char*
 			}
 
 		// year is first two digits in YY format. Buffer expects YYYY format.
-		if ( pString[0] - '0' < 50 ) // RFC 2459 4.1.2.5.1
+		if ( pString[0] < '5' ) // RFC 2459 4.1.2.5.1
 			{
 			*(pBuffer++) = '2';
 			*(pBuffer++) = '0';

src/file_analysis/analyzer/x509/functions.bif

@@ -6,6 +6,8 @@
 #include <openssl/asn1.h>
 #include <openssl/x509_vfy.h>
 #include <openssl/ocsp.h>
+#include <openssl/pem.h>
+#include <openssl/err.h>
 
 // This is the indexed map of X509 certificate stores.
 static map<Val*, X509_STORE*> x509_stores;
@@ -336,8 +338,11 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
 		goto x509_ocsp_cleanup;
 		}
 
-	out = OCSP_basic_verify(basic, NULL, ctx, 0);
-	if ( result < 1 )
+	// We pass OCSP_NOVERIFY to let OCSP_basic_verify skip the chain verification.
+	// With that, it only verifies the signature of the basic response and we are responsible
+	// for the chain ourselves. We have to do that since we cannot get OCSP_basic_verify to use our timestamp.
+	out = OCSP_basic_verify(basic, NULL, ctx, OCSP_NOVERIFY);
+	if ( out < 1 )
 		{
 		rval = x509_result_record(out, ERR_error_string(ERR_get_error(),NULL));
 		goto x509_ocsp_cleanup;