Merge branch 'topic/seth/file-entropy'

* topic/seth/file-entropy:
  Add a file entropy test.
  Fixing a test.
  Updated tests for file entropy analyzer.
  Update and clean up to file entropy measurement.
  First commit of file entropy analyzer.

CHANGES

@@ -1,4 +1,21 @@
 
+2.4-452 | 2016-04-13 01:15:20 -0400
+
+  * Add a simple file entropy analyzer. (Seth Hall)
+
+  * Analyzer and bro script for RFB/VNC protocol (Martin van Hensbergen)
+
+    This analyzer parses the Remote Frame Buffer
+    protocol, usually referred to as the 'VNC protocol'.
+
+    It supports several dialects (3.3, 3.7, 3.8) and
+    also handles the Apple Remote Desktop variant.
+
+    It will log such facts as client/server versions,
+    authentication method used, authentication result,
+    height, width and name of the shared screen.
+
+
 2.4-430 | 2016-04-07 13:36:36 -0700
 
   * Fix regex literal in scripting documentation. (William Tom)

NEWS

@@ -26,6 +26,11 @@ New Functionality
 - Bro now includes the NetControl framework. The framework allows for easy
   interaction of Bro with hard- and software switches, firewalls, etc.
 
+- There is a new file entropy analyzer for files.
+
+- Bro now supports the remote framebuffer protocol (RFB) that is used by
+  VNC servers for remote graphical displays.
+
 - Bro now supports the Radiotap header for 802.11 frames.
 
 - Bro now tracks VLAN IDs. To record them inside the connection log,

VERSION

@@ -1 +1 @@
-2.4-430
+2.4-452

scripts/policy/frameworks/files/entropy-test-all-files.bro

@@ -0,0 +1,20 @@
+
+module Files;
+
+export {
+	redef record Files::Info += {
+		## The information density of the contents of the file, 
+		## expressed as a number of bits per character. 
+		entropy: double &log &optional;
+	};
+}
+
+event file_new(f: fa_file)
+	{
+	Files::add_analyzer(f, Files::ANALYZER_ENTROPY);
+	}
+
+event file_entropy(f: fa_file, ent: entropy_test_result)
+	{
+	f$info$entropy = ent$entropy;
+	}

scripts/test-all-policy.bro

@@ -29,6 +29,7 @@
 @load frameworks/intel/seen/where-locations.bro
 @load frameworks/intel/seen/x509.bro
 @load frameworks/files/detect-MHR.bro
+@load frameworks/files/entropy-test-all-files.bro
 #@load frameworks/files/extract-all-files.bro
 @load frameworks/files/hash-all-files.bro
 @load frameworks/packet-filter/shunt.bro

src/file_analysis/analyzer/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_subdirectory(data_event)
+add_subdirectory(entropy)
 add_subdirectory(extract)
 add_subdirectory(hash)
 add_subdirectory(pe)

src/file_analysis/analyzer/entropy/CMakeLists.txt

@@ -0,0 +1,9 @@
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro FileEntropy)
+bro_plugin_cc(Entropy.cc Plugin.cc ../../Analyzer.cc)
+bro_plugin_bif(events.bif)
+bro_plugin_end()

src/file_analysis/analyzer/entropy/Entropy.cc

@@ -0,0 +1,71 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <string>
+
+#include "Entropy.h"
+#include "util.h"
+#include "Event.h"
+#include "file_analysis/Manager.h"
+
+using namespace file_analysis;
+
+Entropy::Entropy(RecordVal* args, File* file)
+    : file_analysis::Analyzer(file_mgr->GetComponentTag("ENTROPY"), args, file)
+	{
+	//entropy->Init();
+	entropy = new EntropyVal;
+	}
+
+Entropy::~Entropy()
+	{
+	Unref(entropy);
+	}
+
+file_analysis::Analyzer* Entropy::Instantiate(RecordVal* args, File* file)
+	{
+	return new Entropy(args, file);
+	}
+
+bool Entropy::DeliverStream(const u_char* data, uint64 len)
+	{
+	if ( ! fed )
+		fed = len > 0;
+
+	entropy->Feed(data, len);
+	return true;
+	}
+
+bool Entropy::EndOfFile()
+	{
+	Finalize();
+	return false;
+	}
+
+bool Entropy::Undelivered(uint64 offset, uint64 len)
+	{
+	return false;
+	}
+
+void Entropy::Finalize()
+	{
+	//if ( ! entropy->IsValid() || ! fed )
+	if ( ! fed )
+		return;
+
+	val_list* vl = new val_list();
+	vl->append(GetFile()->GetVal()->Ref());
+
+	double montepi, scc, ent, mean, chisq;
+	montepi = scc = ent = mean = chisq = 0.0;
+	entropy->Get(&ent, &chisq, &mean, &montepi, &scc);
+
+	RecordVal* ent_result = new RecordVal(entropy_test_result);
+	ent_result->Assign(0, new Val(ent,     TYPE_DOUBLE));
+	ent_result->Assign(1, new Val(chisq,   TYPE_DOUBLE));
+	ent_result->Assign(2, new Val(mean,    TYPE_DOUBLE));
+	ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE));
+	ent_result->Assign(4, new Val(scc,     TYPE_DOUBLE));
+
+	vl->append(ent_result);
+	mgr.QueueEvent(file_entropy, vl);
+	}

src/file_analysis/analyzer/entropy/Entropy.h

@@ -0,0 +1,84 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef FILE_ANALYSIS_ENTROPY_H
+#define FILE_ANALYSIS_ENTROPY_H
+
+#include <string>
+
+#include "Val.h"
+#include "OpaqueVal.h"
+#include "File.h"
+#include "Analyzer.h"
+
+#include "events.bif.h"
+
+namespace file_analysis {
+
+/**
+ * An analyzer to produce a hash of file contents.
+ */
+class Entropy : public file_analysis::Analyzer {
+public:
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~Entropy();
+
+	/**
+	 * Create a new instance of an Extract analyzer.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @return the new Extract analyzer instance or a null pointer if the
+	 *         the "extraction_file" field of \a args wasn't set.
+	 */
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
+
+	/**
+	 * Incrementally hash next chunk of file contents.
+	 * @param data pointer to start of a chunk of a file data.
+	 * @param len number of bytes in the data chunk.
+	 * @return false if the digest is in an invalid state, else true.
+	 */
+	virtual bool DeliverStream(const u_char* data, uint64 len);
+
+	/**
+	 * Finalizes the hash and raises a "file_entropy_test" event.
+	 * @return always false so analyze will be deteched from file.
+	 */
+	virtual bool EndOfFile();
+
+	/**
+	 * Missing data can't be handled, so just indicate the this analyzer should
+	 * be removed from receiving further data.  The hash will not be finalized.
+	 * @param offset byte offset in file at which missing chunk starts.
+	 * @param len number of missing bytes.
+	 * @return always false so analyzer will detach from file.
+	 */
+	virtual bool Undelivered(uint64 offset, uint64 len);
+
+protected:
+
+	/**
+	 * Constructor.
+	 * @param args the \c AnalyzerArgs value which represents the analyzer.
+	 * @param file the file to which the analyzer will be attached.
+	 * @param hv specific hash calculator object.
+	 * @param kind human readable name of the hash algorithm to use.
+	 */
+	Entropy(RecordVal* args, File* file);
+
+	/**
+	 * If some file contents have been seen, finalizes the hash of them and
+	 * raises the "file_hash" event with the results.
+	 */
+	void Finalize();
+
+private:
+	EntropyVal* entropy;
+	bool fed;
+};
+
+} // namespace file_analysis
+
+#endif

src/file_analysis/analyzer/entropy/Plugin.cc

@@ -0,0 +1,24 @@
+// See the file  in the main distribution directory for copyright.
+
+#include "plugin/Plugin.h"
+
+#include "Entropy.h"
+
+namespace plugin {
+namespace Bro_FileEntropy {
+
+class Plugin : public plugin::Plugin {
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::file_analysis::Component("ENTROPY", ::file_analysis::Entropy::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::FileEntropy";
+		config.description = "Entropy test file content";
+		return config;
+		}
+} plugin;
+
+}
+}

src/file_analysis/analyzer/entropy/events.bif

@@ -0,0 +1,8 @@
+## This event is generated each time file analysis performs
+## entropy testing on a file.
+##
+## f: The file.
+##
+## ent: The results of the entropy testing.
+##
+event file_entropy%(f: fa_file, ent: entropy_test_result%);

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2016-04-12-20-52-34
+#open	2016-04-13-04-57-15
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -110,6 +110,7 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro
     build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro
     build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro
     build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro
@@ -129,4 +130,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2016-04-12-20-52-34
+#close	2016-04-13-04-57-15

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2016-04-12-20-52-45
+#open	2016-04-13-04-57-25
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -110,6 +110,7 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro
     build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro
     build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro
     build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro
@@ -300,4 +301,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2016-04-12-20-52-45
+#close	2016-04-13-04-57-25

testing/btest/Baseline/plugins.hooks/output

@@ -230,7 +230,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Communication::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Conn::LOG)) -> <no result>
@@ -351,7 +351,7 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::check_plugins, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::init, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
@@ -406,6 +406,7 @@
 0.000000   MetaHookPost  LoadFile(./Bro_FTP.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_FTP.functions.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_File.events.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./Bro_FileEntropy.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_FileExtract.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_FileExtract.functions.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_FileHash.events.bif.bro) -> -1
@@ -868,7 +869,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Communication::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Conn::LOG))
@@ -989,7 +990,7 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(NetControl::check_plugins, <frame>, ())
 0.000000   MetaHookPre   CallFunction(NetControl::init, <null>, ())
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, <frame>, ())
@@ -1044,6 +1045,7 @@
 0.000000   MetaHookPre   LoadFile(./Bro_FTP.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_FTP.functions.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_File.events.bif.bro)
+0.000000   MetaHookPre   LoadFile(./Bro_FileEntropy.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_FileExtract.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_FileExtract.functions.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_FileHash.events.bif.bro)
@@ -1505,7 +1507,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@@ -1626,7 +1628,7 @@
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction NetControl::check_plugins()
 0.000000 | HookCallFunction NetControl::init()
 0.000000 | HookCallFunction Notice::want_pp()

testing/btest/Baseline/scripts.base.files.entropy.basic/.stdout

@@ -0,0 +1 @@
+[entropy=4.950189, chi_square=63750.814665, mean=80.496493, monte_carlo_pi=4.0, serial_correlation=0.395907]

testing/btest/scripts/base/files/entropy/basic.test

@@ -0,0 +1,13 @@
+# @TEST-EXEC: bro -r $TRACES/http/get.trace %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+
+event file_new(f: fa_file)
+	{
+	Files::add_analyzer(f, Files::ANALYZER_ENTROPY);
+	}
+
+event file_entropy(f: fa_file, ent: entropy_test_result)
+	{
+	print ent;
+	}