diff --git a/CHANGES b/CHANGES index 0196cded10..890368dc53 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,21 @@ +2.4-452 | 2016-04-13 01:15:20 -0400 + + * Add a simple file entropy analyzer. (Seth Hall) + + * Analyzer and bro script for RFB/VNC protocol (Martin van Hensbergen) + + This analyzer parses the Remote Frame Buffer + protocol, usually referred to as the 'VNC protocol'. + + It supports several dialects (3.3, 3.7, 3.8) and + also handles the Apple Remote Desktop variant. + + It will log such facts as client/server versions, + authentication method used, authentication result, + height, width and name of the shared screen. + + 2.4-430 | 2016-04-07 13:36:36 -0700 * Fix regex literal in scripting documentation. (William Tom) diff --git a/NEWS b/NEWS index 86c3b1891a..7e66ace7aa 100644 --- a/NEWS +++ b/NEWS @@ -26,6 +26,11 @@ New Functionality - Bro now includes the NetControl framework. The framework allows for easy interaction of Bro with hard- and software switches, firewalls, etc. +- There is a new file entropy analyzer for files. + +- Bro now supports the remote framebuffer protocol (RFB) that is used by + VNC servers for remote graphical displays. + - Bro now supports the Radiotap header for 802.11 frames. - Bro now tracks VLAN IDs. To record them inside the connection log, diff --git a/VERSION b/VERSION index e7641ec404..030385b8f8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-430 +2.4-452 diff --git a/scripts/policy/frameworks/files/entropy-test-all-files.bro b/scripts/policy/frameworks/files/entropy-test-all-files.bro new file mode 100644 index 0000000000..fd02b9ecaa --- /dev/null +++ b/scripts/policy/frameworks/files/entropy-test-all-files.bro @@ -0,0 +1,20 @@ + +module Files; + +export { + redef record Files::Info += { + ## The information density of the contents of the file, + ## expressed as a number of bits per character. + entropy: double &log &optional; + }; +} + +event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_ENTROPY); + } + +event file_entropy(f: fa_file, ent: entropy_test_result) + { + f$info$entropy = ent$entropy; + } \ No newline at end of file diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index f0ca204005..f85fdb58b0 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -29,6 +29,7 @@ @load frameworks/intel/seen/where-locations.bro @load frameworks/intel/seen/x509.bro @load frameworks/files/detect-MHR.bro +@load frameworks/files/entropy-test-all-files.bro #@load frameworks/files/extract-all-files.bro @load frameworks/files/hash-all-files.bro @load frameworks/packet-filter/shunt.bro diff --git a/src/file_analysis/analyzer/CMakeLists.txt b/src/file_analysis/analyzer/CMakeLists.txt index 225504c56a..ef17247997 100644 --- a/src/file_analysis/analyzer/CMakeLists.txt +++ b/src/file_analysis/analyzer/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(data_event) +add_subdirectory(entropy) add_subdirectory(extract) add_subdirectory(hash) add_subdirectory(pe) diff --git a/src/file_analysis/analyzer/entropy/CMakeLists.txt b/src/file_analysis/analyzer/entropy/CMakeLists.txt new file mode 100644 index 0000000000..38db5e726a --- /dev/null +++ b/src/file_analysis/analyzer/entropy/CMakeLists.txt @@ -0,0 +1,9 @@ +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro FileEntropy) +bro_plugin_cc(Entropy.cc Plugin.cc ../../Analyzer.cc) +bro_plugin_bif(events.bif) +bro_plugin_end() diff --git a/src/file_analysis/analyzer/entropy/Entropy.cc b/src/file_analysis/analyzer/entropy/Entropy.cc new file mode 100644 index 0000000000..2a1bc72723 --- /dev/null +++ b/src/file_analysis/analyzer/entropy/Entropy.cc @@ -0,0 +1,71 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include "Entropy.h" +#include "util.h" +#include "Event.h" +#include "file_analysis/Manager.h" + +using namespace file_analysis; + +Entropy::Entropy(RecordVal* args, File* file) + : file_analysis::Analyzer(file_mgr->GetComponentTag("ENTROPY"), args, file) + { + //entropy->Init(); + entropy = new EntropyVal; + } + +Entropy::~Entropy() + { + Unref(entropy); + } + +file_analysis::Analyzer* Entropy::Instantiate(RecordVal* args, File* file) + { + return new Entropy(args, file); + } + +bool Entropy::DeliverStream(const u_char* data, uint64 len) + { + if ( ! fed ) + fed = len > 0; + + entropy->Feed(data, len); + return true; + } + +bool Entropy::EndOfFile() + { + Finalize(); + return false; + } + +bool Entropy::Undelivered(uint64 offset, uint64 len) + { + return false; + } + +void Entropy::Finalize() + { + //if ( ! entropy->IsValid() || ! fed ) + if ( ! fed ) + return; + + val_list* vl = new val_list(); + vl->append(GetFile()->GetVal()->Ref()); + + double montepi, scc, ent, mean, chisq; + montepi = scc = ent = mean = chisq = 0.0; + entropy->Get(&ent, &chisq, &mean, &montepi, &scc); + + RecordVal* ent_result = new RecordVal(entropy_test_result); + ent_result->Assign(0, new Val(ent, TYPE_DOUBLE)); + ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE)); + ent_result->Assign(2, new Val(mean, TYPE_DOUBLE)); + ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE)); + ent_result->Assign(4, new Val(scc, TYPE_DOUBLE)); + + vl->append(ent_result); + mgr.QueueEvent(file_entropy, vl); + } diff --git a/src/file_analysis/analyzer/entropy/Entropy.h b/src/file_analysis/analyzer/entropy/Entropy.h new file mode 100644 index 0000000000..6a5075263c --- /dev/null +++ b/src/file_analysis/analyzer/entropy/Entropy.h @@ -0,0 +1,84 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef FILE_ANALYSIS_ENTROPY_H +#define FILE_ANALYSIS_ENTROPY_H + +#include + +#include "Val.h" +#include "OpaqueVal.h" +#include "File.h" +#include "Analyzer.h" + +#include "events.bif.h" + +namespace file_analysis { + +/** + * An analyzer to produce a hash of file contents. + */ +class Entropy : public file_analysis::Analyzer { +public: + + /** + * Destructor. + */ + virtual ~Entropy(); + + /** + * Create a new instance of an Extract analyzer. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @return the new Extract analyzer instance or a null pointer if the + * the "extraction_file" field of \a args wasn't set. + */ + static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file); + + /** + * Incrementally hash next chunk of file contents. + * @param data pointer to start of a chunk of a file data. + * @param len number of bytes in the data chunk. + * @return false if the digest is in an invalid state, else true. + */ + virtual bool DeliverStream(const u_char* data, uint64 len); + + /** + * Finalizes the hash and raises a "file_entropy_test" event. + * @return always false so analyze will be deteched from file. + */ + virtual bool EndOfFile(); + + /** + * Missing data can't be handled, so just indicate the this analyzer should + * be removed from receiving further data. The hash will not be finalized. + * @param offset byte offset in file at which missing chunk starts. + * @param len number of missing bytes. + * @return always false so analyzer will detach from file. + */ + virtual bool Undelivered(uint64 offset, uint64 len); + +protected: + + /** + * Constructor. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @param hv specific hash calculator object. + * @param kind human readable name of the hash algorithm to use. + */ + Entropy(RecordVal* args, File* file); + + /** + * If some file contents have been seen, finalizes the hash of them and + * raises the "file_hash" event with the results. + */ + void Finalize(); + +private: + EntropyVal* entropy; + bool fed; +}; + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/analyzer/entropy/Plugin.cc b/src/file_analysis/analyzer/entropy/Plugin.cc new file mode 100644 index 0000000000..f1dd954cba --- /dev/null +++ b/src/file_analysis/analyzer/entropy/Plugin.cc @@ -0,0 +1,24 @@ +// See the file in the main distribution directory for copyright. + +#include "plugin/Plugin.h" + +#include "Entropy.h" + +namespace plugin { +namespace Bro_FileEntropy { + +class Plugin : public plugin::Plugin { +public: + plugin::Configuration Configure() + { + AddComponent(new ::file_analysis::Component("ENTROPY", ::file_analysis::Entropy::Instantiate)); + + plugin::Configuration config; + config.name = "Bro::FileEntropy"; + config.description = "Entropy test file content"; + return config; + } +} plugin; + +} +} diff --git a/src/file_analysis/analyzer/entropy/events.bif b/src/file_analysis/analyzer/entropy/events.bif new file mode 100644 index 0000000000..a51bb3d39b --- /dev/null +++ b/src/file_analysis/analyzer/entropy/events.bif @@ -0,0 +1,8 @@ +## This event is generated each time file analysis performs +## entropy testing on a file. +## +## f: The file. +## +## ent: The results of the entropy testing. +## +event file_entropy%(f: fa_file, ent: entropy_test_result%); \ No newline at end of file diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 6202ef3b6e..0a30ac0a71 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-12-20-52-34 +#open 2016-04-13-04-57-15 #fields name #types string scripts/base/init-bare.bro @@ -110,6 +110,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro + build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro @@ -129,4 +130,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2016-04-12-20-52-34 +#close 2016-04-13-04-57-15 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index cede55a98e..8b8dca3b12 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-12-20-52-45 +#open 2016-04-13-04-57-25 #fields name #types string scripts/base/init-bare.bro @@ -110,6 +110,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro + build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro @@ -300,4 +301,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-04-12-20-52-45 +#close 2016-04-13-04-57-25 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index e9b5c41650..5e771791ea 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -230,7 +230,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -351,7 +351,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -406,6 +406,7 @@ 0.000000 MetaHookPost LoadFile(./Bro_FTP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_FTP.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_File.events.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_FileEntropy.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_FileExtract.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_FileExtract.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_FileHash.events.bif.bro) -> -1 @@ -868,7 +869,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -989,7 +990,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1044,6 +1045,7 @@ 0.000000 MetaHookPre LoadFile(./Bro_FTP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_FTP.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_File.events.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_FileEntropy.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_FileExtract.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_FileExtract.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_FileHash.events.bif.bro) @@ -1505,7 +1507,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1626,7 +1628,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() diff --git a/testing/btest/Baseline/scripts.base.files.entropy.basic/.stdout b/testing/btest/Baseline/scripts.base.files.entropy.basic/.stdout new file mode 100644 index 0000000000..0682a357e8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.entropy.basic/.stdout @@ -0,0 +1 @@ +[entropy=4.950189, chi_square=63750.814665, mean=80.496493, monte_carlo_pi=4.0, serial_correlation=0.395907] diff --git a/testing/btest/scripts/base/files/entropy/basic.test b/testing/btest/scripts/base/files/entropy/basic.test new file mode 100644 index 0000000000..2b867eb8cb --- /dev/null +++ b/testing/btest/scripts/base/files/entropy/basic.test @@ -0,0 +1,13 @@ +# @TEST-EXEC: bro -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: btest-diff .stdout + + +event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_ENTROPY); + } + +event file_entropy(f: fa_file, ent: entropy_test_result) + { + print ent; + } \ No newline at end of file