Merge branch 'topic/seth/file-entropy'

* topic/seth/file-entropy:
  Add a file entropy test.
  Fixing a test.
  Updated tests for file entropy analyzer.
  Update and clean up to file entropy measurement.
  First commit of file entropy analyzer.
This commit is contained in:
Seth Hall 2016-04-13 01:15:20 -04:00
commit b722fe4540
16 changed files with 269 additions and 11 deletions

17
CHANGES
View file

@ -1,4 +1,21 @@
2.4-452 | 2016-04-13 01:15:20 -0400
* Add a simple file entropy analyzer. (Seth Hall)
* Analyzer and bro script for RFB/VNC protocol (Martin van Hensbergen)
This analyzer parses the Remote Frame Buffer
protocol, usually referred to as the 'VNC protocol'.
It supports several dialects (3.3, 3.7, 3.8) and
also handles the Apple Remote Desktop variant.
It will log such facts as client/server versions,
authentication method used, authentication result,
height, width and name of the shared screen.
2.4-430 | 2016-04-07 13:36:36 -0700
* Fix regex literal in scripting documentation. (William Tom)

5
NEWS
View file

@ -26,6 +26,11 @@ New Functionality
- Bro now includes the NetControl framework. The framework allows for easy
interaction of Bro with hard- and software switches, firewalls, etc.
- There is a new file entropy analyzer for files.
- Bro now supports the remote framebuffer protocol (RFB) that is used by
VNC servers for remote graphical displays.
- Bro now supports the Radiotap header for 802.11 frames.
- Bro now tracks VLAN IDs. To record them inside the connection log,

View file

@ -1 +1 @@
2.4-430
2.4-452

View file

@ -0,0 +1,20 @@
module Files;
export {
redef record Files::Info += {
## The information density of the contents of the file,
## expressed as a number of bits per character.
entropy: double &log &optional;
};
}
event file_new(f: fa_file)
{
Files::add_analyzer(f, Files::ANALYZER_ENTROPY);
}
event file_entropy(f: fa_file, ent: entropy_test_result)
{
f$info$entropy = ent$entropy;
}

View file

@ -29,6 +29,7 @@
@load frameworks/intel/seen/where-locations.bro
@load frameworks/intel/seen/x509.bro
@load frameworks/files/detect-MHR.bro
@load frameworks/files/entropy-test-all-files.bro
#@load frameworks/files/extract-all-files.bro
@load frameworks/files/hash-all-files.bro
@load frameworks/packet-filter/shunt.bro

View file

@ -1,4 +1,5 @@
add_subdirectory(data_event)
add_subdirectory(entropy)
add_subdirectory(extract)
add_subdirectory(hash)
add_subdirectory(pe)

View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro FileEntropy)
bro_plugin_cc(Entropy.cc Plugin.cc ../../Analyzer.cc)
bro_plugin_bif(events.bif)
bro_plugin_end()

View file

@ -0,0 +1,71 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include <string>
#include "Entropy.h"
#include "util.h"
#include "Event.h"
#include "file_analysis/Manager.h"
using namespace file_analysis;
Entropy::Entropy(RecordVal* args, File* file)
: file_analysis::Analyzer(file_mgr->GetComponentTag("ENTROPY"), args, file)
{
//entropy->Init();
entropy = new EntropyVal;
}
Entropy::~Entropy()
{
Unref(entropy);
}
file_analysis::Analyzer* Entropy::Instantiate(RecordVal* args, File* file)
{
return new Entropy(args, file);
}
bool Entropy::DeliverStream(const u_char* data, uint64 len)
{
if ( ! fed )
fed = len > 0;
entropy->Feed(data, len);
return true;
}
bool Entropy::EndOfFile()
{
Finalize();
return false;
}
bool Entropy::Undelivered(uint64 offset, uint64 len)
{
return false;
}
void Entropy::Finalize()
{
//if ( ! entropy->IsValid() || ! fed )
if ( ! fed )
return;
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
double montepi, scc, ent, mean, chisq;
montepi = scc = ent = mean = chisq = 0.0;
entropy->Get(&ent, &chisq, &mean, &montepi, &scc);
RecordVal* ent_result = new RecordVal(entropy_test_result);
ent_result->Assign(0, new Val(ent, TYPE_DOUBLE));
ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE));
ent_result->Assign(2, new Val(mean, TYPE_DOUBLE));
ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE));
ent_result->Assign(4, new Val(scc, TYPE_DOUBLE));
vl->append(ent_result);
mgr.QueueEvent(file_entropy, vl);
}

View file

@ -0,0 +1,84 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef FILE_ANALYSIS_ENTROPY_H
#define FILE_ANALYSIS_ENTROPY_H
#include <string>
#include "Val.h"
#include "OpaqueVal.h"
#include "File.h"
#include "Analyzer.h"
#include "events.bif.h"
namespace file_analysis {
/**
* An analyzer to produce a hash of file contents.
*/
class Entropy : public file_analysis::Analyzer {
public:
/**
* Destructor.
*/
virtual ~Entropy();
/**
* Create a new instance of an Extract analyzer.
* @param args the \c AnalyzerArgs value which represents the analyzer.
* @param file the file to which the analyzer will be attached.
* @return the new Extract analyzer instance or a null pointer if the
* the "extraction_file" field of \a args wasn't set.
*/
static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
/**
* Incrementally hash next chunk of file contents.
* @param data pointer to start of a chunk of a file data.
* @param len number of bytes in the data chunk.
* @return false if the digest is in an invalid state, else true.
*/
virtual bool DeliverStream(const u_char* data, uint64 len);
/**
* Finalizes the hash and raises a "file_entropy_test" event.
* @return always false so analyze will be deteched from file.
*/
virtual bool EndOfFile();
/**
* Missing data can't be handled, so just indicate the this analyzer should
* be removed from receiving further data. The hash will not be finalized.
* @param offset byte offset in file at which missing chunk starts.
* @param len number of missing bytes.
* @return always false so analyzer will detach from file.
*/
virtual bool Undelivered(uint64 offset, uint64 len);
protected:
/**
* Constructor.
* @param args the \c AnalyzerArgs value which represents the analyzer.
* @param file the file to which the analyzer will be attached.
* @param hv specific hash calculator object.
* @param kind human readable name of the hash algorithm to use.
*/
Entropy(RecordVal* args, File* file);
/**
* If some file contents have been seen, finalizes the hash of them and
* raises the "file_hash" event with the results.
*/
void Finalize();
private:
EntropyVal* entropy;
bool fed;
};
} // namespace file_analysis
#endif

View file

@ -0,0 +1,24 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "Entropy.h"
namespace plugin {
namespace Bro_FileEntropy {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::file_analysis::Component("ENTROPY", ::file_analysis::Entropy::Instantiate));
plugin::Configuration config;
config.name = "Bro::FileEntropy";
config.description = "Entropy test file content";
return config;
}
} plugin;
}
}

View file

@ -0,0 +1,8 @@
## This event is generated each time file analysis performs
## entropy testing on a file.
##
## f: The file.
##
## ent: The results of the entropy testing.
##
event file_entropy%(f: fa_file, ent: entropy_test_result%);

View file

@ -3,7 +3,7 @@
#empty_field (empty)
#unset_field -
#path loaded_scripts
#open 2016-04-12-20-52-34
#open 2016-04-13-04-57-15
#fields name
#types string
scripts/base/init-bare.bro
@ -110,6 +110,7 @@ scripts/base/init-bare.bro
build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro
build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro
build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro
build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro
build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro
build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro
build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro
@ -129,4 +130,4 @@ scripts/base/init-bare.bro
build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro
scripts/policy/misc/loaded-scripts.bro
scripts/base/utils/paths.bro
#close 2016-04-12-20-52-34
#close 2016-04-13-04-57-15

View file

@ -3,7 +3,7 @@
#empty_field (empty)
#unset_field -
#path loaded_scripts
#open 2016-04-12-20-52-45
#open 2016-04-13-04-57-25
#fields name
#types string
scripts/base/init-bare.bro
@ -110,6 +110,7 @@ scripts/base/init-bare.bro
build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro
build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro
build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro
build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro
build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro
build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro
build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro
@ -300,4 +301,4 @@ scripts/base/init-default.bro
scripts/base/misc/find-checksum-offloading.bro
scripts/base/misc/find-filtered-trace.bro
scripts/policy/misc/loaded-scripts.bro
#close 2016-04-12-20-52-45
#close 2016-04-13-04-57-25

View file

@ -230,7 +230,7 @@
0.000000 MetaHookPost CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
0.000000 MetaHookPost CallFunction(Log::add_default_filter, <frame>, (Communication::LOG)) -> <no result>
0.000000 MetaHookPost CallFunction(Log::add_default_filter, <frame>, (Conn::LOG)) -> <no result>
@ -351,7 +351,7 @@
0.000000 MetaHookPost CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
0.000000 MetaHookPost CallFunction(NetControl::check_plugins, <frame>, ()) -> <no result>
0.000000 MetaHookPost CallFunction(NetControl::init, <null>, ()) -> <no result>
0.000000 MetaHookPost CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
@ -406,6 +406,7 @@
0.000000 MetaHookPost LoadFile(./Bro_FTP.events.bif.bro) -> -1
0.000000 MetaHookPost LoadFile(./Bro_FTP.functions.bif.bro) -> -1
0.000000 MetaHookPost LoadFile(./Bro_File.events.bif.bro) -> -1
0.000000 MetaHookPost LoadFile(./Bro_FileEntropy.events.bif.bro) -> -1
0.000000 MetaHookPost LoadFile(./Bro_FileExtract.events.bif.bro) -> -1
0.000000 MetaHookPost LoadFile(./Bro_FileExtract.functions.bif.bro) -> -1
0.000000 MetaHookPost LoadFile(./Bro_FileHash.events.bif.bro) -> -1
@ -868,7 +869,7 @@
0.000000 MetaHookPre CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
0.000000 MetaHookPre CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
0.000000 MetaHookPre CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
0.000000 MetaHookPre CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
0.000000 MetaHookPre CallFunction(Log::add_default_filter, <frame>, (Communication::LOG))
0.000000 MetaHookPre CallFunction(Log::add_default_filter, <frame>, (Conn::LOG))
@ -989,7 +990,7 @@
0.000000 MetaHookPre CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
0.000000 MetaHookPre CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
0.000000 MetaHookPre CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
0.000000 MetaHookPre CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(NetControl::check_plugins, <frame>, ())
0.000000 MetaHookPre CallFunction(NetControl::init, <null>, ())
0.000000 MetaHookPre CallFunction(Notice::want_pp, <frame>, ())
@ -1044,6 +1045,7 @@
0.000000 MetaHookPre LoadFile(./Bro_FTP.events.bif.bro)
0.000000 MetaHookPre LoadFile(./Bro_FTP.functions.bif.bro)
0.000000 MetaHookPre LoadFile(./Bro_File.events.bif.bro)
0.000000 MetaHookPre LoadFile(./Bro_FileEntropy.events.bif.bro)
0.000000 MetaHookPre LoadFile(./Bro_FileExtract.events.bif.bro)
0.000000 MetaHookPre LoadFile(./Bro_FileExtract.functions.bif.bro)
0.000000 MetaHookPre LoadFile(./Bro_FileHash.events.bif.bro)
@ -1505,7 +1507,7 @@
0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@ -1626,7 +1628,7 @@
0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1460494592.785314, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1460523470.220624, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction NetControl::check_plugins()
0.000000 | HookCallFunction NetControl::init()
0.000000 | HookCallFunction Notice::want_pp()

View file

@ -0,0 +1 @@
[entropy=4.950189, chi_square=63750.814665, mean=80.496493, monte_carlo_pi=4.0, serial_correlation=0.395907]

View file

@ -0,0 +1,13 @@
# @TEST-EXEC: bro -r $TRACES/http/get.trace %INPUT
# @TEST-EXEC: btest-diff .stdout
event file_new(f: fa_file)
{
Files::add_analyzer(f, Files::ANALYZER_ENTROPY);
}
event file_entropy(f: fa_file, ent: entropy_test_result)
{
print ent;
}