Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-04 07:38:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/master' into topic/seth/files-tracking

Browse source 
Conflicts:
	scripts/base/frameworks/files/main.bro
	src/file_analysis/File.cc
	testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out
This commit is contained in:
Seth Hall 2014-09-23 13:05:39 -04:00
commit 42b2d56279
486 changed files with 106378 additions and 85985 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/file_analysis/Analyzer.cc
View file

@ -3,9 +3,17 @@
#include "Analyzer.h"
#include "Manager.h"
file_analysis::ID file_analysis::Analyzer::id_counter = 0;
file_analysis::Analyzer::~Analyzer()
{
DBG_LOG(DBG_FILE_ANALYSIS, "Destroy file analyzer %s",
file_mgr->GetComponentName(tag));
file_mgr->GetComponentName(tag).c_str());
Unref(args);
}
void file_analysis::Analyzer::SetAnalyzerTag(const file_analysis::Tag& arg_tag)
{
assert(! tag || tag == arg_tag);
tag = arg_tag;
}

54
src/file_analysis/Analyzer.h
View file

@ -13,6 +13,8 @@ namespace file_analysis {
class File;
typedef uint32 ID;
/**
* Base class for analyzers that can be attached to file_analysis::File objects.
*/
@ -25,6 +27,18 @@ public:
*/
virtual ~Analyzer();
/**
* Initializes the analyzer before input processing starts.
*/
virtual void Init()
{ }
/**
* Finishes the analyzer's operation after all input has been parsed.
*/
virtual void Done()
{ }
/**
* Subclasses may override this metod to receive file data non-sequentially.
* @param data points to start of a chunk of file data.
@ -72,6 +86,13 @@ public:
*/
file_analysis::Tag Tag() const { return tag; }
/**
* Returns the analyzer instance's internal ID. These IDs are unique
* across all analyzers instantiated and can thus be used to
* indentify a specific instance.
*/
ID GetID() const { return id; }
/**
* @return the AnalyzerArgs associated with the analyzer.
*/
@ -82,10 +103,19 @@ public:
*/
File* GetFile() const { return file; }
/**
* Sets the tag associated with the analyzer's type. Note that this
* can be called only right after construction, if the constructor
* did not receive a name or tag. The method cannot be used to change
* an existing tag.
*/
void SetAnalyzerTag(const file_analysis::Tag& tag);
protected:
/**
* Constructor. Only derived classes are meant to be instantiated.
* @param arg_tag the tag definining the analyzer's type.
* @param arg_args an \c AnalyzerArgs (script-layer type) value specifiying
* tunable options, if any, related to a particular analyzer type.
* @param arg_file the file to which the the analyzer is being attached.
@ -94,13 +124,35 @@ protected:
: tag(arg_tag),
args(arg_args->Ref()->AsRecordVal()),
file(arg_file)
{}
{
id = ++id_counter;
}
/**
* Constructor. Only derived classes are meant to be instantiated.
* As this version of the constructor does not receive a name or tag,
* SetAnalyzerTag() must be called before the instance can be used.
*
* @param arg_args an \c AnalyzerArgs (script-layer type) value specifiying
* tunable options, if any, related to a particular analyzer type.
* @param arg_file the file to which the the analyzer is being attached.
*/
Analyzer(RecordVal* arg_args, File* arg_file)
: tag(),
args(arg_args->Ref()->AsRecordVal()),
file(arg_file)
{
id = ++id_counter;
}
private:
ID id; /**< Unique instance ID. */
file_analysis::Tag tag; /**< The particular type of the analyzer instance. */
RecordVal* args; /**< \c AnalyzerArgs val gives tunable analyzer params. */
File* file; /**< The file to which the analyzer is attached. */
static ID id_counter;
};
} // namespace file_analysis

22
src/file_analysis/AnalyzerSet.cc
View file

@ -9,7 +9,10 @@ using namespace file_analysis;
static void analyzer_del_func(void* v)
{
delete (file_analysis::Analyzer*) v;
file_analysis::Analyzer* a = (file_analysis::Analyzer*)v;
a->Done();
delete a;
}
AnalyzerSet::AnalyzerSet(File* arg_file) : file(arg_file)
@ -50,7 +53,7 @@ bool AnalyzerSet::Add(file_analysis::Tag tag, RecordVal* args)
if ( analyzer_map.Lookup(key) )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %s skipped for file id"
" %s: already exists", file_mgr->GetComponentName(tag),
" %s: already exists", file_mgr->GetComponentName(tag).c_str(),
file->GetID().c_str());
delete key;
return true;
@ -90,7 +93,7 @@ bool AnalyzerSet::AddMod::Perform(AnalyzerSet* set)
if ( set->analyzer_map.Lookup(key) )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Add analyzer %s skipped for file id"
" %s: already exists", file_mgr->GetComponentName(a->Tag()),
" %s: already exists", file_mgr->GetComponentName(a->Tag()).c_str(),
a->GetFile()->GetID().c_str());
Abort();
@ -98,6 +101,7 @@ bool AnalyzerSet::AddMod::Perform(AnalyzerSet* set)
}
set->Insert(a, key);
return true;
}
@ -116,15 +120,17 @@ bool AnalyzerSet::Remove(file_analysis::Tag tag, HashKey* key)
if ( ! a )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Skip remove analyzer %s for file id %s",
file_mgr->GetComponentName(tag), file->GetID().c_str());
file_mgr->GetComponentName(tag).c_str(), file->GetID().c_str());
return false;
}
DBG_LOG(DBG_FILE_ANALYSIS, "Remove analyzer %s for file id %s",
file_mgr->GetComponentName(tag),
file_mgr->GetComponentName(tag).c_str(),
file->GetID().c_str());
a->Done();
delete a;
return true;
}
@ -163,7 +169,7 @@ file_analysis::Analyzer* AnalyzerSet::InstantiateAnalyzer(Tag tag,
if ( ! a )
{
reporter->Error("Failed file analyzer %s instantiation for file id %s",
file_mgr->GetComponentName(tag), file->GetID().c_str());
file_mgr->GetComponentName(tag).c_str(), file->GetID().c_str());
return 0;
}
@ -173,9 +179,11 @@ file_analysis::Analyzer* AnalyzerSet::InstantiateAnalyzer(Tag tag,
void AnalyzerSet::Insert(file_analysis::Analyzer* a, HashKey* key)
{
DBG_LOG(DBG_FILE_ANALYSIS, "Add analyzer %s for file id %s",
file_mgr->GetComponentName(a->Tag()), file->GetID().c_str());
file_mgr->GetComponentName(a->Tag()).c_str(), file->GetID().c_str());
analyzer_map.Insert(key, a);
delete key;
a->Init();
}
void AnalyzerSet::DrainModifications()

42
src/file_analysis/Component.cc
View file

@ -8,54 +8,24 @@
using namespace file_analysis;
Component::Component(const char* arg_name, factory_callback arg_factory)
: plugin::Component(plugin::component::FILE_ANALYZER),
plugin::TaggedComponent<file_analysis::Tag>()
Component::Component(const std::string& name, factory_callback arg_factory, Tag::subtype_t subtype)
: plugin::Component(plugin::component::FILE_ANALYZER, name),
plugin::TaggedComponent<file_analysis::Tag>(subtype)
{
name = copy_string(arg_name);
canon_name = canonify_name(arg_name);
factory = arg_factory;
}
Component::Component(const Component& other)
: plugin::Component(Type()),
plugin::TaggedComponent<file_analysis::Tag>(other)
{
name = copy_string(other.name);
canon_name = copy_string(other.canon_name);
factory = other.factory;
file_mgr->RegisterComponent(this, "ANALYZER_");
}
Component::~Component()
{
delete [] name;
delete [] canon_name;
}
void Component::Describe(ODesc* d) const
void Component::DoDescribe(ODesc* d) const
{
plugin::Component::Describe(d);
d->Add(name);
d->Add(" (");
if ( factory )
{
d->Add("ANALYZER_");
d->Add(canon_name);
d->Add(CanonicalName());
}
d->Add(")");
}
Component& Component::operator=(const Component& other)
{
plugin::TaggedComponent<file_analysis::Tag>::operator=(other);
if ( &other != this )
{
name = copy_string(other.name);
factory = other.factory;
}
return *this;
}

45
src/file_analysis/Component.h
View file

@ -1,7 +1,7 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef FILE_ANALYZER_PLUGIN_COMPONENT_H
#define FILE_ANALYZER_PLUGIN_COMPONENT_H
#ifndef FILE_ANALYZER_COMPONENT_H
#define FILE_ANALYZER_COMPONENT_H
#include "Tag.h"
#include "plugin/Component.h"
@ -40,51 +40,32 @@ public:
* from file_analysis::Analyzer. This is typically a static \c
* Instatiate() method inside the class that just allocates and
* returns a new instance.
*
* @param subtype A subtype associated with this component that
* further distinguishes it. The subtype will be integrated into the
* analyzer::Tag that the manager associates with this analyzer, and
* analyzer instances can accordingly access it via analyzer::Tag().
* If not used, leave at zero.
*/
Component(const char* name, factory_callback factory);
/**
* Copy constructor.
*/
Component(const Component& other);
Component(const std::string& name, factory_callback factory, Tag::subtype_t subtype = 0);
/**
* Destructor.
*/
~Component();
/**
* Returns the name of the analyzer. This name is unique across all
* analyzers and used to identify it. The returned name is derived
* from what's passed to the constructor but upper-cased and
* canonified to allow being part of a script-level ID.
*/
virtual const char* Name() const { return name; }
/**
* Returns a canonocalized version of the analyzer's name. The
* returned name is derived from what's passed to the constructor but
* upper-cased and transformed to allow being part of a script-level
* ID.
*/
const char* CanonicalName() const { return canon_name; }
/**
* Returns the analyzer's factory function.
*/
factory_callback Factory() const { return factory; }
protected:
/**
* Generates a human-readable description of the component's main
* parameters. This goes into the output of \c "bro -NN".
*/
virtual void Describe(ODesc* d) const;
Component& operator=(const Component& other);
* Overriden from plugin::Component.
*/
virtual void DoDescribe(ODesc* d) const;
private:
const char* name; // The analyzer's name.
const char* canon_name; // The analyzer's canonical name.
factory_callback factory; // The analyzer's factory callback.
};

30
src/file_analysis/File.cc
View file

@ -1,6 +1,7 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include <string>
#include <algorithm>
#include "File.h"
#include "FileTimer.h"
@ -85,7 +86,7 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
{
StaticInit();
DBG_LOG(DBG_FILE_ANALYSIS, "Creating new File object %s", file_id.c_str());
DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Creating new File object", file_id.c_str());
val = new RecordVal(fa_file_type);
val->Assign(id_idx, new StringVal(file_id.c_str()));
@ -104,7 +105,7 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
File::~File()
{
DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str());
DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Destroying File object", id.c_str());
Unref(val);
while ( ! fonc_queue.empty() )
@ -235,6 +236,7 @@ void File::IncrementByteCount(uint64 size, int field_idx)
void File::SetTotalBytes(uint64 size)
{
DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Total bytes %" PRIu64, id.c_str(), size);
val->Assign(total_bytes_idx, new Val(size, TYPE_COUNT));
}
@ -257,11 +259,17 @@ void File::ScheduleInactivityTimer() const
bool File::AddAnalyzer(file_analysis::Tag tag, RecordVal* args)
{
DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Queuing addition of %s analyzer",
id.c_str(), file_mgr->GetComponentName(tag).c_str());
return done ? false : analyzers.QueueAdd(tag, args);
}
bool File::RemoveAnalyzer(file_analysis::Tag tag, RecordVal* args)
{
DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Queuing remove of %s analyzer",
id.c_str(), file_mgr->GetComponentName(tag).c_str());
return done ? false : analyzers.QueueRemove(tag, args);
}
@ -334,8 +342,6 @@ void File::ReplayBOF()
}
BroString* bs = concatenate(bof_buffer.chunks);
val->Assign(bof_buffer_idx, new StringVal(bs));
for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i )
DataIn(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len());
}
@ -352,6 +358,11 @@ void File::DeliverStream(const u_char* data, uint64 len)
FileEvent(file_new);
}
DBG_LOG(DBG_FILE_ANALYSIS, "[%s] %" PRIu64 " bytes in at offset" PRIu64 "; %s [%s]",
id.c_str(), len, offset,
IsComplete() ? "complete" : "incomplete",
fmt_bytes((const char*) data, min((uint64)40, len)), len > 40 ? "..." : "");
file_analysis::Analyzer* a = 0;
IterCookie* c = analyzers.InitForIteration();
while ( (a = analyzers.NextEntry(c)) )
@ -411,7 +422,11 @@ void File::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
IncrementByteCount(len, overflow_bytes_idx);
}
// Deliver to the chunk analyzers.
DBG_LOG(DBG_FILE_ANALYSIS, "[%s] %" PRIu64 " bytes in; %s [%s]",
id.c_str(), len,
IsComplete() ? "complete" : "incomplete",
fmt_bytes((const char*) data, min((uint64)40, len)), len > 40 ? "..." : "");
file_analysis::Analyzer* a = 0;
IterCookie* c = analyzers.InitForIteration();
while ( (a = analyzers.NextEntry(c)) )
@ -449,6 +464,8 @@ void File::DataIn(const u_char* data, uint64 len)
void File::EndOfFile()
{
DBG_LOG(DBG_FILE_ANALYSIS, "[%s] End of file", id.c_str());
if ( done )
return;
@ -475,6 +492,9 @@ void File::EndOfFile()
void File::Gap(uint64 offset, uint64 len)
{
DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Gap of size %" PRIu64 " at offset %" PRIu64,
id.c_str(), len, offset);
analyzers.DrainModifications();
// If we were buffering the beginning of the file, a gap means we've got

109
src/file_analysis/Manager.cc
View file

@ -12,21 +12,26 @@
#include "UID.h"
#include "plugin/Manager.h"
#include "analyzer/Manager.h"
using namespace file_analysis;
TableVal* Manager::disabled = 0;
TableType* Manager::tag_set_type = 0;
string Manager::salt;
Manager::Manager()
: plugin::ComponentManager<file_analysis::Tag,
file_analysis::Component>("Files"),
file_analysis::Component>("Files", "Tag"),
id_map(), ignored(), current_file_id(), magic_state()
{
}
Manager::~Manager()
{
for ( MIMEMap::iterator i = mime_types.begin(); i != mime_types.end(); i++ )
delete i->second;
// Have to assume that too much of Bro has been shutdown by this point
// to do anything more than reclaim memory.
@ -48,11 +53,6 @@ Manager::~Manager()
void Manager::InitPreScript()
{
std::list<Component*> analyzers = plugin_mgr->Components<Component>();
for ( std::list<Component*>::const_iterator i = analyzers.begin();
i != analyzers.end(); ++i )
RegisterComponent(*i, "ANALYZER_");
}
void Manager::InitPostScript()
@ -104,6 +104,7 @@ void Manager::SetHandle(const string& handle)
if ( handle.empty() )
return;
DBG_LOG(DBG_FILE_ANALYSIS, "Set current handle to %s", handle.c_str());
current_file_id = HashHandle(handle);
}
@ -286,6 +287,28 @@ bool Manager::AddAnalyzer(const string& file_id, file_analysis::Tag tag,
return file->AddAnalyzer(tag, args);
}
TableVal* Manager::AddAnalyzersForMIMEType(const string& file_id, const string& mtype,
RecordVal* args)
{
if ( ! tag_set_type )
tag_set_type = internal_type("files_tag_set")->AsTableType();
TableVal* sval = new TableVal(tag_set_type);
TagSet* l = LookupMIMEType(mtype, false);
if ( ! l )
return sval;
for ( TagSet::const_iterator i = l->begin(); i != l->end(); i++ )
{
file_analysis::Tag tag = *i;
if ( AddAnalyzer(file_id, tag, args) )
sval->Assign(tag.AsEnumVal(), 0);
}
return sval;
}
bool Manager::RemoveAnalyzer(const string& file_id, file_analysis::Tag tag,
RecordVal* args) const
{
@ -405,6 +428,9 @@ string Manager::GetFileID(analyzer::Tag tag, Connection* c, bool is_orig)
if ( ! get_file_handle )
return "";
DBG_LOG(DBG_FILE_ANALYSIS, "Raise get_file_handle() for protocol analyzer %s",
analyzer_mgr->GetComponentName(tag).c_str());
EnumVal* tagval = tag.AsEnumVal();
Ref(tagval);
@ -451,11 +477,78 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const
if ( ! c->Factory() )
{
reporter->InternalWarning("file analyzer %s cannot be instantiated "
"dynamically", c->CanonicalName());
"dynamically", c->CanonicalName().c_str());
return 0;
}
return c->Factory()(args, f);
DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %s for file %s",
GetComponentName(tag).c_str(), f->id.c_str());
Analyzer* a = c->Factory()(args, f);
if ( ! a )
reporter->InternalError("file analyzer instantiation failed");
a->SetAnalyzerTag(tag);
return a;
}
Manager::TagSet* Manager::LookupMIMEType(const string& mtype, bool add_if_not_found)
{
MIMEMap::const_iterator i = mime_types.find(to_upper(mtype));
if ( i != mime_types.end() )
return i->second;
if ( ! add_if_not_found )
return 0;
TagSet* l = new TagSet;
mime_types.insert(std::make_pair(to_upper(mtype), l));
return l;
}
bool Manager::RegisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype)
{
Component* p = Lookup(tag);
if ( ! p )
return false;
return RegisterAnalyzerForMIMEType(p->Tag(), mtype->CheckString());
}
bool Manager::RegisterAnalyzerForMIMEType(Tag tag, const string& mtype)
{
TagSet* l = LookupMIMEType(mtype, true);
DBG_LOG(DBG_FILE_ANALYSIS, "Register analyzer %s for MIME type %s",
GetComponentName(tag).c_str(), mtype.c_str());
l->insert(tag);
return true;
}
bool Manager::UnregisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype)
{
Component* p = Lookup(tag);
if ( ! p )
return false;
return UnregisterAnalyzerForMIMEType(p->Tag(), mtype->CheckString());
}
bool Manager::UnregisterAnalyzerForMIMEType(Tag tag, const string& mtype)
{
TagSet* l = LookupMIMEType(mtype, true);
DBG_LOG(DBG_FILE_ANALYSIS, "Unregister analyzer %s for MIME type %s",
GetComponentName(tag).c_str(), mtype.c_str());
l->erase(tag);
return true;
}
RuleMatcher::MIME_Matches* Manager::DetectMIME(const u_char* data, uint64 len,

74
src/file_analysis/Manager.h
View file

@ -253,6 +253,18 @@ public:
bool AddAnalyzer(const string& file_id, file_analysis::Tag tag,
RecordVal* args) const;
/**
* Queue attachment of an all analyzers associated with a given MIME
* type to the file identifier.
*
* @param file_id the file identifier/hash.
* @param mtype the MIME type; comparisions will be performanced case-insensitive.
* @param args a \c AnalyzerArgs value which describes a file analyzer.
* @return A ref'ed \c set[Tag] with all added analyzers.
*/
TableVal* AddAnalyzersForMIMEType(const string& file_id, const string& mtype,
RecordVal* args);
/**
* Queue removal of an analyzer for a given file identifier.
* @param file_id the file identifier/hash.
@ -280,6 +292,62 @@ public:
Analyzer* InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const;
/**
* Registers a MIME type for an analyzer. Once registered, files of
* that MIME type will automatically get a corresponding analyzer
* assigned.
*
* @param tag The analyzer's tag as an enum of script type \c
* Files::Tag.
*
* @param mtype The MIME type. It will be matched case-insenistive.
*
* @return True if successful.
*/
bool RegisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype);
/**
* Registers a MIME type for an analyzer. Once registered, files of
* that MIME type will automatically get a corresponding analyzer
* assigned.
*
* @param tag The analyzer's tag as an enum of script type \c
* Files::Tag.
*
* @param mtype The MIME type. It will be matched case-insenistive.
*
* @return True if successful.
*/
bool RegisterAnalyzerForMIMEType(Tag tag, const string& mtype);
/**
* Unregisters a MIME type for an analyzer.
*
* @param tag The analyzer's tag as an enum of script type \c
* Files::Tag.
*
* @param mtype The MIME type. It will be matched case-insenistive.
*
* @return True if successful (incl. when the type wasn't actually
* registered for the analyzer).
*
*/
bool UnregisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype);
/**
* Unregisters a MIME type for an analyzer.
*
* @param tag The analyzer's tag as an enum of script type \c
* Files::Tag.
*
* @param mtype The MIME type. It will be matched case-insenistive.
*
* @return True if successful (incl. when the type wasn't actually
* registered for the analyzer).
*
*/
bool UnregisterAnalyzerForMIMEType(Tag tag, const string& mtype);
/**
* Returns a set of all matching MIME magic signatures for a given
* chunk of data.
* @param data A chunk of bytes to match magic MIME signatures against.
@ -377,13 +445,19 @@ protected:
static bool IsDisabled(analyzer::Tag tag);
private:
typedef set<Tag> TagSet;
typedef map<string, TagSet*> MIMEMap;
TagSet* LookupMIMEType(const string& mtype, bool add_if_not_found);
PDict(File) id_map; /**< Map file ID to file_analysis::File records. */
PDict(bool) ignored; /**< Ignored files. Will be finally removed on EOF. */
string current_file_id; /**< Hash of what get_file_handle event sets. */
RuleFileMagicState* magic_state; /**< File magic signature match state. */
MIMEMap mime_types;/**< Mapping of MIME types to analyzers. */
static TableVal* disabled; /**< Table of disabled analyzers. */
static TableType* tag_set_type; /**< Type for set[tag]. */
static string salt; /**< A salt added to file handles before hashing. */
};

24
src/file_analysis/analyzer/data_event/Plugin.cc
View file

@ -1,8 +1,24 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "DataEvent.h"
BRO_PLUGIN_BEGIN(Bro, FileDataEvent)
BRO_PLUGIN_DESCRIPTION("Delivers file content via events");
BRO_PLUGIN_FILE_ANALYZER("DATA_EVENT", DataEvent);
BRO_PLUGIN_END
namespace plugin {
namespace Bro_FileDataEvent {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::file_analysis::Component("DATA_EVENT", ::file_analysis::DataEvent::Instantiate));
plugin::Configuration config;
config.name = "Bro::FileDataEvent";
config.description = "Delivers file content";
return config;
}
} plugin;
}
}

26
src/file_analysis/analyzer/extract/Plugin.cc
View file

@ -1,10 +1,24 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "Extract.h"
BRO_PLUGIN_BEGIN(Bro, FileExtract)
BRO_PLUGIN_DESCRIPTION("Extract file content to local file system");
BRO_PLUGIN_FILE_ANALYZER("EXTRACT", Extract);
BRO_PLUGIN_BIF_FILE(events);
BRO_PLUGIN_BIF_FILE(functions);
BRO_PLUGIN_END
namespace plugin {
namespace Bro_FileExtract {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::file_analysis::Component("EXTRACT", ::file_analysis::Extract::Instantiate));
plugin::Configuration config;
config.name = "Bro::FileExtract";
config.description = "Extract file content";
return config;
}
} plugin;
}
}

29
src/file_analysis/analyzer/hash/Plugin.cc
View file

@ -1,11 +1,26 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "Hash.h"
BRO_PLUGIN_BEGIN(Bro, FileHash)
BRO_PLUGIN_DESCRIPTION("Hash file content");
BRO_PLUGIN_FILE_ANALYZER("MD5", MD5);
BRO_PLUGIN_FILE_ANALYZER("SHA1", SHA1);
BRO_PLUGIN_FILE_ANALYZER("SHA256", SHA256);
BRO_PLUGIN_BIF_FILE(events);
BRO_PLUGIN_END
namespace plugin {
namespace Bro_FileHash {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::file_analysis::Component("MD5", ::file_analysis::MD5::Instantiate));
AddComponent(new ::file_analysis::Component("SHA1", ::file_analysis::SHA1::Instantiate));
AddComponent(new ::file_analysis::Component("SHA256", ::file_analysis::SHA256::Instantiate));
plugin::Configuration config;
config.name = "Bro::FileHash";
config.description = "Hash file content";
return config;
}
} plugin;
}
}

26
src/file_analysis/analyzer/unified2/Plugin.cc
View file

@ -1,12 +1,26 @@
// See the file in the main distribution directory for copyright.
// See the file "COPYING" in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "Unified2.h"
BRO_PLUGIN_BEGIN(Bro, Unified2)
BRO_PLUGIN_DESCRIPTION("Analyze Unified2 alert files.");
BRO_PLUGIN_FILE_ANALYZER("UNIFIED2", Unified2);
BRO_PLUGIN_BIF_FILE(events);
BRO_PLUGIN_BIF_FILE(types);
BRO_PLUGIN_END
namespace plugin {
namespace Bro_Unified2 {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::file_analysis::Component("UNIFIED2", ::file_analysis::Unified2::Instantiate));
plugin::Configuration config;
config.name = "Bro::Unified2";
config.description = "Analyze Unified2 alert files.";
return config;
}
} plugin;
}
}

28
src/file_analysis/analyzer/x509/Plugin.cc
View file

@ -1,11 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "X509.h"
BRO_PLUGIN_BEGIN(Bro, X509)
BRO_PLUGIN_DESCRIPTION("X509 certificate parser");
BRO_PLUGIN_FILE_ANALYZER("X509", X509);
BRO_PLUGIN_BIF_FILE(events);
BRO_PLUGIN_BIF_FILE(types);
BRO_PLUGIN_BIF_FILE(functions);
BRO_PLUGIN_END
namespace plugin {
namespace Bro_X509 {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::file_analysis::Component("X509", ::file_analysis::X509::Instantiate));
plugin::Configuration config;
config.name = "Bro::X509";
config.description = "X509 analyzer";
return config;
}
} plugin;
}
}

10
src/file_analysis/analyzer/x509/X509.cc
View file

@ -88,7 +88,7 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
{
::X509* ssl_cert = cert_val->GetCertificate();
char buf[256]; // we need a buffer for some of the openssl functions
char buf[2048]; // we need a buffer for some of the openssl functions
memset(buf, 0, sizeof(buf));
RecordVal* pX509Cert = new RecordVal(BifType::Record::X509::Certificate);
@ -96,14 +96,16 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
pX509Cert->Assign(0, new Val((uint64) X509_get_version(ssl_cert) + 1, TYPE_COUNT));
i2a_ASN1_INTEGER(bio, X509_get_serialNumber(ssl_cert));
int len = BIO_read(bio, &(*buf), sizeof(buf));
int len = BIO_read(bio, buf, sizeof(buf));
pX509Cert->Assign(1, new StringVal(len, buf));
BIO_reset(bio);
X509_NAME_print_ex(bio, X509_get_subject_name(ssl_cert), 0, XN_FLAG_RFC2253);
len = BIO_gets(bio, &(*buf), sizeof(buf));
len = BIO_gets(bio, buf, sizeof(buf));
pX509Cert->Assign(2, new StringVal(len, buf));
BIO_reset(bio);
X509_NAME_print_ex(bio, X509_get_issuer_name(ssl_cert), 0, XN_FLAG_RFC2253);
len = BIO_gets(bio, &(*buf), sizeof(buf));
len = BIO_gets(bio, buf, sizeof(buf));
pX509Cert->Assign(3, new StringVal(len, buf));
BIO_free(bio);

69
src/file_analysis/analyzer/x509/functions.bif
View file

@ -104,6 +104,39 @@ STACK_OF(X509)* x509_get_untrusted_stack(VectorVal* certs_vec)
return untrusted_certs;
}
// We need this function to be able to identify the signer certificate of an
// OCSP request out of a list of possible certificates.
X509* x509_get_ocsp_signer(STACK_OF(X509) *certs, OCSP_RESPID *rid)
{
// We support two lookup types - either by response id or by key.
if ( rid->type == V_OCSP_RESPID_NAME )
return X509_find_by_subject(certs, rid->value.byName);
// There only should be name and type - but let's be sure...
if ( rid->type != V_OCSP_RESPID_KEY )
return 0;
// Just like OpenSSL, we just support SHA-1 lookups and bail out otherwhise.
if ( rid->value.byKey->length != SHA_DIGEST_LENGTH )
return 0;
unsigned char* key_hash = rid->value.byKey->data;
for ( int i = 0; i < sk_X509_num(certs); ++i )
{
unsigned char digest[SHA_DIGEST_LENGTH];
X509* cert = sk_X509_value(certs, i);
if ( ! X509_pubkey_digest(cert, EVP_sha1(), digest, NULL) )
// digest failed for this certificate, try with next
continue;
if ( memcmp(digest, key_hash, SHA_DIGEST_LENGTH) == 0 )
// keys match, return certificate
return cert;
}
return 0;
}
%%}
## Parses a certificate into an X509::Certificate structure.
@ -221,6 +254,7 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
int out = -1;
int result = -1;
X509* issuer_certificate = 0;
X509* signer = 0;
OCSP_RESPONSE *resp = d2i_OCSP_RESPONSE(NULL, &start, ocsp_reply->Len());
if ( ! resp )
{
@ -250,19 +284,47 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
// inject the certificates in the certificate list of the OCSP reply, they actually are used during
// the lookup.
// Yay.
if ( ! basic->certs )
{
basic->certs = sk_X509_new_null();
if ( ! basic->certs )
{
rval = x509_result_record(-1, "Could not allocate basic x509 stack");
goto x509_ocsp_cleanup;
}
}
issuer_certificate = 0;
for ( int i = 0; i < sk_X509_num(untrusted_certs); i++)
{
sk_X509_push(basic->certs, X509_dup(sk_X509_value(untrusted_certs, i)));
if ( X509_NAME_cmp(X509_get_issuer_name(cert), X509_get_subject_name(sk_X509_value(untrusted_certs, i))) )
if ( X509_NAME_cmp(X509_get_issuer_name(cert), X509_get_subject_name(sk_X509_value(untrusted_certs, i))) == 0 )
issuer_certificate = sk_X509_value(untrusted_certs, i);
}
// Because we actually want to be able to give nice error messages that show why we were
// not able to verify the OCSP response - do our own verification logic first.
signer = x509_get_ocsp_signer(basic->certs, basic->tbsResponseData->responderId);
/*
Do this perhaps - OpenSSL also cannot do it, so I do not really feel bad about it.
Needs a different lookup because the root store is no stack of X509 certs
if ( !s igner )
// if we did not find it in the certificates that were sent, search in the root store
signer = x509_get_ocsp_signer(basic->certs, basic->tbsResponseData->responderId);
*/
if ( ! signer )
{
rval = x509_result_record(-1, "Could not find OCSP responder certificate");
goto x509_ocsp_cleanup;
}
csc = X509_STORE_CTX_new();
X509_STORE_CTX_init(csc, ctx, sk_X509_value(basic->certs, 0), basic->certs);
X509_STORE_CTX_init(csc, ctx, signer, basic->certs);
X509_STORE_CTX_set_time(csc, 0, (time_t) verify_time);
X509_STORE_CTX_set_purpose(csc, X509_PURPOSE_OCSP_HELPER);
@ -281,7 +343,6 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
goto x509_ocsp_cleanup;
}
// ok, now we verified the OCSP response. This means that we have a valid chain tying it
// to a root that we trust and that the signature also hopefully is valid. This does not yet
// mean that the ocsp response actually matches the certificate the server send us or that
@ -322,7 +383,7 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
goto x509_ocsp_cleanup;
}
if ( ! OCSP_id_cmp(certid, single->certId) )
if ( OCSP_id_cmp(certid, single->certId) != 0 )
return x509_result_record(-1, "OCSP reply is not for host certificate");
// next - check freshness of proof...

17
src/file_analysis/file_analysis.bif
View file

@ -47,6 +47,16 @@ function Files::__add_analyzer%(file_id: string, tag: Files::Tag, args: any%): b
return new Val(result, TYPE_BOOL);
%}
## :bro:see:`Files::add_analyzers_for_mime_type`.
function Files::__add_analyzers_for_mime_type%(file_id: string, mtype: string, args: any%): files_tag_set
%{
using BifType::Record::Files::AnalyzerArgs;
RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
Val* analyzers = file_mgr->AddAnalyzersForMIMEType(file_id->CheckString(), mtype->CheckString(), rv);
Unref(rv);
return analyzers;
%}
## :bro:see:`Files::remove_analyzer`.
function Files::__remove_analyzer%(file_id: string, tag: Files::Tag, args: any%): bool
%{
@ -71,6 +81,13 @@ function Files::__analyzer_name%(tag: Files::Tag%) : string
return new StringVal(file_mgr->GetComponentName(tag));
%}
## :bro:see:`Files::register_for_mime_type`.
function Files::__register_for_mime_type%(id: Analyzer::Tag, mt: string%) : bool
%{
bool result = file_mgr->RegisterAnalyzerForMIMEType(id->AsEnumVal(), mt);
return new Val(result, TYPE_BOOL);
%}
module GLOBAL;
## For use within a :bro:see:`get_file_handle` handler to set a unique