Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

FileAnalysis: change terminology s/action/analyzer

Browse source
This commit is contained in:
Jon Siwek 2013-04-11 14:53:54 -05:00
parent e81f2ae7b0
commit b8c98b8bf7
30 changed files with 575 additions and 570 deletions
Download patch file Download diff file Expand all files Collapse all files

49
scripts/base/frameworks/file-analysis/main.bro
View file

@ -12,13 +12,13 @@ export {
LOG
};
## A structure which represents a desired file analysis action to take.
type ActionArgs: record {
## The type of action.
act: Action;
## A structure which represents a desired type of file analysis.
type AnalyzerArgs: record {
## The type of analysis.
tag: Analyzer;
## The local filename to which to write an extracted file. Must be
## set when *act* is :bro:see:`FileAnalysis::ACTION_EXTRACT`.
## set when *tag* is :bro:see:`FileAnalysis::ANALYZER_EXTRACT`.
extract_filename: string &optional;
## An event which will be generated for all new file contents,
@ -60,8 +60,7 @@ export {
missing_bytes: count &log &default=0;
## The number of not all-in-sequence bytes in the file stream that
## were delivered to file actions/analyzers due to reassembly buffer
## overflow.
## were delivered to file analyzers due to reassembly buffer overflow.
overflow_bytes: count &log &default=0;
## The amount of time between receiving new data for this file that
@ -83,10 +82,10 @@ export {
## Connection UIDS over which the file was transferred.
conn_uids: set[string] &log;
## A set of action types taken during the file analysis.
actions_taken: set[Action] &log;
## A set of analysis types done during the file analysis.
analyzers: set[Analyzer] &log;
## Local filenames of file extraction actions.
## Local filenames of extracted files.
extracted_files: set[string] &log;
## An MD5 digest of the file contents.
@ -139,26 +138,26 @@ export {
## for the *id* isn't currently active.
global postpone_timeout: function(f: fa_file): bool;
## Adds an action to the analysis of a given file.
## Adds an analyzer to the analysis of a given file.
##
## f: the file.
##
## args: the action type to add along with any arguments it takes.
## args: the analyzer type to add along with any arguments it takes.
##
## Returns: true if the action will be added, or false if analysis
## Returns: true if the analyzer will be added, or false if analysis
## for the *id* isn't currently active or the *args*
## were invalid for the action type.
global add_action: function(f: fa_file, args: ActionArgs): bool;
## were invalid for the analyzer type.
global add_analyzer: function(f: fa_file, args: AnalyzerArgs): bool;
## Removes an action from the analysis of a given file.
## Removes an analyzer from the analysis of a given file.
##
## f: the file.
##
## args: the action (type and args) to remove.
## args: the analyzer (type and args) to remove.
##
## Returns: true if the action will be removed, or false if analysis
## Returns: true if the analyzer will be removed, or false if analysis
## for the *id* isn't currently active.
global remove_action: function(f: fa_file, args: ActionArgs): bool;
global remove_analyzer: function(f: fa_file, args: AnalyzerArgs): bool;
## Stops/ignores any further analysis of a given file.
##
@ -260,22 +259,22 @@ function postpone_timeout(f: fa_file): bool
return __postpone_timeout(f$id);
}
function add_action(f: fa_file, args: ActionArgs): bool
function add_analyzer(f: fa_file, args: AnalyzerArgs): bool
{
if ( ! __add_action(f$id, args) ) return F;
if ( ! __add_analyzer(f$id, args) ) return F;
set_info(f);
add f$info$actions_taken[args$act];
add f$info$analyzers[args$tag];
if ( args$act == FileAnalysis::ACTION_EXTRACT )
if ( args$tag == FileAnalysis::ANALYZER_EXTRACT )
add f$info$extracted_files[args$extract_filename];
return T;
}
function remove_action(f: fa_file, args: ActionArgs): bool
function remove_analyzer(f: fa_file, args: AnalyzerArgs): bool
{
return __remove_action(f$id, args);
return __remove_analyzer(f$id, args);
}
function stop(f: fa_file): bool

3
scripts/base/init-bare.bro
View file

@ -358,8 +358,7 @@ type fa_file: record {
missing_bytes: count &default=0;
## The number of not all-in-sequence bytes in the file stream that
## were delivered to file actions/analyzers due to reassembly buffer
## overflow.
## were delivered to file analyzers due to reassembly buffer overflow.
overflow_bytes: count &default=0;
## The amount of time between receiving new data for this file that

8
scripts/base/protocols/ftp/file-extract.bro
View file

@ -38,8 +38,8 @@ event file_new(f: fa_file) &priority=5
if ( f?$mime_type && extract_file_types in f$mime_type )
{
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=get_extraction_name(f)]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
$extract_filename=get_extraction_name(f)]);
return;
}
@ -55,8 +55,8 @@ event file_new(f: fa_file) &priority=5
if ( ! s$extract_file ) next;
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=get_extraction_name(f)]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
$extract_filename=get_extraction_name(f)]);
return;
}
}

8
scripts/base/protocols/http/file-extract.bro
View file

@ -44,8 +44,8 @@ event file_new(f: fa_file) &priority=5
if ( f?$mime_type && extract_file_types in f$mime_type )
{
fname = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
$extract_filename=fname]);
for ( cid in f$conns )
{
@ -68,8 +68,8 @@ event file_new(f: fa_file) &priority=5
if ( ! c$http$extract_file ) next;
fname = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
$extract_filename=fname]);
extracting = T;
break;
}

4
scripts/base/protocols/http/file-hash.bro
View file

@ -30,7 +30,7 @@ event file_new(f: fa_file) &priority=5
if ( f?$mime_type && generate_md5 in f$mime_type )
{
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
return;
}
@ -44,7 +44,7 @@ event file_new(f: fa_file) &priority=5
if ( ! c$http$calc_md5 ) next;
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
return;
}
}

8
scripts/base/protocols/irc/dcc-send.bro
View file

@ -101,8 +101,8 @@ event file_new(f: fa_file) &priority=5
if ( f?$mime_type && extract_file_types in f$mime_type )
{
fname = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
$extract_filename=fname]);
set_dcc_extraction_file(f, fname);
return;
}
@ -120,8 +120,8 @@ event file_new(f: fa_file) &priority=5
if ( ! s$extract_file ) next;
fname = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
$extract_filename=fname]);
s$extraction_file = fname;
return;
}

15
scripts/base/protocols/smtp/entities.bro
View file

@ -123,8 +123,9 @@ event file_new(f: fa_file) &priority=5
if ( ! extracting )
{
fname = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
FileAnalysis::add_analyzer(f,
[$tag=FileAnalysis::ANALYZER_EXTRACT,
$extract_filename=fname]);
extracting = T;
++extract_count;
}
@ -133,7 +134,7 @@ event file_new(f: fa_file) &priority=5
}
if ( c$smtp$current_entity$calc_md5 )
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
}
}
@ -141,12 +142,12 @@ function check_extract_by_type(f: fa_file)
{
if ( extract_file_types !in f$mime_type ) return;
if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
if ( f?$info && FileAnalysis::ANALYZER_EXTRACT in f$info$analyzers )
return;
local fname: string = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
$extract_filename=fname]);
if ( ! f?$conns ) return;
@ -163,7 +164,7 @@ function check_md5_by_type(f: fa_file)
if ( never_calc_md5 ) return;
if ( generate_md5 !in f$mime_type ) return;
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
}
event file_new(f: fa_file) &priority=5

4
scripts/policy/frameworks/intel/smtp-url-extraction.bro
View file

@ -26,6 +26,6 @@ event file_new(f: fa_file) &priority=5
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_DATA_EVENT,
$stream_event=intel_mime_data]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_DATA_EVENT,
$stream_event=intel_mime_data]);
}

4
src/CMakeLists.txt
View file

@ -453,8 +453,8 @@ set(bro_SRCS
file_analysis/File.cc
file_analysis/FileTimer.cc
file_analysis/FileID.h
file_analysis/Action.h
file_analysis/ActionSet.cc
file_analysis/Analyzer.h
file_analysis/AnalyzerSet.cc
file_analysis/Extract.cc
file_analysis/Hash.cc
file_analysis/DataEvent.cc

20
src/file_analysis.bif
View file

@ -23,24 +23,24 @@ function FileAnalysis::__set_timeout_interval%(file_id: string, t: interval%): b
return new Val(result, TYPE_BOOL);
%}
## :bro:see:`FileAnalysis::add_action`.
function FileAnalysis::__add_action%(file_id: string, args: any%): bool
## :bro:see:`FileAnalysis::add_analyzer`.
function FileAnalysis::__add_analyzer%(file_id: string, args: any%): bool
%{
using file_analysis::FileID;
using BifType::Record::FileAnalysis::ActionArgs;
RecordVal* rv = args->AsRecordVal()->CoerceTo(ActionArgs);
bool result = file_mgr->AddAction(FileID(file_id->CheckString()), rv);
using BifType::Record::FileAnalysis::AnalyzerArgs;
RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
bool result = file_mgr->AddAnalyzer(FileID(file_id->CheckString()), rv);
Unref(rv);
return new Val(result, TYPE_BOOL);
%}
## :bro:see:`FileAnalysis::remove_action`.
function FileAnalysis::__remove_action%(file_id: string, args: any%): bool
## :bro:see:`FileAnalysis::remove_analyzer`.
function FileAnalysis::__remove_analyzer%(file_id: string, args: any%): bool
%{
using file_analysis::FileID;
using BifType::Record::FileAnalysis::ActionArgs;
RecordVal* rv = args->AsRecordVal()->CoerceTo(ActionArgs);
bool result = file_mgr->RemoveAction(FileID(file_id->CheckString()), rv);
using BifType::Record::FileAnalysis::AnalyzerArgs;
RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
bool result = file_mgr->RemoveAnalyzer(FileID(file_id->CheckString()), rv);
Unref(rv);
return new Val(result, TYPE_BOOL);
%}

101
src/file_analysis/Action.h
View file

@ -1,101 +0,0 @@
#ifndef FILE_ANALYSIS_ACTION_H
#define FILE_ANALYSIS_ACTION_H
#include "Val.h"
#include "NetVar.h"
namespace file_analysis {
typedef BifEnum::FileAnalysis::Action ActionTag;
class File;
/**
* Base class for actions that can be attached to a file_analysis::File object.
*/
class Action {
public:
virtual ~Action()
{
DBG_LOG(DBG_FILE_ANALYSIS, "Destroy action %d", tag);
Unref(args);
}
/**
* Subclasses may override this to receive file data non-sequentially.
* @return true if the action is still in a valid state to continue
* receiving data/events or false if it's essentially "done".
*/
virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset)
{ return true; }
/**
* Subclasses may override this to receive file sequentially.
* @return true if the action is still in a valid state to continue
* receiving data/events or false if it's essentially "done".
*/
virtual bool DeliverStream(const u_char* data, uint64 len)
{ return true; }
/**
* Subclasses may override this to specifically handle an EOF signal,
* which means no more data is going to be incoming and the action/analyzer
* may be deleted/cleaned up soon.
* @return true if the action is still in a valid state to continue
* receiving data/events or false if it's essentially "done".
*/
virtual bool EndOfFile()
{ return true; }
/**
* Subclasses may override this to handle missing data in a file stream.
* @return true if the action is still in a valid state to continue
* receiving data/events or false if it's essentially "done".
*/
virtual bool Undelivered(uint64 offset, uint64 len)
{ return true; }
/**
* @return the action type enum value.
*/
ActionTag Tag() const { return tag; }
/**
* @return the ActionArgs associated with the aciton.
*/
RecordVal* Args() const { return args; }
/**
* @return the file_analysis::File object to which the action is attached.
*/
File* GetFile() const { return file; }
/**
* @return the action tag equivalent of the 'act' field from the ActionArgs
* value \a args.
*/
static ActionTag ArgsTag(const RecordVal* args)
{
using BifType::Record::FileAnalysis::ActionArgs;
return static_cast<ActionTag>(
args->Lookup(ActionArgs->FieldOffset("act"))->AsEnum());
}
protected:
Action(RecordVal* arg_args, File* arg_file)
: tag(Action::ArgsTag(arg_args)), args(arg_args->Ref()->AsRecordVal()),
file(arg_file)
{}
ActionTag tag;
RecordVal* args;
File* file;
};
typedef Action* (*ActionInstantiator)(RecordVal* args, File* file);
} // namespace file_analysis
#endif

185
src/file_analysis/ActionSet.cc
View file

@ -1,185 +0,0 @@
#include "ActionSet.h"
#include "File.h"
#include "Action.h"
#include "Extract.h"
#include "DataEvent.h"
#include "Hash.h"
using namespace file_analysis;
// keep in order w/ declared enum values in file_analysis.bif
static ActionInstantiator action_factory[] = {
file_analysis::Extract::Instantiate,
file_analysis::MD5::Instantiate,
file_analysis::SHA1::Instantiate,
file_analysis::SHA256::Instantiate,
file_analysis::DataEvent::Instantiate,
};
static void action_del_func(void* v)
{
delete (Action*) v;
}
ActionSet::ActionSet(File* arg_file) : file(arg_file)
{
TypeList* t = new TypeList();
t->Append(BifType::Record::FileAnalysis::ActionArgs->Ref());
action_hash = new CompositeHash(t);
Unref(t);
action_map.SetDeleteFunc(action_del_func);
}
ActionSet::~ActionSet()
{
while ( ! mod_queue.empty() )
{
Modification* mod = mod_queue.front();
mod->Abort();
delete mod;
mod_queue.pop();
}
delete action_hash;
}
bool ActionSet::AddAction(RecordVal* args)
{
HashKey* key = GetKey(args);
if ( action_map.Lookup(key) )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate action %d skipped for file id"
" %s: already exists", Action::ArgsTag(args),
file->GetID().c_str());
delete key;
return true;
}
Action* act = InstantiateAction(args);
if ( ! act )
{
delete key;
return false;
}
InsertAction(act, key);
return true;
}
bool ActionSet::QueueAddAction(RecordVal* args)
{
HashKey* key = GetKey(args);
Action* act = InstantiateAction(args);
if ( ! act )
{
delete key;
return false;
}
mod_queue.push(new Add(act, key));
return true;
}
bool ActionSet::Add::Perform(ActionSet* set)
{
if ( set->action_map.Lookup(key) )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d skipped for file id"
" %s: already exists", act->Tag(),
act->GetFile()->GetID().c_str());
Abort();
return true;
}
set->InsertAction(act, key);
return true;
}
bool ActionSet::RemoveAction(const RecordVal* args)
{
return RemoveAction(Action::ArgsTag(args), GetKey(args));
}
bool ActionSet::RemoveAction(ActionTag tag, HashKey* key)
{
Action* act = (Action*) action_map.Remove(key);
delete key;
if ( ! act )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Skip remove action %d for file id %s",
tag, file->GetID().c_str());
return false;
}
DBG_LOG(DBG_FILE_ANALYSIS, "Remove action %d for file id %s", act->Tag(),
file->GetID().c_str());
delete act;
return true;
}
bool ActionSet::QueueRemoveAction(const RecordVal* args)
{
HashKey* key = GetKey(args);
ActionTag tag = Action::ArgsTag(args);
mod_queue.push(new Remove(tag, key));
return action_map.Lookup(key);
}
bool ActionSet::Remove::Perform(ActionSet* set)
{
return set->RemoveAction(tag, key);
}
HashKey* ActionSet::GetKey(const RecordVal* args) const
{
HashKey* key = action_hash->ComputeHash(args, 1);
if ( ! key )
reporter->InternalError("ActionArgs type mismatch");
return key;
}
Action* ActionSet::InstantiateAction(RecordVal* args) const
{
Action* act = action_factory[Action::ArgsTag(args)](args, file);
if ( ! act )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate action %d failed for file id",
" %s", Action::ArgsTag(args), file->GetID().c_str());
return 0;
}
return act;
}
void ActionSet::InsertAction(Action* act, HashKey* key)
{
DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d for file id %s", act->Tag(),
file->GetID().c_str());
action_map.Insert(key, act);
delete key;
}
void ActionSet::DrainModifications()
{
if ( mod_queue.empty() ) return;
DBG_LOG(DBG_FILE_ANALYSIS, "Start flushing action mod queue of file id %s",
file->GetID().c_str());
do
{
Modification* mod = mod_queue.front();
mod->Perform(this);
delete mod;
mod_queue.pop();
} while ( ! mod_queue.empty() );
DBG_LOG(DBG_FILE_ANALYSIS, "End flushing action mod queue of file id %s",
file->GetID().c_str());
}

109
src/file_analysis/ActionSet.h
View file

@ -1,109 +0,0 @@
#ifndef FILE_ANALYSIS_ACTIONSET_H
#define FILE_ANALYSIS_ACTIONSET_H
#include <queue>
#include "Action.h"
#include "Dict.h"
#include "CompHash.h"
#include "Val.h"
namespace file_analysis {
class File;
declare(PDict,Action);
/**
* A set of file analysis actions indexed by ActionArgs. Allows queueing
* of addition/removals so that those modifications can happen at well-defined
* times (e.g. to make sure a loop iterator isn't invalidated).
*/
class ActionSet {
public:
ActionSet(File* arg_file);
~ActionSet();
/**
* @return true if action was instantiated/attached, else false.
*/
bool AddAction(RecordVal* args);
/**
* @return true if action was able to be instantiated, else false.
*/
bool QueueAddAction(RecordVal* args);
/**
* @return false if action didn't exist and so wasn't removed, else true.
*/
bool RemoveAction(const RecordVal* args);
/**
* @return true if action exists at time of call, else false;
*/
bool QueueRemoveAction(const RecordVal* args);
/**
* Perform all queued modifications to the currently active actions.
*/
void DrainModifications();
IterCookie* InitForIteration() const
{ return action_map.InitForIteration(); }
Action* NextEntry(IterCookie* c)
{ return action_map.NextEntry(c); }
protected:
HashKey* GetKey(const RecordVal* args) const;
Action* InstantiateAction(RecordVal* args) const;
void InsertAction(Action* act, HashKey* key);
bool RemoveAction(ActionTag tag, HashKey* key);
File* file;
CompositeHash* action_hash; /**< ActionArgs hashes Action map lookup. */
PDict(Action) action_map; /**< Actions indexed by ActionArgs. */
class Modification {
public:
virtual ~Modification() {}
virtual bool Perform(ActionSet* set) = 0;
virtual void Abort() = 0;
};
class Add : public Modification {
public:
Add(Action* arg_act, HashKey* arg_key)
: Modification(), act(arg_act), key(arg_key) {}
virtual ~Add() {}
virtual bool Perform(ActionSet* set);
virtual void Abort() { delete act; delete key; }
protected:
Action* act;
HashKey* key;
};
class Remove : public Modification {
public:
Remove(ActionTag arg_tag, HashKey* arg_key)
: Modification(), tag(arg_tag), key(arg_key) {}
virtual ~Remove() {}
virtual bool Perform(ActionSet* set);
virtual void Abort() { delete key; }
protected:
ActionTag tag;
HashKey* key;
};
typedef queue<Modification*> ModQueue;
ModQueue mod_queue;
};
} // namespace file_analysiss
#endif

103
src/file_analysis/Analyzer.h Normal file
View file

@ -0,0 +1,103 @@
#ifndef FILE_ANALYSIS_ANALYZER_H
#define FILE_ANALYSIS_ANALYZER_H
#include "Val.h"
#include "NetVar.h"
namespace file_analysis {
typedef BifEnum::FileAnalysis::Analyzer FA_Tag;
class File;
/**
* Base class for analyzers that can be attached to file_analysis::File objects.
*/
class Analyzer {
public:
virtual ~Analyzer()
{
DBG_LOG(DBG_FILE_ANALYSIS, "Destroy file analyzer %d", tag);
Unref(args);
}
/**
* Subclasses may override this to receive file data non-sequentially.
* @return true if the analyzer is still in a valid state to continue
* receiving data/events or false if it's essentially "done".
*/
virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset)
{ return true; }
/**
* Subclasses may override this to receive file sequentially.
* @return true if the analyzer is still in a valid state to continue
* receiving data/events or false if it's essentially "done".
*/
virtual bool DeliverStream(const u_char* data, uint64 len)
{ return true; }
/**
* Subclasses may override this to specifically handle an EOF signal,
* which means no more data is going to be incoming and the analyzer
* may be deleted/cleaned up soon.
* @return true if the analyzer is still in a valid state to continue
* receiving data/events or false if it's essentially "done".
*/
virtual bool EndOfFile()
{ return true; }
/**
* Subclasses may override this to handle missing data in a file stream.
* @return true if the analyzer is still in a valid state to continue
* receiving data/events or false if it's essentially "done".
*/
virtual bool Undelivered(uint64 offset, uint64 len)
{ return true; }
/**
* @return the analyzer type enum value.
*/
FA_Tag Tag() const { return tag; }
/**
* @return the AnalyzerArgs associated with the analyzer.
*/
RecordVal* Args() const { return args; }
/**
* @return the file_analysis::File object to which the analyzer is attached.
*/
File* GetFile() const { return file; }
/**
* @return the analyzer tag equivalent of the 'tag' field from the
* AnalyzerArgs value \a args.
*/
static FA_Tag ArgsTag(const RecordVal* args)
{
using BifType::Record::FileAnalysis::AnalyzerArgs;
return static_cast<FA_Tag>(
args->Lookup(AnalyzerArgs->FieldOffset("tag"))->AsEnum());
}
protected:
Analyzer(RecordVal* arg_args, File* arg_file)
: tag(file_analysis::Analyzer::ArgsTag(arg_args)),
args(arg_args->Ref()->AsRecordVal()),
file(arg_file)
{}
FA_Tag tag;
RecordVal* args;
File* file;
};
typedef file_analysis::Analyzer* (*AnalyzerInstantiator)(RecordVal* args,
File* file);
} // namespace file_analysis
#endif

188
src/file_analysis/AnalyzerSet.cc Normal file
View file

@ -0,0 +1,188 @@
#include "AnalyzerSet.h"
#include "File.h"
#include "Analyzer.h"
#include "Extract.h"
#include "DataEvent.h"
#include "Hash.h"
using namespace file_analysis;
// keep in order w/ declared enum values in file_analysis.bif
static AnalyzerInstantiator analyzer_factory[] = {
file_analysis::Extract::Instantiate,
file_analysis::MD5::Instantiate,
file_analysis::SHA1::Instantiate,
file_analysis::SHA256::Instantiate,
file_analysis::DataEvent::Instantiate,
};
static void analyzer_del_func(void* v)
{
delete (file_analysis::Analyzer*) v;
}
AnalyzerSet::AnalyzerSet(File* arg_file) : file(arg_file)
{
TypeList* t = new TypeList();
t->Append(BifType::Record::FileAnalysis::AnalyzerArgs->Ref());
analyzer_hash = new CompositeHash(t);
Unref(t);
analyzer_map.SetDeleteFunc(analyzer_del_func);
}
AnalyzerSet::~AnalyzerSet()
{
while ( ! mod_queue.empty() )
{
Modification* mod = mod_queue.front();
mod->Abort();
delete mod;
mod_queue.pop();
}
delete analyzer_hash;
}
bool AnalyzerSet::Add(RecordVal* args)
{
HashKey* key = GetKey(args);
if ( analyzer_map.Lookup(key) )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %d skipped for file id"
" %s: already exists", file_analysis::Analyzer::ArgsTag(args),
file->GetID().c_str());
delete key;
return true;
}
file_analysis::Analyzer* a = InstantiateAnalyzer(args);
if ( ! a )
{
delete key;
return false;
}
Insert(a, key);
return true;
}
bool AnalyzerSet::QueueAdd(RecordVal* args)
{
HashKey* key = GetKey(args);
file_analysis::Analyzer* a = InstantiateAnalyzer(args);
if ( ! a )
{
delete key;
return false;
}
mod_queue.push(new AddMod(a, key));
return true;
}
bool AnalyzerSet::AddMod::Perform(AnalyzerSet* set)
{
if ( set->analyzer_map.Lookup(key) )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Add analyzer %d skipped for file id"
" %s: already exists", a->Tag(), a->GetFile()->GetID().c_str());
Abort();
return true;
}
set->Insert(a, key);
return true;
}
bool AnalyzerSet::Remove(const RecordVal* args)
{
return Remove(file_analysis::Analyzer::ArgsTag(args), GetKey(args));
}
bool AnalyzerSet::Remove(FA_Tag tag, HashKey* key)
{
file_analysis::Analyzer* a =
(file_analysis::Analyzer*) analyzer_map.Remove(key);
delete key;
if ( ! a )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Skip remove analyzer %d for file id %s",
tag, file->GetID().c_str());
return false;
}
DBG_LOG(DBG_FILE_ANALYSIS, "Remove analyzer %d for file id %s", a->Tag(),
file->GetID().c_str());
delete a;
return true;
}
bool AnalyzerSet::QueueRemove(const RecordVal* args)
{
HashKey* key = GetKey(args);
FA_Tag tag = file_analysis::Analyzer::ArgsTag(args);
mod_queue.push(new RemoveMod(tag, key));
return analyzer_map.Lookup(key);
}
bool AnalyzerSet::RemoveMod::Perform(AnalyzerSet* set)
{
return set->Remove(tag, key);
}
HashKey* AnalyzerSet::GetKey(const RecordVal* args) const
{
HashKey* key = analyzer_hash->ComputeHash(args, 1);
if ( ! key )
reporter->InternalError("AnalyzerArgs type mismatch");
return key;
}
file_analysis::Analyzer* AnalyzerSet::InstantiateAnalyzer(RecordVal* args) const
{
file_analysis::Analyzer* a =
analyzer_factory[file_analysis::Analyzer::ArgsTag(args)](args, file);
if ( ! a )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %d failed for file id",
" %s", file_analysis::Analyzer::ArgsTag(args),
file->GetID().c_str());
return 0;
}
return a;
}
void AnalyzerSet::Insert(file_analysis::Analyzer* a, HashKey* key)
{
DBG_LOG(DBG_FILE_ANALYSIS, "Add analyzer %d for file id %s", a->Tag(),
file->GetID().c_str());
analyzer_map.Insert(key, a);
delete key;
}
void AnalyzerSet::DrainModifications()
{
if ( mod_queue.empty() ) return;
DBG_LOG(DBG_FILE_ANALYSIS, "Start analyzer mod queue flush of file id %s",
file->GetID().c_str());
do
{
Modification* mod = mod_queue.front();
mod->Perform(this);
delete mod;
mod_queue.pop();
} while ( ! mod_queue.empty() );
DBG_LOG(DBG_FILE_ANALYSIS, "End flushing analyzer mod queue of file id %s",
file->GetID().c_str());
}

109
src/file_analysis/AnalyzerSet.h Normal file
View file

@ -0,0 +1,109 @@
#ifndef FILE_ANALYSIS_ANALYZERSET_H
#define FILE_ANALYSIS_ANALYZERSET_H
#include <queue>
#include "Analyzer.h"
#include "Dict.h"
#include "CompHash.h"
#include "Val.h"
namespace file_analysis {
class File;
declare(PDict,Analyzer);
/**
* A set of file analysis analyzers indexed by AnalyzerArgs. Allows queueing
* of addition/removals so that those modifications can happen at well-defined
* times (e.g. to make sure a loop iterator isn't invalidated).
*/
class AnalyzerSet {
public:
AnalyzerSet(File* arg_file);
~AnalyzerSet();
/**
* @return true if analyzer was instantiated/attached, else false.
*/
bool Add(RecordVal* args);
/**
* @return true if analyzer was able to be instantiated, else false.
*/
bool QueueAdd(RecordVal* args);
/**
* @return false if analyzer didn't exist and so wasn't removed, else true.
*/
bool Remove(const RecordVal* args);
/**
* @return true if analyzer exists at time of call, else false;
*/
bool QueueRemove(const RecordVal* args);
/**
* Perform all queued modifications to the currently active analyzers.
*/
void DrainModifications();
IterCookie* InitForIteration() const
{ return analyzer_map.InitForIteration(); }
file_analysis::Analyzer* NextEntry(IterCookie* c)
{ return analyzer_map.NextEntry(c); }
protected:
HashKey* GetKey(const RecordVal* args) const;
file_analysis::Analyzer* InstantiateAnalyzer(RecordVal* args) const;
void Insert(file_analysis::Analyzer* a, HashKey* key);
bool Remove(FA_Tag tag, HashKey* key);
File* file;
CompositeHash* analyzer_hash; /**< AnalyzerArgs hashes. */
PDict(file_analysis::Analyzer) analyzer_map; /**< Indexed by AnalyzerArgs. */
class Modification {
public:
virtual ~Modification() {}
virtual bool Perform(AnalyzerSet* set) = 0;
virtual void Abort() = 0;
};
class AddMod : public Modification {
public:
AddMod(file_analysis::Analyzer* arg_a, HashKey* arg_key)
: Modification(), a(arg_a), key(arg_key) {}
virtual ~AddMod() {}
virtual bool Perform(AnalyzerSet* set);
virtual void Abort() { delete a; delete key; }
protected:
file_analysis::Analyzer* a;
HashKey* key;
};
class RemoveMod : public Modification {
public:
RemoveMod(FA_Tag arg_tag, HashKey* arg_key)
: Modification(), tag(arg_tag), key(arg_key) {}
virtual ~RemoveMod() {}
virtual bool Perform(AnalyzerSet* set);
virtual void Abort() { delete key; }
protected:
FA_Tag tag;
HashKey* key;
};
typedef queue<Modification*> ModQueue;
ModQueue mod_queue;
};
} // namespace file_analysiss
#endif

10
src/file_analysis/DataEvent.cc
View file

@ -9,18 +9,18 @@ using namespace file_analysis;
DataEvent::DataEvent(RecordVal* args, File* file,
EventHandlerPtr ce, EventHandlerPtr se)
: Action(args, file), chunk_event(ce), stream_event(se)
: file_analysis::Analyzer(args, file), chunk_event(ce), stream_event(se)
{
}
Action* DataEvent::Instantiate(RecordVal* args, File* file)
file_analysis::Analyzer* DataEvent::Instantiate(RecordVal* args, File* file)
{
using BifType::Record::FileAnalysis::ActionArgs;
using BifType::Record::FileAnalysis::AnalyzerArgs;
const char* chunk_field = "chunk_event";
const char* stream_field = "stream_event";
int chunk_off = ActionArgs->FieldOffset(chunk_field);
int stream_off = ActionArgs->FieldOffset(stream_field);
int chunk_off = AnalyzerArgs->FieldOffset(chunk_field);
int stream_off = AnalyzerArgs->FieldOffset(stream_field);
Val* chunk_val = args->Lookup(chunk_off);
Val* stream_val = args->Lookup(stream_off);

8
src/file_analysis/DataEvent.h
View file

@ -5,17 +5,17 @@
#include "Val.h"
#include "File.h"
#include "Action.h"
#include "Analyzer.h"
namespace file_analysis {
/**
* An action to send file data to script-layer events.
* An analyzer to send file data to script-layer events.
*/
class DataEvent : public Action {
class DataEvent : public file_analysis::Analyzer {
public:
static Action* Instantiate(RecordVal* args, File* file);
static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);

8
src/file_analysis/Extract.cc
View file

@ -6,7 +6,7 @@
using namespace file_analysis;
Extract::Extract(RecordVal* args, File* file, const string& arg_filename)
: Action(args, file), filename(arg_filename)
: file_analysis::Analyzer(args, file), filename(arg_filename)
{
fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
@ -25,11 +25,11 @@ Extract::~Extract()
safe_close(fd);
}
Action* Extract::Instantiate(RecordVal* args, File* file)
file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file)
{
using BifType::Record::FileAnalysis::ActionArgs;
using BifType::Record::FileAnalysis::AnalyzerArgs;
const char* field = "extract_filename";
Val* v = args->Lookup(ActionArgs->FieldOffset(field));
Val* v = args->Lookup(AnalyzerArgs->FieldOffset(field));
if ( ! v ) return 0;

8
src/file_analysis/Extract.h
View file

@ -5,17 +5,17 @@
#include "Val.h"
#include "File.h"
#include "Action.h"
#include "Analyzer.h"
namespace file_analysis {
/**
* An action to simply extract files to disk.
* An analyzer to extract files to disk.
*/
class Extract : public Action {
class Extract : public file_analysis::Analyzer {
public:
static Action* Instantiate(RecordVal* args, File* file);
static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
virtual ~Extract();

77
src/file_analysis/File.cc
View file

@ -4,11 +4,12 @@
#include "File.h"
#include "FileTimer.h"
#include "FileID.h"
#include "Analyzer.h"
#include "Manager.h"
#include "Reporter.h"
#include "Val.h"
#include "Type.h"
#include "Analyzer.h"
#include "../Analyzer.h"
#include "Event.h"
using namespace file_analysis;
@ -77,7 +78,7 @@ void File::StaticInit()
File::File(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
: id(""), unique(unique), val(0), postpone_timeout(false),
first_chunk(true), missed_bof(false), need_reassembly(false), done(false),
actions(this)
analyzers(this)
{
StaticInit();
@ -98,7 +99,7 @@ File::File(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
if ( conn )
{
// add source and connection fields
val->Assign(source_idx, new StringVal(Analyzer::GetTagName(tag)));
val->Assign(source_idx, new StringVal(::Analyzer::GetTagName(tag)));
UpdateConnectionFields(conn);
}
else
@ -215,14 +216,14 @@ void File::ScheduleInactivityTimer() const
timer_mgr->Add(new FileTimer(network_time, id, GetTimeoutInterval()));
}
bool File::AddAction(RecordVal* args)
bool File::AddAnalyzer(RecordVal* args)
{
return done ? false : actions.QueueAddAction(args);
return done ? false : analyzers.QueueAdd(args);
}
bool File::RemoveAction(const RecordVal* args)
bool File::RemoveAnalyzer(const RecordVal* args)
{
return done ? false : actions.QueueRemoveAction(args);
return done ? false : analyzers.QueueRemove(args);
}
bool File::BufferBOF(const u_char* data, uint64 len)
@ -286,7 +287,7 @@ void File::ReplayBOF()
void File::DataIn(const u_char* data, uint64 len, uint64 offset)
{
actions.DrainModifications();
analyzers.DrainModifications();
if ( first_chunk )
{
@ -296,16 +297,16 @@ void File::DataIn(const u_char* data, uint64 len, uint64 offset)
first_chunk = false;
}
Action* act = 0;
IterCookie* c = actions.InitForIteration();
file_analysis::Analyzer* a = 0;
IterCookie* c = analyzers.InitForIteration();
while ( (act = actions.NextEntry(c)) )
while ( (a = analyzers.NextEntry(c)) )
{
if ( ! act->DeliverChunk(data, len, offset) )
actions.QueueRemoveAction(act->Args());
if ( ! a->DeliverChunk(data, len, offset) )
analyzers.QueueRemove(a->Args());
}
actions.DrainModifications();
analyzers.DrainModifications();
// TODO: check reassembly requirement based on buffer size in record
if ( need_reassembly )
@ -320,7 +321,7 @@ void File::DataIn(const u_char* data, uint64 len, uint64 offset)
void File::DataIn(const u_char* data, uint64 len)
{
actions.DrainModifications();
analyzers.DrainModifications();
if ( BufferBOF(data, len) ) return;
@ -331,25 +332,25 @@ void File::DataIn(const u_char* data, uint64 len)
missed_bof = false;
}
Action* act = 0;
IterCookie* c = actions.InitForIteration();
file_analysis::Analyzer* a = 0;
IterCookie* c = analyzers.InitForIteration();
while ( (act = actions.NextEntry(c)) )
while ( (a = analyzers.NextEntry(c)) )
{
if ( ! act->DeliverStream(data, len) )
if ( ! a->DeliverStream(data, len) )
{
actions.QueueRemoveAction(act->Args());
analyzers.QueueRemove(a->Args());
continue;
}
uint64 offset = LookupFieldDefaultCount(seen_bytes_idx) +
LookupFieldDefaultCount(missing_bytes_idx);
if ( ! act->DeliverChunk(data, len, offset) )
actions.QueueRemoveAction(act->Args());
if ( ! a->DeliverChunk(data, len, offset) )
analyzers.QueueRemove(a->Args());
}
actions.DrainModifications();
analyzers.DrainModifications();
IncrementByteCount(len, seen_bytes_idx);
}
@ -357,42 +358,42 @@ void File::EndOfFile()
{
if ( done ) return;
actions.DrainModifications();
analyzers.DrainModifications();
// Send along anything that's been buffered, but never flushed.
ReplayBOF();
done = true;
Action* act = 0;
IterCookie* c = actions.InitForIteration();
file_analysis::Analyzer* a = 0;
IterCookie* c = analyzers.InitForIteration();
while ( (act = actions.NextEntry(c)) )
while ( (a = analyzers.NextEntry(c)) )
{
if ( ! act->EndOfFile() )
actions.QueueRemoveAction(act->Args());
if ( ! a->EndOfFile() )
analyzers.QueueRemove(a->Args());
}
FileEvent(file_state_remove);
actions.DrainModifications();
analyzers.DrainModifications();
}
void File::Gap(uint64 offset, uint64 len)
{
actions.DrainModifications();
analyzers.DrainModifications();
// If we were buffering the beginning of the file, a gap means we've got
// as much contiguous stuff at the beginning as possible, so work with that.
ReplayBOF();
Action* act = 0;
IterCookie* c = actions.InitForIteration();
file_analysis::Analyzer* a = 0;
IterCookie* c = analyzers.InitForIteration();
while ( (act = actions.NextEntry(c)) )
while ( (a = analyzers.NextEntry(c)) )
{
if ( ! act->Undelivered(offset, len) )
actions.QueueRemoveAction(act->Args());
if ( ! a->Undelivered(offset, len) )
analyzers.QueueRemove(a->Args());
}
if ( FileEventAvailable(file_gap) )
@ -404,7 +405,7 @@ void File::Gap(uint64 offset, uint64 len)
FileEvent(file_gap, vl);
}
actions.DrainModifications();
analyzers.DrainModifications();
IncrementByteCount(len, missing_bytes_idx);
}
@ -430,6 +431,6 @@ void File::FileEvent(EventHandlerPtr h, val_list* vl)
{
// immediate feedback is required for these events.
mgr.Drain();
actions.DrainModifications();
analyzers.DrainModifications();
}
}

26
src/file_analysis/File.h
View file

@ -8,7 +8,7 @@
#include "AnalyzerTags.h"
#include "Conn.h"
#include "Val.h"
#include "ActionSet.h"
#include "AnalyzerSet.h"
#include "FileID.h"
#include "BroString.h"
@ -79,35 +79,35 @@ public:
void ScheduleInactivityTimer() const;
/**
* Queues attaching an action. Only one action per type can be attached at
* a time unless the arguments differ.
* @return false if action can't be instantiated, else true.
* Queues attaching an analyzer. Only one analyzer per type can be attached
* at a time unless the arguments differ.
* @return false if analyzer can't be instantiated, else true.
*/
bool AddAction(RecordVal* args);
bool AddAnalyzer(RecordVal* args);
/**
* Queues removal of an action.
* @return true if action was active at time of call, else false.
* Queues removal of an analyzer.
* @return true if analyzer was active at time of call, else false.
*/
bool RemoveAction(const RecordVal* args);
bool RemoveAnalyzer(const RecordVal* args);
/**
* Pass in non-sequential data and deliver to attached actions/analyzers.
* Pass in non-sequential data and deliver to attached analyzers.
*/
void DataIn(const u_char* data, uint64 len, uint64 offset);
/**
* Pass in sequential data and deliver to attached actions/analyzers.
* Pass in sequential data and deliver to attached analyzers.
*/
void DataIn(const u_char* data, uint64 len);
/**
* Inform attached actions/analyzers about end of file being seen.
* Inform attached analyzers about end of file being seen.
*/
void EndOfFile();
/**
* Inform attached actions/analyzers about a gap in file stream.
* Inform attached analyzers about a gap in file stream.
*/
void Gap(uint64 offset, uint64 len);
@ -184,7 +184,7 @@ protected:
bool missed_bof; /**< Flags that we missed start of file. */
bool need_reassembly; /**< Whether file stream reassembly is needed. */
bool done; /**< If this object is about to be deleted. */
ActionSet actions;
AnalyzerSet analyzers;
struct BOF_Buffer {
BOF_Buffer() : full(false), replayed(false), size(0) {}

2
src/file_analysis/Hash.cc
View file

@ -7,7 +7,7 @@
using namespace file_analysis;
Hash::Hash(RecordVal* args, File* file, HashVal* hv, const char* arg_kind)
: Action(args, file), hash(hv), fed(false), kind(arg_kind)
: file_analysis::Analyzer(args, file), hash(hv), fed(false), kind(arg_kind)
{
hash->Init();
}

12
src/file_analysis/Hash.h
View file

@ -6,14 +6,14 @@
#include "Val.h"
#include "OpaqueVal.h"
#include "File.h"
#include "Action.h"
#include "Analyzer.h"
namespace file_analysis {
/**
* An action to produce a hash of file contents.
* An analyzer to produce a hash of file contents.
*/
class Hash : public Action {
class Hash : public file_analysis::Analyzer {
public:
virtual ~Hash();
@ -38,7 +38,7 @@ protected:
class MD5 : public Hash {
public:
static Action* Instantiate(RecordVal* args, File* file)
static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
{ return file_hash ? new MD5(args, file) : 0; }
protected:
@ -51,7 +51,7 @@ protected:
class SHA1 : public Hash {
public:
static Action* Instantiate(RecordVal* args, File* file)
static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
{ return file_hash ? new SHA1(args, file) : 0; }
protected:
@ -64,7 +64,7 @@ protected:
class SHA256 : public Hash {
public:
static Action* Instantiate(RecordVal* args, File* file)
static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
{ return file_hash ? new SHA256(args, file) : 0; }
protected:

10
src/file_analysis/Manager.cc
View file

@ -3,7 +3,7 @@
#include "Manager.h"
#include "File.h"
#include "Action.h"
#include "Analyzer.h"
#include "Var.h"
#include "Event.h"
@ -167,22 +167,22 @@ bool Manager::SetTimeoutInterval(const FileID& file_id, double interval) const
return true;
}
bool Manager::AddAction(const FileID& file_id, RecordVal* args) const
bool Manager::AddAnalyzer(const FileID& file_id, RecordVal* args) const
{
File* file = Lookup(file_id);
if ( ! file ) return false;
return file->AddAction(args);
return file->AddAnalyzer(args);
}
bool Manager::RemoveAction(const FileID& file_id, const RecordVal* args) const
bool Manager::RemoveAnalyzer(const FileID& file_id, const RecordVal* args) const
{
File* file = Lookup(file_id);
if ( ! file ) return false;
return file->RemoveAction(args);
return file->RemoveAnalyzer(args);
}
File* Manager::GetFile(const string& unique, Connection* conn,

16
src/file_analysis/Manager.h
View file

@ -102,18 +102,18 @@ public:
bool SetTimeoutInterval(const FileID& file_id, double interval) const;
/**
* Queue attachment of an action to the file identifier. Multiple actions
* of a given type can be attached per file identifier at a time as long as
* the arguments differ.
* @return false if the action failed to be instantiated, else true.
* Queue attachment of an analzer to the file identifier. Multiple
* analyzers of a given type can be attached per file identifier at a time
* as long as the arguments differ.
* @return false if the analyzer failed to be instantiated, else true.
*/
bool AddAction(const FileID& file_id, RecordVal* args) const;
bool AddAnalyzer(const FileID& file_id, RecordVal* args) const;
/**
* Queue removal of an action for a given file identifier.
* @return true if the action is active at the time of call, else false.
* Queue removal of an analyzer for a given file identifier.
* @return true if the analyzer is active at the time of call, else false.
*/
bool RemoveAction(const FileID& file_id, const RecordVal* args) const;
bool RemoveAnalyzer(const FileID& file_id, const RecordVal* args) const;
/**
* @return whether the file mapped to \a unique is being ignored.

14
src/types.bif
View file

@ -229,25 +229,25 @@ type gtp_gsn_addr: record;
module FileAnalysis;
type ActionArgs: record;
type AnalyzerArgs: record;
## An enumeration of various file analysis actions that can be taken.
enum Action %{
enum Analyzer %{
## Extract a file to local filesystem
ACTION_EXTRACT,
ANALYZER_EXTRACT,
## Calculate an MD5 digest of the file's contents.
ACTION_MD5,
ANALYZER_MD5,
## Calculate an SHA1 digest of the file's contents.
ACTION_SHA1,
ANALYZER_SHA1,
## Calculate an SHA256 digest of the file's contents.
ACTION_SHA256,
ANALYZER_SHA256,
## Deliver the file contents to the script-layer in an event.
ACTION_DATA_EVENT,
ANALYZER_DATA_EVENT,
%}
module GLOBAL;

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log
View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path file_analysis
#open 2013-04-11-17-29-51
#fields id parent_id source last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size mime_type timedout conn_uids actions_taken extracted_files md5 sha1 sha256
#open 2013-04-11-19-37-28
#fields id parent_id source last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size mime_type timedout conn_uids analyzers extracted_files md5 sha1 sha256
#types string string string time count count count count interval count string bool table[string] table[enum] table[string] string string string
Cx92a0ym5R8 - HTTP 1362692527.009775 4705 4705 0 0 120.000000 1024 set F UWkUyAuUGXf FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
#close 2013-04-11-17-29-51
Cx92a0ym5R8 - HTTP 1362692527.009775 4705 4705 0 0 120.000000 1024 set F UWkUyAuUGXf FileAnalysis::ANALYZER_SHA1,FileAnalysis::ANALYZER_EXTRACT,FileAnalysis::ANALYZER_DATA_EVENT,FileAnalysis::ANALYZER_MD5,FileAnalysis::ANALYZER_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
#close 2013-04-11-19-37-28

6
testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro
View file

@ -10,9 +10,9 @@ redef test_get_file_name = function(f: fa_file): string
event file_new(f: fa_file) &priority=-10
{
for ( act in test_file_actions )
FileAnalysis::remove_action(f, act);
for ( tag in test_file_analyzers )
FileAnalysis::remove_analyzer(f, tag);
local filename = test_get_file_name(f);
FileAnalysis::remove_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::remove_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
$extract_filename=filename]);
}

22
testing/scripts/file-analysis-test.bro
View file

@ -1,7 +1,7 @@
global test_file_analysis_source: string = "" &redef;
global test_file_actions: set[FileAnalysis::ActionArgs];
global test_file_analyzers: set[FileAnalysis::AnalyzerArgs];
global test_get_file_name: function(f: fa_file): string =
function(f: fa_file): string { return ""; } &redef;
@ -29,16 +29,16 @@ event file_new(f: fa_file)
if ( test_file_analysis_source == "" ||
f$source == test_file_analysis_source )
{
for ( act in test_file_actions )
FileAnalysis::add_action(f, act);
for ( tag in test_file_analyzers )
FileAnalysis::add_analyzer(f, tag);
local filename: string = test_get_file_name(f);
if ( filename != "" )
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_DATA_EVENT,
$chunk_event=file_chunk,
$stream_event=file_stream]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
$extract_filename=filename]);
FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_DATA_EVENT,
$chunk_event=file_chunk,
$stream_event=file_stream]);
}
if ( f?$bof_buffer )
@ -96,7 +96,7 @@ event file_state_remove(f: fa_file)
event bro_init()
{
add test_file_actions[[$act=FileAnalysis::ACTION_MD5]];
add test_file_actions[[$act=FileAnalysis::ACTION_SHA1]];
add test_file_actions[[$act=FileAnalysis::ACTION_SHA256]];
add test_file_analyzers[[$tag=FileAnalysis::ANALYZER_MD5]];
add test_file_analyzers[[$tag=FileAnalysis::ANALYZER_SHA1]];
add test_file_analyzers[[$tag=FileAnalysis::ANALYZER_SHA256]];
}