Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-03 07:08:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Reorganizing file analysis source code.

Browse source
This commit is contained in:
Jon Siwek 2013-02-14 16:07:42 -06:00
parent f04d189d3f
commit f8af42cf9a
14 changed files with 675 additions and 603 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/CMakeLists.txt
View file

@ -329,7 +329,6 @@ set(bro_SRCS
FTP.cc
File.cc
FileAnalyzer.cc
FileAnalysisManager.cc
Finger.cc
FlowSrc.cc
Frag.cc
@ -449,6 +448,12 @@ set(bro_SRCS
input/readers/Raw.cc
input/readers/Benchmark.cc
file_analysis/Manager.cc
file_analysis/Info.cc
file_analysis/InfoTimer.cc
file_analysis/Action.h
file_analysis/Extract.cc
nb_dns.c
digest.h
)

351
src/FileAnalysisManager.h
View file

@ -1,351 +0,0 @@
#ifndef FILE_ANALYSIS_MANAGER_H
#define FILE_ANALYSIS_MANAGER_H
#include <string>
#include <map>
#include <vector>
#include "Conn.h"
#include "Analyzer.h"
#include "Timer.h"
#include "Val.h"
#include "Reporter.h"
namespace file_analysis {
class Info;
/**
* Base class for actions that can be attached to a file_analysis::Info object.
*/
class Action {
public:
virtual ~Action() {}
/**
* Subclasses may override this to receive file data non-sequentially.
*/
virtual void DeliverChunk(const u_char* data, uint64 len, uint64 offset) {}
/**
* Subclasses may override this to receive file sequentially.
*/
virtual void DeliverStream(const u_char* data, uint64 len) {}
/**
* Subclasses may override this to specifically handle the end of a file.
*/
virtual void EndOfFile() {}
/**
* Subclasses may override this to handle missing data in a file stream.
*/
virtual void Undelivered(uint64 offset, uint64 len) {}
protected:
Action(Info* arg_info);
Info* info;
};
typedef Action* (*ActionInstantiator)(const RecordVal* args, Info* info);
/**
* An action to simply extract files to disk.
*/
class Extract : Action {
public:
static Action* Instantiate(const RecordVal* args, Info* info);
~Extract();
virtual void DeliverChunk(const u_char* data, uint64 len, uint64 offset);
protected:
Extract(Info* arg_info, const string& arg_filename);
string filename;
int fd;
};
/**
* Wrapper class around \c FileAnalysis::Info record values from script layer.
*/
class Info {
public:
~Info();
/**
* @return value (seconds) of the "timeout_interval" field from #val record.
*/
double TimeoutInterval() const;
/**
* @return value of the "file_id" field from #val record.
*/
string FileID() const;
/**
* @return #last_activity_time
*/
double LastActivityTime() const { return last_activity_time; }
/**
* Refreshes #last_activity_time with current network time.
*/
void UpdateLastActivityTime() { last_activity_time = network_time; }
/**
* Set "total_bytes" field of #val record to \a size.
*/
void SetTotalBytes(uint64 size);
/**
* Compares "seen_bytes" field to "total_bytes" field of #val record
* and returns true if the comparison indicates the full file was seen.
* If "total_bytes" hasn't been set yet, it returns false.
*/
bool IsComplete() const;
/**
* Create a timer to be dispatched after the amount of time indicated by
* the "timeout_interval" field of the #val record in order to check if
* #last_activity_time is old enough to timeout analysis of the file.
*/
void ScheduleInactivityTimer() const;
/**
* Attaches an action. Only one action per type can be attached at a time.
* @return true if the action was attached, else false.
*/
bool AddAction(EnumVal* act, RecordVal* args);
/**
* Removes an action.
* @return true if the action was removed, else false.
*/
bool RemoveAction(EnumVal* act);
/**
* Pass in non-sequential data and deliver to attached actions/analyzers.
*/
void DataIn(const u_char* data, uint64 len, uint64 offset);
/**
* Pass in sequential data and deliver to attached actions/analyzers.
*/
void DataIn(const u_char* data, uint64 len);
/**
* Inform attached actions/analyzers about end of file being seen.
*/
void EndOfFile();
/**
* Inform attached actions/analyzers about a gap in file stream.
*/
void Gap(uint64 offset, uint64 len);
protected:
friend class Manager;
/**
* Constructor; only file_analysis::Manager should be creating these.
*/
Info(const string& file_id, Connection* conn = 0,
const string& protocol = "");
/**
* Updates the "conn_ids" and "conn_uids" fields in #val record with the
* \c conn_id and UID taken from \a conn.
*/
void UpdateConnectionFields(Connection* conn);
/**
* Increment a byte count field of #val record by \a size.
*/
void IncrementByteCount(uint64 size, int field_idx);
/**
* Wrapper to RecordVal::LookupWithDefault for the field in #val at index
* \a idx which automatically unrefs the Val and returns a converted value.
*/
uint64 LookupFieldDefaultCount(int idx) const;
/**
* Wrapper to RecordVal::LookupWithDefault for the field in #val at index
* \a idx which automatically unrefs the Val and returns a converted value.
*/
double LookupFieldDefaultInterval(int idx) const;
RecordVal* val; /**< \c FileAnalysis::Info from script layer. */
double last_activity_time; /**< Time of last activity. */
bool postpone_timeout; /**< Whether postponing timeout is requested. */
bool need_reassembly; /**< Whether file stream reassembly is needed. */
typedef map<int, Action*> ActionMap;
ActionMap actions;
/**
* @return the field offset in #val record corresponding to \a field_name.
*/
static int Idx(const string& field_name);
/**
* Initializes the index offsets for fields in \c FileAnalysis::info record.
*/
static void InitFieldIndices();
static int file_id_idx;
static int parent_file_id_idx;
static int protocol_idx;
static int conn_uids_idx;
static int conn_ids_idx;
static int seen_bytes_idx;
static int total_bytes_idx;
static int missing_bytes_idx;
static int overflow_bytes_idx;
static int timeout_interval_idx;
static int actions_idx;
static int action_args_idx;
};
/**
* Timer to periodically check if file analysis for a given file is inactive.
*/
class InfoTimer : public Timer {
public:
InfoTimer(double t, const string& id, double interval)
: Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id) { }
~InfoTimer() { }
/**
* Check inactivity of file_analysis::Info corresponding to #file_id,
* reschedule if active, else call file_analysis::Manager::Timeout.
*/
void Dispatch(double t, int is_expire);
protected:
string file_id;
};
/**
* Main entry point for interacting with file analysis.
*/
class Manager {
public:
Manager();
~Manager();
/**
* Times out any active file analysis to prepare for shutdown.
*/
void Terminate();
/**
* Pass in non-sequential file data.
*/
void DataIn(const string& file_id, const u_char* data, uint64 len,
uint64 offset, Connection* conn = 0,
const string& protocol = "");
/**
* Pass in sequential file data.
*/
void DataIn(const string& file_id, const u_char* data, uint64 len,
Connection* conn = 0, const string& protocol = "");
/**
* Signal the end of file data.
*/
void EndOfFile(const string& file_id, Connection* conn = 0,
const string& protocol = "");
/**
* Signal a gap in the file data stream.
*/
void Gap(const string& file_id, uint64 offset, uint64 len,
Connection* conn = 0, const string& protocol = "");
/**
* Provide the expected number of bytes that comprise a file.
*/
void SetSize(const string& file_id, uint64 size, Connection* conn = 0,
const string& protocol = "");
/**
* Discard the file_analysis::Info object associated with \a file_id.
* @return false if file identifier did not map to anything, else true.
*/
bool RemoveFile(const string& file_id);
/**
* If called during \c FileAnalysis::policy evaluation for a
* \c FileAnalysis::TRIGGER_TIMEOUT, requests deferral of analysis timeout.
*/
bool PostponeTimeout(const string& file_id) const;
/**
* Attaches an action to the file identifier. Only one action of a given
* type can be attached per file identifier at a time.
* @return true if the action was attached, else false.
*/
bool AddAction(const string& file_id, EnumVal* act, RecordVal* args) const;
/**
* Removes an action for a given file identifier.
* @return true if the action was removed, else false.
*/
bool RemoveAction(const string& file_id, EnumVal* act) const;
/**
* Calls the \c FileAnalysis::policy hook.
*/
static void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info);
protected:
friend class InfoTimer;
typedef map<string, Info*> FileMap;
/**
* @return the Info object mapped to \a file_id. One is created if mapping
* doesn't exist. If it did exist, the activity time is refreshed
* and connection-related fields of the record value may be updated.
*/
Info* IDtoInfo(const string& file_id, Connection* conn = 0,
const string& protocol = "");
/**
* @return the Info object mapped to \a file_id, or a null pointer if no
* mapping exists.
*/
Info* Lookup(const string& file_id) const;
/**
* Evaluate timeout policy for a file and remove the Info object mapped to
* \a file_id if needed.
*/
void Timeout(const string& file_id, bool is_terminating = ::terminating);
FileMap file_map; /**< Map strings to \c FileAnalysis::Info records. */
};
} // namespace file_analysis
extern file_analysis::Manager* file_mgr;
#endif

2
src/FileAnalyzer.cc
View file

@ -1,6 +1,6 @@
#include <algorithm>
#include "FileAnalysisManager.h"
#include "file_analysis/Manager.h"
#include "FileAnalyzer.h"
#include "Reporter.h"

2
src/file_analysis.bif
View file

@ -3,7 +3,7 @@
module FileAnalysis;
%%{
#include "FileAnalysisManager.h"
#include "file_analysis/Manager.h"
%%}
type Info: record;

49
src/file_analysis/Action.h Normal file
View file

@ -0,0 +1,49 @@
#ifndef FILE_ANALYSIS_ACTION_H
#define FILE_ANALYSIS_ACTION_H
#include "Val.h"
namespace file_analysis {
class Info;
/**
* Base class for actions that can be attached to a file_analysis::Info object.
*/
class Action {
public:
virtual ~Action() {}
/**
* Subclasses may override this to receive file data non-sequentially.
*/
virtual void DeliverChunk(const u_char* data, uint64 len, uint64 offset) {}
/**
* Subclasses may override this to receive file sequentially.
*/
virtual void DeliverStream(const u_char* data, uint64 len) {}
/**
* Subclasses may override this to specifically handle the end of a file.
*/
virtual void EndOfFile() {}
/**
* Subclasses may override this to handle missing data in a file stream.
*/
virtual void Undelivered(uint64 offset, uint64 len) {}
protected:
Action(Info* arg_info) {}
Info* info;
};
typedef Action* (*ActionInstantiator)(const RecordVal* args, Info* info);
} // namespace file_analysis
#endif

46
src/file_analysis/Extract.cc Normal file
View file

@ -0,0 +1,46 @@
#include <string>
#include "Extract.h"
#include "util.h"
using namespace file_analysis;
Extract::Extract(Info* arg_info, const string& arg_filename)
: Action(arg_info), filename(arg_filename)
{
fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
if ( fd < 0 )
{
fd = 0;
char buf[128];
strerror_r(errno, buf, sizeof(buf));
reporter->Error("cannot open %s: %s", filename.c_str(), buf);
}
}
Extract::~Extract()
{
if ( fd )
safe_close(fd);
}
Action* Extract::Instantiate(const RecordVal* args, Info* info)
{
const char* field = "extract_filename";
int off = BifType::Record::FileAnalysis::ActionArgs->FieldOffset(field);
Val* v = args->Lookup(off);
if ( ! v ) return 0;
return new Extract(info, v->AsString()->CheckString());
}
void Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
{
Action::DeliverChunk(data, len, offset);
if ( ! fd ) return;
safe_pwrite(fd, data, len, offset);
}

33
src/file_analysis/Extract.h Normal file
View file

@ -0,0 +1,33 @@
#ifndef FILE_ANALYSIS_EXTRACT_H
#define FILE_ANALYSIS_EXTRACT_H
#include <string>
#include "Val.h"
#include "Info.h"
namespace file_analysis {
/**
* An action to simply extract files to disk.
*/
class Extract : Action {
public:
static Action* Instantiate(const RecordVal* args, Info* info);
~Extract();
virtual void DeliverChunk(const u_char* data, uint64 len, uint64 offset);
protected:
Extract(Info* arg_info, const string& arg_filename);
string filename;
int fd;
};
} // namespace file_analysis
#endif

255
src/FileAnalysisManager.cc → src/file_analysis/Info.cc
View file

@ -1,7 +1,12 @@
#include <vector>
#include <string>
#include "FileAnalysisManager.h"
#include "util.h"
#include "Info.h"
#include "InfoTimer.h"
#include "Reporter.h"
#include "Val.h"
#include "Action.h"
#include "Extract.h"
using namespace file_analysis;
@ -10,50 +15,6 @@ static ActionInstantiator action_factory[] = {
Extract::Instantiate,
};
Action::Action(Info* arg_info) : info(arg_info)
{
}
Extract::Extract(Info* arg_info, const string& arg_filename)
: Action(arg_info), filename(arg_filename)
{
fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
if ( fd < 0 )
{
fd = 0;
char buf[128];
strerror_r(errno, buf, sizeof(buf));
reporter->Error("cannot open %s: %s", filename.c_str(), buf);
}
}
Extract::~Extract()
{
if ( fd )
safe_close(fd);
}
Action* Extract::Instantiate(const RecordVal* args, Info* info)
{
const char* field = "extract_filename";
int off = BifType::Record::FileAnalysis::ActionArgs->FieldOffset(field);
Val* v = args->Lookup(off);
if ( ! v ) return 0;
return new Extract(info, v->AsString()->CheckString());
}
void Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
{
Action::DeliverChunk(data, len, offset);
if ( ! fd ) return;
safe_pwrite(fd, data, len, offset);
}
static TableVal* empty_conn_id_set()
{
TypeList* set_index = new TypeList(conn_id);
@ -295,203 +256,3 @@ void Info::Gap(uint64 offset, uint64 len)
IncrementByteCount(len, missing_bytes_idx);
}
void InfoTimer::Dispatch(double t, int is_expire)
{
Info* info = file_mgr->Lookup(file_id);
if ( ! info ) return;
double last_active = info->LastActivityTime();
double inactive_time = t > last_active ? t - last_active : 0.0;
DBG_LOG(DBG_FILE_ANALYSIS, "Checking inactivity for %s, last active at %f, "
"inactive for %f", file_id.c_str(), last_active, inactive_time);
if ( last_active == 0.0 )
{
// was created when network_time was zero, so re-schedule w/ valid time
info->UpdateLastActivityTime();
info->ScheduleInactivityTimer();
return;
}
if ( inactive_time >= info->TimeoutInterval() )
file_mgr->Timeout(file_id);
else if ( ! is_expire )
info->ScheduleInactivityTimer();
}
Manager::Manager()
{
}
Manager::~Manager()
{
Terminate();
}
void Manager::Terminate()
{
vector<string> keys;
for ( FileMap::iterator it = file_map.begin(); it != file_map.end(); ++it )
keys.push_back(it->first);
for ( size_t i = 0; i < keys.size(); ++i )
Timeout(keys[i], true);
}
static void check_file_done(Info* info)
{
if ( info->IsComplete() )
{
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, info);
file_mgr->RemoveFile(info->FileID());
}
}
void Manager::DataIn(const string& file_id, const u_char* data, uint64 len,
uint64 offset, Connection* conn, const string& protocol)
{
Info* info = IDtoInfo(file_id, conn, protocol);
info->DataIn(data, len, offset);
check_file_done(info);
}
void Manager::DataIn(const string& file_id, const u_char* data, uint64 len,
Connection* conn, const string& protocol)
{
Info* info = IDtoInfo(file_id, conn, protocol);
info->DataIn(data, len);
check_file_done(info);
}
void Manager::EndOfFile(const string& file_id, Connection* conn,
const string& protocol)
{
Info* info = IDtoInfo(file_id, conn, protocol);
info->EndOfFile();
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_EOF, info);
}
void Manager::Gap(const string& file_id, uint64 offset, uint64 len,
Connection* conn, const string& protocol)
{
Info* info = IDtoInfo(file_id, conn, protocol);
info->Gap(offset, len);
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_GAP, info);
}
void Manager::SetSize(const string& file_id, uint64 size,
Connection* conn, const string& protocol)
{
Info* info = IDtoInfo(file_id, conn, protocol);
info->SetTotalBytes(size);
check_file_done(info);
}
void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info)
{
const ID* id = global_scope()->Lookup("FileAnalysis::policy");
assert(id);
const Func* hook = id->ID_Val()->AsFunc();
val_list vl(2);
vl.append(new EnumVal(t, BifType::Enum::FileAnalysis::Trigger));
vl.append(info->val->Ref());
info->postpone_timeout = false;
Val* result = hook->Call(&vl);
Unref(result);
}
bool Manager::PostponeTimeout(const string& file_id) const
{
Info* info = Lookup(file_id);
if ( ! info ) return false;
info->postpone_timeout = true;
return true;
}
bool Manager::AddAction(const string& file_id, EnumVal* act,
RecordVal* args) const
{
Info* info = Lookup(file_id);
if ( ! info ) return false;
return info->AddAction(act, args);
}
bool Manager::RemoveAction(const string& file_id, EnumVal* act) const
{
Info* info = Lookup(file_id);
if ( ! info ) return false;
return info->RemoveAction(act);
}
Info* Manager::IDtoInfo(const string& file_id, Connection* conn,
const string& protocol)
{
Info* rval = file_map[file_id];
if ( ! rval )
{
rval = file_map[file_id] = new Info(file_id, conn, protocol);
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, rval);
}
else
{
rval->UpdateLastActivityTime();
rval->UpdateConnectionFields(conn);
}
return rval;
}
Info* Manager::Lookup(const string& file_id) const
{
FileMap::const_iterator it = file_map.find(file_id);
if ( it == file_map.end() ) return 0;
return it->second;
}
void Manager::Timeout(const string& file_id, bool is_terminating)
{
Info* info = Lookup(file_id);
if ( ! info ) return;
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info);
if ( info->postpone_timeout && ! is_terminating )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Postpone file analysis timeout for %s",
info->FileID().c_str());
info->UpdateLastActivityTime();
info->ScheduleInactivityTimer();
return;
}
DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s",
info->FileID().c_str());
RemoveFile(file_id);
}
bool Manager::RemoveFile(const string& file_id)
{
FileMap::iterator it = file_map.find(file_id);
if ( it == file_map.end() ) return false;
delete it->second;
file_map.erase(it);
return true;
}

160
src/file_analysis/Info.h Normal file
View file

@ -0,0 +1,160 @@
#ifndef FILE_ANALYSIS_INFO_H
#define FILE_ANALYSIS_INFO_H
#include <string>
#include <map>
#include "Conn.h"
#include "Val.h"
#include "Action.h"
namespace file_analysis {
/**
* Wrapper class around \c FileAnalysis::Info record values from script layer.
*/
class Info {
public:
~Info();
/**
* @return value (seconds) of the "timeout_interval" field from #val record.
*/
double TimeoutInterval() const;
/**
* @return value of the "file_id" field from #val record.
*/
string FileID() const;
/**
* @return #last_activity_time
*/
double LastActivityTime() const { return last_activity_time; }
/**
* Refreshes #last_activity_time with current network time.
*/
void UpdateLastActivityTime() { last_activity_time = network_time; }
/**
* Set "total_bytes" field of #val record to \a size.
*/
void SetTotalBytes(uint64 size);
/**
* Compares "seen_bytes" field to "total_bytes" field of #val record
* and returns true if the comparison indicates the full file was seen.
* If "total_bytes" hasn't been set yet, it returns false.
*/
bool IsComplete() const;
/**
* Create a timer to be dispatched after the amount of time indicated by
* the "timeout_interval" field of the #val record in order to check if
* #last_activity_time is old enough to timeout analysis of the file.
*/
void ScheduleInactivityTimer() const;
/**
* Attaches an action. Only one action per type can be attached at a time.
* @return true if the action was attached, else false.
*/
bool AddAction(EnumVal* act, RecordVal* args);
/**
* Removes an action.
* @return true if the action was removed, else false.
*/
bool RemoveAction(EnumVal* act);
/**
* Pass in non-sequential data and deliver to attached actions/analyzers.
*/
void DataIn(const u_char* data, uint64 len, uint64 offset);
/**
* Pass in sequential data and deliver to attached actions/analyzers.
*/
void DataIn(const u_char* data, uint64 len);
/**
* Inform attached actions/analyzers about end of file being seen.
*/
void EndOfFile();
/**
* Inform attached actions/analyzers about a gap in file stream.
*/
void Gap(uint64 offset, uint64 len);
protected:
friend class Manager;
/**
* Constructor; only file_analysis::Manager should be creating these.
*/
Info(const string& file_id, Connection* conn = 0,
const string& protocol = "");
/**
* Updates the "conn_ids" and "conn_uids" fields in #val record with the
* \c conn_id and UID taken from \a conn.
*/
void UpdateConnectionFields(Connection* conn);
/**
* Increment a byte count field of #val record by \a size.
*/
void IncrementByteCount(uint64 size, int field_idx);
/**
* Wrapper to RecordVal::LookupWithDefault for the field in #val at index
* \a idx which automatically unrefs the Val and returns a converted value.
*/
uint64 LookupFieldDefaultCount(int idx) const;
/**
* Wrapper to RecordVal::LookupWithDefault for the field in #val at index
* \a idx which automatically unrefs the Val and returns a converted value.
*/
double LookupFieldDefaultInterval(int idx) const;
RecordVal* val; /**< \c FileAnalysis::Info from script layer. */
double last_activity_time; /**< Time of last activity. */
bool postpone_timeout; /**< Whether postponing timeout is requested. */
bool need_reassembly; /**< Whether file stream reassembly is needed. */
typedef map<int, Action*> ActionMap;
ActionMap actions;
/**
* @return the field offset in #val record corresponding to \a field_name.
*/
static int Idx(const string& field_name);
/**
* Initializes the index offsets for fields in \c FileAnalysis::info record.
*/
static void InitFieldIndices();
static int file_id_idx;
static int parent_file_id_idx;
static int protocol_idx;
static int conn_uids_idx;
static int conn_ids_idx;
static int seen_bytes_idx;
static int total_bytes_idx;
static int missing_bytes_idx;
static int overflow_bytes_idx;
static int timeout_interval_idx;
static int actions_idx;
static int action_args_idx;
};
} // namespace file_analysis
#endif

30
src/file_analysis/InfoTimer.cc Normal file
View file

@ -0,0 +1,30 @@
#include "Manager.h"
#include "Info.h"
using namespace file_analysis;
void InfoTimer::Dispatch(double t, int is_expire)
{
Info* info = file_mgr->Lookup(file_id);
if ( ! info ) return;
double last_active = info->LastActivityTime();
double inactive_time = t > last_active ? t - last_active : 0.0;
DBG_LOG(DBG_FILE_ANALYSIS, "Checking inactivity for %s, last active at %f, "
"inactive for %f", file_id.c_str(), last_active, inactive_time);
if ( last_active == 0.0 )
{
// was created when network_time was zero, so re-schedule w/ valid time
info->UpdateLastActivityTime();
info->ScheduleInactivityTimer();
return;
}
if ( inactive_time >= info->TimeoutInterval() )
file_mgr->Timeout(file_id);
else if ( ! is_expire )
info->ScheduleInactivityTimer();
}

33
src/file_analysis/InfoTimer.h Normal file
View file

@ -0,0 +1,33 @@
#ifndef FILE_ANALYSIS_INFOTIMER_H
#define FILE_ANALYSIS_INFOTIMER_H
#include "Timer.h"
#include <string>
namespace file_analysis {
/**
* Timer to periodically check if file analysis for a given file is inactive.
*/
class InfoTimer : public Timer {
public:
InfoTimer(double t, const string& id, double interval)
: Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id) {}
~InfoTimer() {}
/**
* Check inactivity of file_analysis::Info corresponding to #file_id,
* reschedule if active, else call file_analysis::Manager::Timeout.
*/
void Dispatch(double t, int is_expire);
protected:
string file_id;
};
} // namespace file_analysis
#endif

181
src/file_analysis/Manager.cc Normal file
View file

@ -0,0 +1,181 @@
#include <vector>
#include <string>
#include "Manager.h"
#include "Info.h"
using namespace file_analysis;
Manager::Manager()
{
}
Manager::~Manager()
{
Terminate();
}
void Manager::Terminate()
{
vector<string> keys;
for ( FileMap::iterator it = file_map.begin(); it != file_map.end(); ++it )
keys.push_back(it->first);
for ( size_t i = 0; i < keys.size(); ++i )
Timeout(keys[i], true);
}
static void check_file_done(Info* info)
{
if ( info->IsComplete() )
{
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, info);
file_mgr->RemoveFile(info->FileID());
}
}
void Manager::DataIn(const string& file_id, const u_char* data, uint64 len,
uint64 offset, Connection* conn, const string& protocol)
{
Info* info = IDtoInfo(file_id, conn, protocol);
info->DataIn(data, len, offset);
check_file_done(info);
}
void Manager::DataIn(const string& file_id, const u_char* data, uint64 len,
Connection* conn, const string& protocol)
{
Info* info = IDtoInfo(file_id, conn, protocol);
info->DataIn(data, len);
check_file_done(info);
}
void Manager::EndOfFile(const string& file_id, Connection* conn,
const string& protocol)
{
Info* info = IDtoInfo(file_id, conn, protocol);
info->EndOfFile();
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_EOF, info);
}
void Manager::Gap(const string& file_id, uint64 offset, uint64 len,
Connection* conn, const string& protocol)
{
Info* info = IDtoInfo(file_id, conn, protocol);
info->Gap(offset, len);
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_GAP, info);
}
void Manager::SetSize(const string& file_id, uint64 size,
Connection* conn, const string& protocol)
{
Info* info = IDtoInfo(file_id, conn, protocol);
info->SetTotalBytes(size);
check_file_done(info);
}
void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info)
{
const ID* id = global_scope()->Lookup("FileAnalysis::policy");
assert(id);
const Func* hook = id->ID_Val()->AsFunc();
val_list vl(2);
vl.append(new EnumVal(t, BifType::Enum::FileAnalysis::Trigger));
vl.append(info->val->Ref());
info->postpone_timeout = false;
Val* result = hook->Call(&vl);
Unref(result);
}
bool Manager::PostponeTimeout(const string& file_id) const
{
Info* info = Lookup(file_id);
if ( ! info ) return false;
info->postpone_timeout = true;
return true;
}
bool Manager::AddAction(const string& file_id, EnumVal* act,
RecordVal* args) const
{
Info* info = Lookup(file_id);
if ( ! info ) return false;
return info->AddAction(act, args);
}
bool Manager::RemoveAction(const string& file_id, EnumVal* act) const
{
Info* info = Lookup(file_id);
if ( ! info ) return false;
return info->RemoveAction(act);
}
Info* Manager::IDtoInfo(const string& file_id, Connection* conn,
const string& protocol)
{
Info* rval = file_map[file_id];
if ( ! rval )
{
rval = file_map[file_id] = new Info(file_id, conn, protocol);
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, rval);
}
else
{
rval->UpdateLastActivityTime();
rval->UpdateConnectionFields(conn);
}
return rval;
}
Info* Manager::Lookup(const string& file_id) const
{
FileMap::const_iterator it = file_map.find(file_id);
if ( it == file_map.end() ) return 0;
return it->second;
}
void Manager::Timeout(const string& file_id, bool is_terminating)
{
Info* info = Lookup(file_id);
if ( ! info ) return;
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info);
if ( info->postpone_timeout && ! is_terminating )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Postpone file analysis timeout for %s",
info->FileID().c_str());
info->UpdateLastActivityTime();
info->ScheduleInactivityTimer();
return;
}
DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s",
info->FileID().c_str());
RemoveFile(file_id);
}
bool Manager::RemoveFile(const string& file_id)
{
FileMap::iterator it = file_map.find(file_id);
if ( it == file_map.end() ) return false;
delete it->second;
file_map.erase(it);
return true;
}

125
src/file_analysis/Manager.h Normal file
View file

@ -0,0 +1,125 @@
#ifndef FILE_ANALYSIS_MANAGER_H
#define FILE_ANALYSIS_MANAGER_H
#include <string>
#include <map>
#include "Net.h"
#include "Conn.h"
#include "Val.h"
#include "Info.h"
#include "InfoTimer.h"
namespace file_analysis {
/**
* Main entry point for interacting with file analysis.
*/
class Manager {
public:
Manager();
~Manager();
/**
* Times out any active file analysis to prepare for shutdown.
*/
void Terminate();
/**
* Pass in non-sequential file data.
*/
void DataIn(const string& file_id, const u_char* data, uint64 len,
uint64 offset, Connection* conn = 0,
const string& protocol = "");
/**
* Pass in sequential file data.
*/
void DataIn(const string& file_id, const u_char* data, uint64 len,
Connection* conn = 0, const string& protocol = "");
/**
* Signal the end of file data.
*/
void EndOfFile(const string& file_id, Connection* conn = 0,
const string& protocol = "");
/**
* Signal a gap in the file data stream.
*/
void Gap(const string& file_id, uint64 offset, uint64 len,
Connection* conn = 0, const string& protocol = "");
/**
* Provide the expected number of bytes that comprise a file.
*/
void SetSize(const string& file_id, uint64 size, Connection* conn = 0,
const string& protocol = "");
/**
* Discard the file_analysis::Info object associated with \a file_id.
* @return false if file identifier did not map to anything, else true.
*/
bool RemoveFile(const string& file_id);
/**
* If called during \c FileAnalysis::policy evaluation for a
* \c FileAnalysis::TRIGGER_TIMEOUT, requests deferral of analysis timeout.
*/
bool PostponeTimeout(const string& file_id) const;
/**
* Attaches an action to the file identifier. Only one action of a given
* type can be attached per file identifier at a time.
* @return true if the action was attached, else false.
*/
bool AddAction(const string& file_id, EnumVal* act, RecordVal* args) const;
/**
* Removes an action for a given file identifier.
* @return true if the action was removed, else false.
*/
bool RemoveAction(const string& file_id, EnumVal* act) const;
/**
* Calls the \c FileAnalysis::policy hook.
*/
static void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info);
protected:
friend class InfoTimer;
typedef map<string, Info*> FileMap;
/**
* @return the Info object mapped to \a file_id. One is created if mapping
* doesn't exist. If it did exist, the activity time is refreshed
* and connection-related fields of the record value may be updated.
*/
Info* IDtoInfo(const string& file_id, Connection* conn = 0,
const string& protocol = "");
/**
* @return the Info object mapped to \a file_id, or a null pointer if no
* mapping exists.
*/
Info* Lookup(const string& file_id) const;
/**
* Evaluate timeout policy for a file and remove the Info object mapped to
* \a file_id if needed.
*/
void Timeout(const string& file_id, bool is_terminating = ::terminating);
FileMap file_map; /**< Map strings to \c FileAnalysis::Info records. */
};
} // namespace file_analysis
extern file_analysis::Manager* file_mgr;
#endif

2
src/main.cc
View file

@ -58,7 +58,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
#include "logging/Manager.h"
#include "logging/writers/Ascii.h"
#include "FileAnalysisManager.h"
#include "file_analysis/Manager.h"
#include "binpac_bro.h"