FileAnalysis: file handles now set from events.

Versus from synchronous function calls, which doesn't work well because
the function call can see a script-layer state that doesn't reflect
the state as it will be in terms of the event/network stream.
This commit is contained in:
Jon Siwek 2013-03-25 15:37:58 -05:00
parent 00a1de3593
commit 84a0c2fdac
21 changed files with 362 additions and 392 deletions

View file

@ -5,22 +5,14 @@
#include "Info.h"
#include "Action.h"
#include "Var.h"
#include "Event.h"
using namespace file_analysis;
void DrainTimer::Dispatch(double t, int is_expire)
{
using BifConst::FileAnalysis::pending_file_drain_interval;
DBG_LOG(DBG_FILE_ANALYSIS, "DrainTimer dispatched");
file_mgr->DrainPending();
if ( ! is_expire )
timer_mgr->Add(new DrainTimer(pending_file_drain_interval));
}
TableVal* Manager::disabled = 0;
Manager::Manager() : is_draining(false)
Manager::Manager()
{
using BifConst::FileAnalysis::pending_file_drain_interval;
timer_mgr->Add(new DrainTimer(pending_file_drain_interval));
}
Manager::~Manager()
@ -28,78 +20,8 @@ Manager::~Manager()
Terminate();
}
string Manager::GetFileHandle(Analyzer* root, Connection* conn,
bool is_orig) const
{
static TableVal* table = 0;
if ( ! table )
table = internal_val("FileAnalysis::handle_callbacks")->AsTableVal();
if ( ! root ) return "";
Val* index = new Val(root->GetTag(), TYPE_COUNT);
const Val* callback = table->Lookup(index);
Unref(index);
if ( callback )
{
val_list vl(2);
vl.append(conn->BuildConnVal());
vl.append(new Val(is_orig, TYPE_BOOL));
Val* result = callback->AsFunc()->Call(&vl);
if ( result )
{
string rval = result->AsString()->CheckString();
Unref(result);
if ( ! rval.empty() ) return rval;
}
}
for ( analyzer_list::const_iterator it = root->GetChildren().begin();
it != root->GetChildren().end(); ++it )
{
string rval = GetFileHandle((*it), conn, is_orig);
if ( ! rval.empty() ) return rval;
}
return "";
}
string Manager::GetFileHandle(Connection* conn, bool is_orig) const
{
if ( ! conn ) return "";
return GetFileHandle(conn->GetRootAnalyzer(), conn, is_orig);
}
void Manager::DrainPending()
{
if ( is_draining ) return;
is_draining = true;
PendingList::iterator it = pending.begin();
while ( it != pending.end() )
{
if ( (*it)->Retry() || (*it)->IsStale() )
{
delete *it;
pending.erase(it++);
}
else
++it;
}
is_draining = false;
}
void Manager::Terminate()
{
DrainPending();
vector<FileID> keys;
for ( IDMap::iterator it = id_map.begin(); it != id_map.end(); ++it )
keys.push_back(it->first);
@ -107,24 +29,38 @@ void Manager::Terminate()
Timeout(keys[i], true);
}
bool Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
void Manager::ReceiveHandle(const string& handle)
{
if ( pending.empty() )
reporter->InternalError("File analysis underflow");
PendingFile* pf = pending.front();
if ( ! handle.empty() )
pf->Finish(handle);
delete pf;
pending.pop();
}
void Manager::EventDrainDone()
{
if ( pending.empty() ) return;
reporter->Error("Too few return_file_handle() calls, discarding pending"
" file analysis input.");
while ( ! pending.empty() )
{
delete pending.front();
pending.pop();
}
}
void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
{
DrainPending();
string unique = GetFileHandle(conn, is_orig);
if ( ! unique.empty() )
{
DataIn(data, len, offset, GetInfo(unique, conn, tag));
return true;
}
if ( ! is_draining )
pending.push_back(new PendingDataInChunk(data, len, offset, tag, conn,
is_orig));
return false;
if ( IsDisabled(tag) ) return;
if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
pending.push(new PendingDataInChunk(data, len, offset, tag, conn));
}
void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
@ -136,8 +72,6 @@ void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
Info* info)
{
DrainPending();
if ( ! info ) return;
info->DataIn(data, len, offset);
@ -146,24 +80,12 @@ void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
RemoveFile(info->GetUnique());
}
bool Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
void Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
Connection* conn, bool is_orig)
{
DrainPending();
string unique = GetFileHandle(conn, is_orig);
if ( ! unique.empty() )
{
DataIn(data, len, GetInfo(unique, conn, tag));
return true;
}
if ( ! is_draining )
pending.push_back(new PendingDataInStream(data, len, tag, conn,
is_orig));
return false;
if ( IsDisabled(tag) ) return;
if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
pending.push(new PendingDataInStream(data, len, tag, conn));
}
void Manager::DataIn(const u_char* data, uint64 len, const string& unique)
@ -173,8 +95,6 @@ void Manager::DataIn(const u_char* data, uint64 len, const string& unique)
void Manager::DataIn(const u_char* data, uint64 len, Info* info)
{
DrainPending();
if ( ! info ) return;
info->DataIn(data, len);
@ -183,53 +103,30 @@ void Manager::DataIn(const u_char* data, uint64 len, Info* info)
RemoveFile(info->GetUnique());
}
void Manager::EndOfFile(Connection* conn)
void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn)
{
EndOfFile(conn, true);
EndOfFile(conn, false);
EndOfFile(tag, conn, true);
EndOfFile(tag, conn, false);
}
bool Manager::EndOfFile(Connection* conn, bool is_orig)
void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
{
DrainPending();
string unique = GetFileHandle(conn, is_orig);
if ( ! unique.empty() )
{
RemoveFile(unique);
return true;
}
if ( ! is_draining )
pending.push_back(new PendingEOF(conn, is_orig));
return false;
if ( IsDisabled(tag) ) return;
if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
pending.push(new PendingEOF(tag, conn));
}
void Manager::EndOfFile(const string& unique)
{
DrainPending();
RemoveFile(unique);
}
bool Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag,
void Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag,
Connection* conn, bool is_orig)
{
DrainPending();
string unique = GetFileHandle(conn, is_orig);
if ( ! unique.empty() )
{
Gap(offset, len, GetInfo(unique, conn, tag));
return true;
}
if ( ! is_draining )
pending.push_back(new PendingGap(offset, len, tag, conn, is_orig));
return false;
if ( IsDisabled(tag) ) return;
if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
pending.push(new PendingGap(offset, len, tag, conn));
}
void Manager::Gap(uint64 offset, uint64 len, const string& unique)
@ -239,30 +136,17 @@ void Manager::Gap(uint64 offset, uint64 len, const string& unique)
void Manager::Gap(uint64 offset, uint64 len, Info* info)
{
DrainPending();
if ( ! info ) return;
info->Gap(offset, len);
}
bool Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
void Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
bool is_orig)
{
DrainPending();
string unique = GetFileHandle(conn, is_orig);
if ( ! unique.empty() )
{
SetSize(size, GetInfo(unique, conn, tag));
return true;
}
if ( ! is_draining )
pending.push_back(new PendingSize(size, tag, conn, is_orig));
return false;
if ( IsDisabled(tag) ) return;
if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
pending.push(new PendingSize(size, tag, conn));
}
void Manager::SetSize(uint64 size, const string& unique)
@ -272,8 +156,6 @@ void Manager::SetSize(uint64 size, const string& unique)
void Manager::SetSize(uint64 size, Info* info)
{
DrainPending();
if ( ! info ) return;
info->SetTotalBytes(size);
@ -282,7 +164,6 @@ void Manager::SetSize(uint64 size, Info* info)
RemoveFile(info->GetUnique());
}
void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info)
{
if ( IsIgnored(info->GetUnique()) ) return;
@ -372,8 +253,6 @@ Info* Manager::Lookup(const FileID& file_id) const
void Manager::Timeout(const FileID& file_id, bool is_terminating)
{
DrainPending();
Info* info = Lookup(file_id);
if ( ! info ) return;
@ -433,3 +312,34 @@ bool Manager::IsIgnored(const string& unique)
{
return ignored.find(unique) != ignored.end();
}
bool Manager::IsDisabled(AnalyzerTag::Tag tag)
{
if ( ! disabled )
disabled = internal_const_val("FileAnalysis::disable")->AsTableVal();
Val* index = new Val(tag, TYPE_COUNT);
Val* yield = disabled->Lookup(index);
Unref(index);
if ( ! yield ) return false;
bool rval = yield->AsBool();
Unref(yield);
return rval;
}
bool Manager::QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
bool is_orig)
{
if ( ! get_file_handle ) return false;
val_list* vl = new val_list();
vl->append(new Val(tag, TYPE_COUNT));
vl->append(conn->BuildConnVal());
vl->append(new Val(is_orig, TYPE_BOOL));
mgr.QueueEvent(get_file_handle, vl);
return true;
}

View file

@ -4,7 +4,7 @@
#include <string>
#include <map>
#include <set>
#include <list>
#include <queue>
#include "Net.h"
#include "AnalyzerTags.h"
@ -20,15 +20,6 @@
namespace file_analysis {
class DrainTimer : public Timer {
public:
DrainTimer(double interval)
: Timer(network_time + interval, TIMER_FILE_ANALYSIS_DRAIN) {}
void Dispatch(double t, int is_expire);
};
/**
* Main entry point for interacting with file analysis.
*/
@ -44,10 +35,23 @@ public:
*/
void Terminate();
/**
* Associates a handle with the next element in the #pending queue, which
* will immediately push that element all the way through the file analysis
* framework, possibly evaluating any policy hooks.
*/
void ReceiveHandle(const string& handle);
/**
* Called when all events have been drained from the event queue.
* There should be no pending file input/data at this point.
*/
void EventDrainDone();
/**
* Pass in non-sequential file data.
*/
bool DataIn(const u_char* data, uint64 len, uint64 offset,
void DataIn(const u_char* data, uint64 len, uint64 offset,
AnalyzerTag::Tag tag, Connection* conn, bool is_orig);
void DataIn(const u_char* data, uint64 len, uint64 offset,
const string& unique);
@ -57,7 +61,7 @@ public:
/**
* Pass in sequential file data.
*/
bool DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
void DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
Connection* conn, bool is_orig);
void DataIn(const u_char* data, uint64 len, const string& unique);
void DataIn(const u_char* data, uint64 len, Info* info);
@ -65,14 +69,14 @@ public:
/**
* Signal the end of file data.
*/
void EndOfFile(Connection* conn);
bool EndOfFile(Connection* conn, bool is_orig);
void EndOfFile(AnalyzerTag::Tag tag, Connection* conn);
void EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig);
void EndOfFile(const string& unique);
/**
* Signal a gap in the file data stream.
*/
bool Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, Connection* conn,
void Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, Connection* conn,
bool is_orig);
void Gap(uint64 offset, uint64 len, const string& unique);
void Gap(uint64 offset, uint64 len, Info* info);
@ -80,7 +84,7 @@ public:
/**
* Provide the expected number of bytes that comprise a file.
*/
bool SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
void SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
bool is_orig);
void SetSize(uint64 size, const string& unique);
void SetSize(uint64 size, Info* info);
@ -120,13 +124,12 @@ public:
protected:
friend class InfoTimer;
friend class DrainTimer;
friend class PendingFile;
typedef map<string, Info*> StrMap;
typedef set<string> StrSet;
typedef map<FileID, Info*> IDMap;
typedef list<PendingFile*> PendingList;
typedef queue<PendingFile*> PendingQueue;
/**
* @return the Info object mapped to \a unique or a null pointer if analysis
@ -138,18 +141,6 @@ protected:
Info* GetInfo(const string& unique, Connection* conn = 0,
AnalyzerTag::Tag tag = AnalyzerTag::Error);
/**
* @return a string which can uniquely identify the file being transported
* over the connection. A script-layer function is evaluated in
* order to determine the unique string. An empty string means
* a unique handle for the file couldn't be determined at the time
* time the function was evaluated (possibly because some events
* have not yet been drained from the queue).
*/
string GetFileHandle(Connection* conn, bool is_orig) const;
string GetFileHandle(Analyzer* root, Connection* conn,
bool is_orig) const;
/**
* @return the Info object mapped to \a file_id, or a null pointer if no
* mapping exists.
@ -174,18 +165,23 @@ protected:
bool IsIgnored(const string& unique);
/**
* Attempts to forward the data from any pending file contents, i.e.
* those for which a unique file handle string could not immediately
* be determined.
* @return whether file analysis is disabled for the given analyzer.
*/
void DrainPending();
static bool IsDisabled(AnalyzerTag::Tag tag);
/**
* Queues \c get_file_handle event in order to retrieve unique file handle.
* @return true if there is a handler for the event, else false.
*/
static bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
bool is_orig);
StrMap str_map; /**< Map unique strings to \c FileAnalysis::Info records. */
IDMap id_map; /**< Map file IDs to \c FileAnalysis::Info records. */
StrSet ignored; /**< Ignored files. Will be finally removed on EOF. */
PendingList pending; /**< Files awaiting a unique handle. */
PendingQueue pending; /**< Files awaiting a unique handle. */
bool is_draining;
static TableVal* disabled; /**< Table of disabled analyzers. */
};
} // namespace file_analysis

View file

@ -20,10 +20,8 @@ static string conn_str(Connection* c)
return rval;
}
PendingFile::PendingFile(Connection* arg_conn, bool arg_is_orig,
AnalyzerTag::Tag arg_tag)
: conn(arg_conn), is_orig(arg_is_orig), creation_time(network_time),
tag(arg_tag)
PendingFile::PendingFile(Connection* arg_conn, AnalyzerTag::Tag arg_tag)
: conn(arg_conn), tag(arg_tag)
{
Ref(conn);
DBG_LOG(DBG_FILE_ANALYSIS, "New pending file: %s", conn_str(conn).c_str());
@ -36,31 +34,24 @@ PendingFile::~PendingFile()
conn_str(conn).c_str());
}
bool PendingFile::IsStale() const
Info* PendingFile::GetInfo(const string& handle) const
{
using BifConst::FileAnalysis::pending_file_timeout;
if ( creation_time + pending_file_timeout < network_time )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Stale pending file: %s",
conn_str(conn).c_str());
return true;
}
return false;
return file_mgr->GetInfo(handle, conn, tag);
}
PendingDataInChunk::PendingDataInChunk(const u_char* arg_data, uint64 arg_len,
uint64 arg_offset,
AnalyzerTag::Tag arg_tag,
Connection* arg_conn, bool arg_is_orig)
: PendingFile(arg_conn, arg_is_orig, arg_tag), len(arg_len),
Connection* arg_conn)
: PendingFile(arg_conn, arg_tag), len(arg_len),
offset(arg_offset)
{
copy_data(&data, arg_data, len);
}
bool PendingDataInChunk::Retry() const
void PendingDataInChunk::Finish(const string& handle) const
{
return file_mgr->DataIn(data, len, offset, tag, conn, is_orig);
file_mgr->DataIn(data, len, offset, GetInfo(handle));
}
PendingDataInChunk::~PendingDataInChunk()
@ -70,15 +61,15 @@ PendingDataInChunk::~PendingDataInChunk()
PendingDataInStream::PendingDataInStream(const u_char* arg_data, uint64 arg_len,
AnalyzerTag::Tag arg_tag,
Connection* arg_conn, bool arg_is_orig)
: PendingFile(arg_conn, arg_is_orig, arg_tag), len(arg_len)
Connection* arg_conn)
: PendingFile(arg_conn, arg_tag), len(arg_len)
{
copy_data(&data, arg_data, len);
}
bool PendingDataInStream::Retry() const
void PendingDataInStream::Finish(const string& handle) const
{
return file_mgr->DataIn(data, len, tag, conn, is_orig);
file_mgr->DataIn(data, len, GetInfo(handle));
}
PendingDataInStream::~PendingDataInStream()
@ -87,35 +78,34 @@ PendingDataInStream::~PendingDataInStream()
}
PendingGap::PendingGap(uint64 arg_offset, uint64 arg_len,
AnalyzerTag::Tag arg_tag, Connection* arg_conn,
bool arg_is_orig)
: PendingFile(arg_conn, arg_is_orig, arg_tag), offset(arg_offset),
AnalyzerTag::Tag arg_tag, Connection* arg_conn)
: PendingFile(arg_conn, arg_tag), offset(arg_offset),
len(arg_len)
{
}
bool PendingGap::Retry() const
void PendingGap::Finish(const string& handle) const
{
return file_mgr->Gap(offset, len, tag, conn, is_orig);
file_mgr->Gap(offset, len, GetInfo(handle));
}
PendingEOF::PendingEOF(Connection* arg_conn, bool arg_is_orig)
: PendingFile(arg_conn, arg_is_orig)
PendingEOF::PendingEOF(AnalyzerTag::Tag arg_tag, Connection* arg_conn)
: PendingFile(arg_conn, arg_tag)
{
}
bool PendingEOF::Retry() const
void PendingEOF::Finish(const string& handle) const
{
return file_mgr->EndOfFile(conn, is_orig);
file_mgr->EndOfFile(handle);
}
PendingSize::PendingSize(uint64 arg_size, AnalyzerTag::Tag arg_tag,
Connection* arg_conn, bool arg_is_orig)
: PendingFile(arg_conn, arg_is_orig, arg_tag), size(arg_size)
Connection* arg_conn)
: PendingFile(arg_conn, arg_tag), size(arg_size)
{
}
bool PendingSize::Retry() const
void PendingSize::Finish(const string& handle) const
{
return file_mgr->SetSize(size, tag, conn, is_orig);
file_mgr->SetSize(size, GetInfo(handle));
}

View file

@ -3,6 +3,7 @@
#include "AnalyzerTags.h"
#include "Conn.h"
#include "Info.h"
namespace file_analysis {
@ -11,18 +12,16 @@ public:
virtual ~PendingFile();
virtual bool Retry() const = 0;
bool IsStale() const;
virtual void Finish(const string& handle) const = 0;
protected:
PendingFile(Connection* arg_conn, bool arg_is_orig,
PendingFile(Connection* arg_conn,
AnalyzerTag::Tag arg_tag = AnalyzerTag::Error);
Info* GetInfo(const string& handle) const;
Connection* conn;
bool is_orig;
double creation_time;
AnalyzerTag::Tag tag;
};
@ -30,12 +29,12 @@ class PendingDataInChunk : public PendingFile {
public:
PendingDataInChunk(const u_char* arg_data, uint64 arg_len,
uint64 arg_offset, AnalyzerTag::Tag tag,
Connection* arg_conn, bool arg_is_orig);
uint64 arg_offset, AnalyzerTag::Tag arg_tag,
Connection* arg_conn);
virtual ~PendingDataInChunk();
virtual bool Retry() const;
virtual void Finish(const string& handle) const;
protected:
@ -48,12 +47,11 @@ class PendingDataInStream : public PendingFile {
public:
PendingDataInStream(const u_char* arg_data, uint64 arg_len,
AnalyzerTag::Tag tag, Connection* arg_conn,
bool arg_is_orig);
AnalyzerTag::Tag arg_tag, Connection* arg_conn);
virtual ~PendingDataInStream();
virtual bool Retry() const;
virtual void Finish(const string& handle) const;
protected:
@ -64,10 +62,10 @@ protected:
class PendingGap : public PendingFile {
public:
PendingGap(uint64 arg_offset, uint64 arg_len, AnalyzerTag::Tag tag,
Connection* arg_conn, bool arg_is_orig);
PendingGap(uint64 arg_offset, uint64 arg_len, AnalyzerTag::Tag arg_tag,
Connection* arg_conn);
virtual bool Retry() const;
virtual void Finish(const string& handle) const;
protected:
@ -78,18 +76,18 @@ protected:
class PendingEOF : public PendingFile {
public:
PendingEOF(Connection* arg_conn, bool arg_is_orig);
PendingEOF(AnalyzerTag::Tag arg_tag, Connection* arg_conn);
virtual bool Retry() const;
virtual void Finish(const string& handle) const;
};
class PendingSize : public PendingFile {
public:
PendingSize(uint64 arg_size, AnalyzerTag::Tag tag, Connection* arg_conn,
bool arg_is_orig);
PendingSize(uint64 arg_size, AnalyzerTag::Tag arg_tag,
Connection* arg_conn);
virtual bool Retry() const;
virtual void Finish(const string& handle) const;
protected: