mirror of
https://github.com/zeek/zeek.git
synced 2025-10-10 18:48:20 +00:00
More work on the interface to add/remove file analysis actions.
Added the file extraction action and did other misc. cleanup. Most of the minimal core features/support for file analysis should be working at this point, just have to start fleshing things out.
This commit is contained in:
Jon Siwek
parent
b9d204005d
commit
f04d189d3f
8 changed files with 398 additions and 74 deletions
|
|
@ -5,6 +5,11 @@
|
|||
|
||||
using namespace file_analysis;
|
||||
|
||||
// keep in order w/ declared enum values in file_analysis.bif
|
||||
static ActionInstantiator action_factory[] = {
|
||||
Extract::Instantiate,
|
||||
};
|
||||
|
||||
Action::Action(Info* arg_info) : info(arg_info)
|
||||
{
|
||||
}
|
||||
|
|
@ -12,11 +17,41 @@ Action::Action(Info* arg_info) : info(arg_info)
|
|||
Extract::Extract(Info* arg_info, const string& arg_filename)
|
||||
: Action(arg_info), filename(arg_filename)
|
||||
{
|
||||
fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
|
||||
|
||||
if ( fd < 0 )
|
||||
{
|
||||
fd = 0;
|
||||
char buf[128];
|
||||
strerror_r(errno, buf, sizeof(buf));
|
||||
reporter->Error("cannot open %s: %s", filename.c_str(), buf);
|
||||
}
|
||||
}
|
||||
|
||||
void Extract::DeliverStream(const u_char* data, uint64 len)
|
||||
Extract::~Extract()
|
||||
{
|
||||
// TODO: write data to filename
|
||||
if ( fd )
|
||||
safe_close(fd);
|
||||
}
|
||||
|
||||
Action* Extract::Instantiate(const RecordVal* args, Info* info)
|
||||
{
|
||||
const char* field = "extract_filename";
|
||||
int off = BifType::Record::FileAnalysis::ActionArgs->FieldOffset(field);
|
||||
Val* v = args->Lookup(off);
|
||||
|
||||
if ( ! v ) return 0;
|
||||
|
||||
return new Extract(info, v->AsString()->CheckString());
|
||||
}
|
||||
|
||||
void Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
|
||||
{
|
||||
Action::DeliverChunk(data, len, offset);
|
||||
|
||||
if ( ! fd ) return;
|
||||
|
||||
safe_pwrite(fd, data, len, offset);
|
||||
}
|
||||
|
||||
static TableVal* empty_conn_id_set()
|
||||
|
|
@ -51,26 +86,36 @@ int Info::conn_uids_idx = -1;
|
|||
int Info::conn_ids_idx = -1;
|
||||
int Info::seen_bytes_idx = -1;
|
||||
int Info::total_bytes_idx = -1;
|
||||
int Info::undelivered_idx = -1;
|
||||
int Info::missing_bytes_idx = -1;
|
||||
int Info::overflow_bytes_idx = -1;
|
||||
int Info::timeout_interval_idx = -1;
|
||||
int Info::actions_idx = -1;
|
||||
int Info::action_args_idx = -1;
|
||||
|
||||
void Info::InitFieldIndices()
|
||||
{
|
||||
if ( file_id_idx != -1 ) return;
|
||||
file_id_idx = Idx("file_id");
|
||||
parent_file_id_idx = Idx("parent_file_id");
|
||||
protocol_idx = Idx("protocol");
|
||||
conn_uids_idx = Idx("conn_uids");
|
||||
conn_ids_idx = Idx("conn_ids");
|
||||
seen_bytes_idx = Idx("seen_bytes");
|
||||
total_bytes_idx = Idx("total_bytes");
|
||||
missing_bytes_idx = Idx("missing_bytes");
|
||||
overflow_bytes_idx = Idx("overflow_bytes");
|
||||
timeout_interval_idx = Idx("timeout_interval");
|
||||
actions_idx = Idx("actions");
|
||||
action_args_idx = Idx("action_args");
|
||||
}
|
||||
|
||||
Info::Info(const string& file_id, Connection* conn, const string& protocol)
|
||||
: val(0), last_activity_time(network_time), postpone_timeout(false)
|
||||
: val(0), last_activity_time(network_time), postpone_timeout(false),
|
||||
need_reassembly(false)
|
||||
{
|
||||
DBG_LOG(DBG_FILE_ANALYSIS, "Creating new Info object %s", file_id.c_str());
|
||||
|
||||
if ( file_id_idx == -1 )
|
||||
{
|
||||
file_id_idx = Idx("file_id");
|
||||
parent_file_id_idx = Idx("parent_file_id");
|
||||
protocol_idx = Idx("protocol");
|
||||
conn_uids_idx = Idx("conn_uids");
|
||||
conn_ids_idx = Idx("conn_ids");
|
||||
seen_bytes_idx = Idx("seen_bytes");
|
||||
total_bytes_idx = Idx("total_bytes");
|
||||
undelivered_idx = Idx("undelivered");
|
||||
timeout_interval_idx = Idx("timeout_interval");
|
||||
}
|
||||
InitFieldIndices();
|
||||
|
||||
val = new RecordVal(BifType::Record::FileAnalysis::Info);
|
||||
// TODO: hash/prettify file_id for script layer presentation
|
||||
|
|
@ -82,13 +127,13 @@ Info::Info(const string& file_id, Connection* conn, const string& protocol)
|
|||
val->Assign(protocol_idx, new StringVal(protocol.c_str()));
|
||||
|
||||
ScheduleInactivityTimer();
|
||||
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, this);
|
||||
}
|
||||
|
||||
Info::~Info()
|
||||
{
|
||||
for ( size_t i = 0; i < analyzers.size(); ++i )
|
||||
delete analyzers[i];
|
||||
ActionMap::const_iterator it;
|
||||
for ( it = actions.begin(); it != actions.end(); ++it )
|
||||
delete it->second;
|
||||
|
||||
DBG_LOG(DBG_FILE_ANALYSIS, "Destroying Info object %s", FileID().c_str());
|
||||
Unref(val);
|
||||
|
|
@ -109,7 +154,7 @@ void Info::UpdateConnectionFields(Connection* conn)
|
|||
conn_ids->AsTableVal()->Assign(get_conn_id_val(conn), 0);
|
||||
}
|
||||
|
||||
uint64 Info::FieldDefaultCount(int idx) const
|
||||
uint64 Info::LookupFieldDefaultCount(int idx) const
|
||||
{
|
||||
Val* v = val->LookupWithDefault(idx);
|
||||
uint64 rval = v->AsCount();
|
||||
|
|
@ -117,7 +162,7 @@ uint64 Info::FieldDefaultCount(int idx) const
|
|||
return rval;
|
||||
}
|
||||
|
||||
double Info::FieldDefaultInterval(int idx) const
|
||||
double Info::LookupFieldDefaultInterval(int idx) const
|
||||
{
|
||||
Val* v = val->LookupWithDefault(idx);
|
||||
double rval = v->AsInterval();
|
||||
|
|
@ -136,7 +181,7 @@ int Info::Idx(const string& field)
|
|||
|
||||
double Info::TimeoutInterval() const
|
||||
{
|
||||
return FieldDefaultInterval(timeout_interval_idx);
|
||||
return LookupFieldDefaultInterval(timeout_interval_idx);
|
||||
}
|
||||
|
||||
string Info::FileID() const
|
||||
|
|
@ -144,10 +189,10 @@ string Info::FileID() const
|
|||
return val->Lookup(file_id_idx)->AsString()->CheckString();
|
||||
}
|
||||
|
||||
void Info::IncrementSeenBytes(uint64 size)
|
||||
void Info::IncrementByteCount(uint64 size, int field_idx)
|
||||
{
|
||||
uint64 old = FieldDefaultCount(seen_bytes_idx);
|
||||
val->Assign(seen_bytes_idx, new Val(old + size, TYPE_COUNT));
|
||||
uint64 old = LookupFieldDefaultCount(field_idx);
|
||||
val->Assign(field_idx, new Val(old + size, TYPE_COUNT));
|
||||
}
|
||||
|
||||
void Info::SetTotalBytes(uint64 size)
|
||||
|
|
@ -159,7 +204,7 @@ bool Info::IsComplete() const
|
|||
{
|
||||
Val* total = val->Lookup(total_bytes_idx);
|
||||
if ( ! total ) return false;
|
||||
if ( FieldDefaultCount(seen_bytes_idx) >= total->AsCount() )
|
||||
if ( LookupFieldDefaultCount(seen_bytes_idx) >= total->AsCount() )
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
|
@ -169,6 +214,88 @@ void Info::ScheduleInactivityTimer() const
|
|||
timer_mgr->Add(new InfoTimer(network_time, FileID(), TimeoutInterval()));
|
||||
}
|
||||
|
||||
bool Info::AddAction(EnumVal* act, RecordVal* args)
|
||||
{
|
||||
if ( actions.find(act->AsEnum()) != actions.end() ) return false;
|
||||
|
||||
Action* a = action_factory[act->AsEnum()](args, this);
|
||||
|
||||
if ( ! a ) return false;
|
||||
|
||||
DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d for file id %s", act->AsEnum(),
|
||||
FileID().c_str());
|
||||
actions[act->AsEnum()] = a;
|
||||
|
||||
VectorVal* av = val->LookupWithDefault(actions_idx)->AsVectorVal();
|
||||
VectorVal* aav = val->LookupWithDefault(action_args_idx)->AsVectorVal();
|
||||
|
||||
av->Assign(av->Size(), act->Ref(), 0);
|
||||
aav->Assign(aav->Size(), args->Ref(), 0);
|
||||
|
||||
Unref(av);
|
||||
Unref(aav);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Info::RemoveAction(EnumVal* act)
|
||||
{
|
||||
ActionMap::iterator it = actions.find(act->AsEnum());
|
||||
|
||||
if ( it == actions.end() ) return false;
|
||||
|
||||
DBG_LOG(DBG_FILE_ANALYSIS, "Remove action %d for file id %s", act->AsEnum(),
|
||||
FileID().c_str());
|
||||
delete it->second;
|
||||
actions.erase(it);
|
||||
return true;
|
||||
}
|
||||
|
||||
void Info::DataIn(const u_char* data, uint64 len, uint64 offset)
|
||||
{
|
||||
ActionMap::const_iterator it;
|
||||
for ( it = actions.begin(); it != actions.end(); ++it )
|
||||
it->second->DeliverChunk(data, len, offset);
|
||||
|
||||
// TODO: check reassembly requirement based on buffer size in record
|
||||
if ( ! need_reassembly ) return;
|
||||
|
||||
// TODO: reassembly stuff, possibly having to deliver chunks if buffer full
|
||||
// and incrememt overflow bytes
|
||||
|
||||
IncrementByteCount(len, seen_bytes_idx);
|
||||
}
|
||||
|
||||
void Info::DataIn(const u_char* data, uint64 len)
|
||||
{
|
||||
ActionMap::const_iterator it;
|
||||
for ( it = actions.begin(); it != actions.end(); ++it )
|
||||
{
|
||||
it->second->DeliverStream(data, len);
|
||||
uint64 offset = LookupFieldDefaultCount(seen_bytes_idx) +
|
||||
LookupFieldDefaultCount(missing_bytes_idx);
|
||||
it->second->DeliverChunk(data, len, offset);
|
||||
}
|
||||
|
||||
IncrementByteCount(len, seen_bytes_idx);
|
||||
}
|
||||
|
||||
void Info::EndOfFile()
|
||||
{
|
||||
ActionMap::const_iterator it;
|
||||
for ( it = actions.begin(); it != actions.end(); ++it )
|
||||
it->second->EndOfFile();
|
||||
}
|
||||
|
||||
void Info::Gap(uint64 offset, uint64 len)
|
||||
{
|
||||
ActionMap::const_iterator it;
|
||||
for ( it = actions.begin(); it != actions.end(); ++it )
|
||||
it->second->Undelivered(offset, len);
|
||||
|
||||
IncrementByteCount(len, missing_bytes_idx);
|
||||
}
|
||||
|
||||
void InfoTimer::Dispatch(double t, int is_expire)
|
||||
{
|
||||
Info* info = file_mgr->Lookup(file_id);
|
||||
|
|
@ -218,7 +345,7 @@ static void check_file_done(Info* info)
|
|||
if ( info->IsComplete() )
|
||||
{
|
||||
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, info);
|
||||
file_mgr->Remove(info->FileID());
|
||||
file_mgr->RemoveFile(info->FileID());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -226,9 +353,7 @@ void Manager::DataIn(const string& file_id, const u_char* data, uint64 len,
|
|||
uint64 offset, Connection* conn, const string& protocol)
|
||||
{
|
||||
Info* info = IDtoInfo(file_id, conn, protocol);
|
||||
info->UpdateLastActivityTime();
|
||||
info->UpdateConnectionFields(conn);
|
||||
// TODO: more stuff
|
||||
info->DataIn(data, len, offset);
|
||||
check_file_done(info);
|
||||
}
|
||||
|
||||
|
|
@ -236,9 +361,7 @@ void Manager::DataIn(const string& file_id, const u_char* data, uint64 len,
|
|||
Connection* conn, const string& protocol)
|
||||
{
|
||||
Info* info = IDtoInfo(file_id, conn, protocol);
|
||||
info->UpdateLastActivityTime();
|
||||
info->UpdateConnectionFields(conn);
|
||||
// TODO: more stuff
|
||||
info->DataIn(data, len);
|
||||
check_file_done(info);
|
||||
}
|
||||
|
||||
|
|
@ -246,18 +369,22 @@ void Manager::EndOfFile(const string& file_id, Connection* conn,
|
|||
const string& protocol)
|
||||
{
|
||||
Info* info = IDtoInfo(file_id, conn, protocol);
|
||||
info->UpdateLastActivityTime();
|
||||
info->UpdateConnectionFields(conn);
|
||||
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, info);
|
||||
Remove(file_id);
|
||||
info->EndOfFile();
|
||||
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_EOF, info);
|
||||
}
|
||||
|
||||
void Manager::Gap(const string& file_id, uint64 offset, uint64 len,
|
||||
Connection* conn, const string& protocol)
|
||||
{
|
||||
Info* info = IDtoInfo(file_id, conn, protocol);
|
||||
info->Gap(offset, len);
|
||||
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_GAP, info);
|
||||
}
|
||||
|
||||
void Manager::SetSize(const string& file_id, uint64 size,
|
||||
Connection* conn, const string& protocol)
|
||||
{
|
||||
Info* info = IDtoInfo(file_id, conn, protocol);
|
||||
info->UpdateLastActivityTime();
|
||||
info->UpdateConnectionFields(conn);
|
||||
info->SetTotalBytes(size);
|
||||
check_file_done(info);
|
||||
}
|
||||
|
|
@ -288,12 +415,41 @@ bool Manager::PostponeTimeout(const string& file_id) const
|
|||
return true;
|
||||
}
|
||||
|
||||
bool Manager::AddAction(const string& file_id, EnumVal* act,
|
||||
RecordVal* args) const
|
||||
{
|
||||
Info* info = Lookup(file_id);
|
||||
|
||||
if ( ! info ) return false;
|
||||
|
||||
return info->AddAction(act, args);
|
||||
}
|
||||
|
||||
bool Manager::RemoveAction(const string& file_id, EnumVal* act) const
|
||||
{
|
||||
Info* info = Lookup(file_id);
|
||||
|
||||
if ( ! info ) return false;
|
||||
|
||||
return info->RemoveAction(act);
|
||||
}
|
||||
|
||||
Info* Manager::IDtoInfo(const string& file_id, Connection* conn,
|
||||
const string& protocol)
|
||||
{
|
||||
Info* rval = file_map[file_id];
|
||||
|
||||
if ( ! rval )
|
||||
{
|
||||
rval = file_map[file_id] = new Info(file_id, conn, protocol);
|
||||
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, rval);
|
||||
}
|
||||
else
|
||||
{
|
||||
rval->UpdateLastActivityTime();
|
||||
rval->UpdateConnectionFields(conn);
|
||||
}
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
|
|
@ -312,7 +468,7 @@ void Manager::Timeout(const string& file_id, bool is_terminating)
|
|||
|
||||
if ( ! info ) return;
|
||||
|
||||
EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info);
|
||||
Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info);
|
||||
|
||||
if ( info->postpone_timeout && ! is_terminating )
|
||||
{
|
||||
|
|
@ -326,15 +482,16 @@ void Manager::Timeout(const string& file_id, bool is_terminating)
|
|||
DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s",
|
||||
info->FileID().c_str());
|
||||
|
||||
Remove(file_id);
|
||||
RemoveFile(file_id);
|
||||
}
|
||||
|
||||
void Manager::Remove(const string& file_id)
|
||||
bool Manager::RemoveFile(const string& file_id)
|
||||
{
|
||||
FileMap::iterator it = file_map.find(file_id);
|
||||
|
||||
if ( it == file_map.end() ) return;
|
||||
if ( it == file_map.end() ) return false;
|
||||
|
||||
delete it->second;
|
||||
file_map.erase(it);
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue