mirror of
https://github.com/zeek/zeek.git
synced 2025-10-04 15:48:19 +00:00
Merge remote-tracking branch 'origin/master' into topic/seth/files-tracking
Conflicts: src/Reassem.cc src/Reassem.h src/analyzer/protocol/tcp/TCP_Reassembler.cc testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log
This commit is contained in:
Seth Hall
commit
8d9940c8c3
658 changed files with 22041 additions and 5836 deletions
|
|
@ -20,13 +20,30 @@ string Manager::salt;
|
|||
|
||||
Manager::Manager()
|
||||
: plugin::ComponentManager<file_analysis::Tag,
|
||||
file_analysis::Component>("Files")
|
||||
file_analysis::Component>("Files"),
|
||||
id_map(), ignored(), current_file_id(), magic_state()
|
||||
{
|
||||
}
|
||||
|
||||
Manager::~Manager()
|
||||
{
|
||||
Terminate();
|
||||
// Have to assume that too much of Bro has been shutdown by this point
|
||||
// to do anything more than reclaim memory.
|
||||
|
||||
File* f;
|
||||
bool* b;
|
||||
|
||||
IterCookie* it = id_map.InitForIteration();
|
||||
|
||||
while ( (f = id_map.NextEntry(it)) )
|
||||
delete f;
|
||||
|
||||
it = ignored.InitForIteration();
|
||||
|
||||
while( (b = ignored.NextEntry(it)) )
|
||||
delete b;
|
||||
|
||||
delete magic_state;
|
||||
}
|
||||
|
||||
void Manager::InitPreScript()
|
||||
|
|
@ -42,15 +59,30 @@ void Manager::InitPostScript()
|
|||
{
|
||||
}
|
||||
|
||||
void Manager::InitMagic()
|
||||
{
|
||||
delete magic_state;
|
||||
magic_state = rule_matcher->InitFileMagic();
|
||||
}
|
||||
|
||||
void Manager::Terminate()
|
||||
{
|
||||
vector<string> keys;
|
||||
|
||||
for ( IDMap::iterator it = id_map.begin(); it != id_map.end(); ++it )
|
||||
keys.push_back(it->first);
|
||||
IterCookie* it = id_map.InitForIteration();
|
||||
HashKey* key;
|
||||
|
||||
while ( id_map.NextEntry(key, it) )
|
||||
{
|
||||
keys.push_back(string(static_cast<const char*>(key->Key()),
|
||||
key->Size()));
|
||||
delete key;
|
||||
}
|
||||
|
||||
for ( size_t i = 0; i < keys.size(); ++i )
|
||||
Timeout(keys[i], true);
|
||||
|
||||
mgr.Drain();
|
||||
}
|
||||
|
||||
string Manager::HashHandle(const string& handle) const
|
||||
|
|
@ -75,36 +107,47 @@ void Manager::SetHandle(const string& handle)
|
|||
current_file_id = HashHandle(handle);
|
||||
}
|
||||
|
||||
void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
|
||||
analyzer::Tag tag, Connection* conn, bool is_orig)
|
||||
string Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
|
||||
analyzer::Tag tag, Connection* conn, bool is_orig,
|
||||
const string& precomputed_id)
|
||||
{
|
||||
GetFileHandle(tag, conn, is_orig);
|
||||
File* file = GetFile(current_file_id, conn, tag, is_orig);
|
||||
string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
|
||||
File* file = GetFile(id, conn, tag, is_orig);
|
||||
|
||||
if ( ! file )
|
||||
return;
|
||||
return "";
|
||||
|
||||
file->DataIn(data, len, offset);
|
||||
|
||||
if ( file->IsComplete() )
|
||||
{
|
||||
RemoveFile(file->GetID());
|
||||
return "";
|
||||
}
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
void Manager::DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
|
||||
Connection* conn, bool is_orig)
|
||||
string Manager::DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
|
||||
Connection* conn, bool is_orig, const string& precomputed_id)
|
||||
{
|
||||
GetFileHandle(tag, conn, is_orig);
|
||||
string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
|
||||
// Sequential data input shouldn't be going over multiple conns, so don't
|
||||
// do the check to update connection set.
|
||||
File* file = GetFile(current_file_id, conn, tag, is_orig, false);
|
||||
File* file = GetFile(id, conn, tag, is_orig, false);
|
||||
|
||||
if ( ! file )
|
||||
return;
|
||||
return "";
|
||||
|
||||
file->DataIn(data, len);
|
||||
|
||||
if ( file->IsComplete() )
|
||||
{
|
||||
RemoveFile(file->GetID());
|
||||
return "";
|
||||
}
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
void Manager::DataIn(const u_char* data, uint64 len, const string& file_id,
|
||||
|
|
@ -133,8 +176,7 @@ void Manager::EndOfFile(analyzer::Tag tag, Connection* conn)
|
|||
void Manager::EndOfFile(analyzer::Tag tag, Connection* conn, bool is_orig)
|
||||
{
|
||||
// Don't need to create a file if we're just going to remove it right away.
|
||||
GetFileHandle(tag, conn, is_orig);
|
||||
RemoveFile(current_file_id);
|
||||
RemoveFile(GetFileID(tag, conn, is_orig));
|
||||
}
|
||||
|
||||
void Manager::EndOfFile(const string& file_id)
|
||||
|
|
@ -142,31 +184,37 @@ void Manager::EndOfFile(const string& file_id)
|
|||
RemoveFile(file_id);
|
||||
}
|
||||
|
||||
void Manager::Gap(uint64 offset, uint64 len, analyzer::Tag tag,
|
||||
Connection* conn, bool is_orig)
|
||||
string Manager::Gap(uint64 offset, uint64 len, analyzer::Tag tag,
|
||||
Connection* conn, bool is_orig, const string& precomputed_id)
|
||||
{
|
||||
GetFileHandle(tag, conn, is_orig);
|
||||
File* file = GetFile(current_file_id, conn, tag, is_orig);
|
||||
string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
|
||||
File* file = GetFile(id, conn, tag, is_orig);
|
||||
|
||||
if ( ! file )
|
||||
return;
|
||||
return "";
|
||||
|
||||
file->Gap(offset, len);
|
||||
return id;
|
||||
}
|
||||
|
||||
void Manager::SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
|
||||
bool is_orig)
|
||||
string Manager::SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
|
||||
bool is_orig, const string& precomputed_id)
|
||||
{
|
||||
GetFileHandle(tag, conn, is_orig);
|
||||
File* file = GetFile(current_file_id, conn, tag, is_orig);
|
||||
string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
|
||||
File* file = GetFile(id, conn, tag, is_orig);
|
||||
|
||||
if ( ! file )
|
||||
return;
|
||||
return "";
|
||||
|
||||
file->SetTotalBytes(size);
|
||||
|
||||
if ( file->IsComplete() )
|
||||
{
|
||||
RemoveFile(file->GetID());
|
||||
return "";
|
||||
}
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
bool Manager::SetTimeoutInterval(const string& file_id, double interval) const
|
||||
|
|
@ -258,11 +306,12 @@ File* Manager::GetFile(const string& file_id, Connection* conn,
|
|||
if ( IsIgnored(file_id) )
|
||||
return 0;
|
||||
|
||||
File* rval = id_map[file_id];
|
||||
File* rval = id_map.Lookup(file_id.c_str());
|
||||
|
||||
if ( ! rval )
|
||||
{
|
||||
rval = id_map[file_id] = new File(file_id, conn, tag, is_orig);
|
||||
rval = new File(file_id, conn, tag, is_orig);
|
||||
id_map.Insert(file_id.c_str(), rval);
|
||||
rval->ScheduleInactivityTimer();
|
||||
|
||||
if ( IsIgnored(file_id) )
|
||||
|
|
@ -281,12 +330,7 @@ File* Manager::GetFile(const string& file_id, Connection* conn,
|
|||
|
||||
File* Manager::LookupFile(const string& file_id) const
|
||||
{
|
||||
IDMap::const_iterator it = id_map.find(file_id);
|
||||
|
||||
if ( it == id_map.end() )
|
||||
return 0;
|
||||
|
||||
return it->second;
|
||||
return id_map.Lookup(file_id.c_str());
|
||||
}
|
||||
|
||||
void Manager::Timeout(const string& file_id, bool is_terminating)
|
||||
|
|
@ -317,48 +361,49 @@ void Manager::Timeout(const string& file_id, bool is_terminating)
|
|||
|
||||
bool Manager::IgnoreFile(const string& file_id)
|
||||
{
|
||||
if ( id_map.find(file_id) == id_map.end() )
|
||||
if ( ! id_map.Lookup(file_id.c_str()) )
|
||||
return false;
|
||||
|
||||
DBG_LOG(DBG_FILE_ANALYSIS, "Ignore FileID %s", file_id.c_str());
|
||||
|
||||
ignored.insert(file_id);
|
||||
|
||||
delete ignored.Insert(file_id.c_str(), new bool);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Manager::RemoveFile(const string& file_id)
|
||||
{
|
||||
IDMap::iterator it = id_map.find(file_id);
|
||||
HashKey key(file_id.c_str());
|
||||
// Can't remove from the dictionary/map right away as invoking EndOfFile
|
||||
// may cause some events to be executed which actually depend on the file
|
||||
// still being in the dictionary/map.
|
||||
File* f = static_cast<File*>(id_map.Lookup(&key));
|
||||
|
||||
if ( it == id_map.end() )
|
||||
if ( ! f )
|
||||
return false;
|
||||
|
||||
DBG_LOG(DBG_FILE_ANALYSIS, "Remove FileID %s", file_id.c_str());
|
||||
|
||||
it->second->EndOfFile();
|
||||
|
||||
delete it->second;
|
||||
id_map.erase(file_id);
|
||||
ignored.erase(file_id);
|
||||
|
||||
f->EndOfFile();
|
||||
delete f;
|
||||
id_map.Remove(&key);
|
||||
delete static_cast<bool*>(ignored.Remove(&key));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Manager::IsIgnored(const string& file_id)
|
||||
{
|
||||
return ignored.find(file_id) != ignored.end();
|
||||
return ignored.Lookup(file_id.c_str()) != 0;
|
||||
}
|
||||
|
||||
void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig)
|
||||
string Manager::GetFileID(analyzer::Tag tag, Connection* c, bool is_orig)
|
||||
{
|
||||
current_file_id.clear();
|
||||
|
||||
if ( IsDisabled(tag) )
|
||||
return;
|
||||
return "";
|
||||
|
||||
if ( ! get_file_handle )
|
||||
return;
|
||||
return "";
|
||||
|
||||
EnumVal* tagval = tag.AsEnumVal();
|
||||
Ref(tagval);
|
||||
|
|
@ -370,6 +415,7 @@ void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig)
|
|||
|
||||
mgr.QueueEvent(get_file_handle, vl);
|
||||
mgr.Drain(); // need file handle immediately so we don't have to buffer data
|
||||
return current_file_id;
|
||||
}
|
||||
|
||||
bool Manager::IsDisabled(analyzer::Tag tag)
|
||||
|
|
@ -411,3 +457,47 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const
|
|||
|
||||
return c->Factory()(args, f);
|
||||
}
|
||||
|
||||
RuleMatcher::MIME_Matches* Manager::DetectMIME(const u_char* data, uint64 len,
|
||||
RuleMatcher::MIME_Matches* rval) const
|
||||
{
|
||||
if ( ! magic_state )
|
||||
reporter->InternalError("file magic signature state not initialized");
|
||||
|
||||
rval = rule_matcher->Match(magic_state, data, len, rval);
|
||||
rule_matcher->ClearFileMagicState(magic_state);
|
||||
return rval;
|
||||
}
|
||||
|
||||
string Manager::DetectMIME(const u_char* data, uint64 len) const
|
||||
{
|
||||
RuleMatcher::MIME_Matches matches;
|
||||
DetectMIME(data, len, &matches);
|
||||
|
||||
if ( matches.empty() )
|
||||
return "";
|
||||
|
||||
return *(matches.begin()->second.begin());
|
||||
}
|
||||
|
||||
VectorVal* file_analysis::GenMIMEMatchesVal(const RuleMatcher::MIME_Matches& m)
|
||||
{
|
||||
VectorVal* rval = new VectorVal(mime_matches);
|
||||
|
||||
for ( RuleMatcher::MIME_Matches::const_iterator it = m.begin();
|
||||
it != m.end(); ++it )
|
||||
{
|
||||
RecordVal* element = new RecordVal(mime_match);
|
||||
|
||||
for ( set<string>::const_iterator it2 = it->second.begin();
|
||||
it2 != it->second.end(); ++it2 )
|
||||
{
|
||||
element->Assign(0, new Val(it->first, TYPE_INT));
|
||||
element->Assign(1, new StringVal(*it2));
|
||||
}
|
||||
|
||||
rval->Assign(rval->Size(), element);
|
||||
}
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue