Merge remote-tracking branch 'origin/topic/seth/faf-updates'

* origin/topic/seth/faf-updates: (27 commits)
  Undoing the FTP tests I updated earlier.
  Update the last two btest FAF tests.
  File analysis fixes and test updates.
  Fix a bug with getting analyzer tags.
  A few test updates.
  Some tests work now (at least they all don't fail anymore!)
  Forgot a file.
  Added protocol description functions that provide a super compressed log representation.
  Fix a bug where orig file information in http wasn't working right.
  Added mime types to http.log
  Clean up queued but unused file_over_new_connections event args.
  Add jar files to the default MHR lookups.
  Adding CAB files for MHR checking.
  Improve malware hash registry script.
  Fix a small issue with finding smtp entities.
  Added support for files to the notice framework.
  Make the custom libmagic database a git submodule.
  Add an is_orig parameter to file_over_new_connection event.
  Make magic for emitting application/msword mime type less strict.
  Disable more libmagic builtin checks that override the magic database.
  ...

Conflicts:
	doc/scripts/DocSourcesList.cmake
	scripts/base/init-bare.bro
	scripts/test-all-policy.bro
	testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
	testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
This commit is contained in:
Robin Sommer 2013-07-29 14:21:52 -07:00
commit 984e9793db
196 changed files with 1548 additions and 5033 deletions

View file

@ -75,7 +75,8 @@ void File::StaticInit()
File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
bool is_orig)
: id(file_id), val(0), postpone_timeout(false), first_chunk(true),
missed_bof(false), need_reassembly(false), done(false), analyzers(this)
missed_bof(false), need_reassembly(false), done(false),
did_file_new_event(false), analyzers(this)
{
StaticInit();
@ -89,7 +90,7 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
// add source, connection, is_orig fields
SetSource(analyzer_mgr->GetAnalyzerName(tag));
val->Assign(is_orig_idx, new Val(is_orig, TYPE_BOOL));
UpdateConnectionFields(conn);
UpdateConnectionFields(conn, is_orig);
}
UpdateLastActivityTime();
@ -99,6 +100,13 @@ File::~File()
{
DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str());
Unref(val);
// Queue may not be empty in the case where only content gaps were seen.
while ( ! fonc_queue.empty() )
{
delete_vals(fonc_queue.front().second);
fonc_queue.pop();
}
}
void File::UpdateLastActivityTime()
@ -111,18 +119,15 @@ double File::GetLastActivityTime() const
return val->Lookup(last_active_idx)->AsTime();
}
void File::UpdateConnectionFields(Connection* conn)
void File::UpdateConnectionFields(Connection* conn, bool is_orig)
{
if ( ! conn )
return;
Val* conns = val->Lookup(conns_idx);
bool is_first = false;
if ( ! conns )
{
is_first = true;
conns = empty_connection_table();
val->Assign(conns_idx, conns);
}
@ -133,12 +138,18 @@ void File::UpdateConnectionFields(Connection* conn)
Val* conn_val = conn->BuildConnVal();
conns->AsTableVal()->Assign(idx, conn_val);
if ( ! is_first && FileEventAvailable(file_over_new_connection) )
if ( FileEventAvailable(file_over_new_connection) )
{
val_list* vl = new val_list();
vl->append(val->Ref());
vl->append(conn_val->Ref());
FileEvent(file_over_new_connection, vl);
vl->append(new Val(is_orig, TYPE_BOOL));
if ( did_file_new_event )
FileEvent(file_over_new_connection, vl);
else
fonc_queue.push(pair<EventHandlerPtr, val_list*>(
file_over_new_connection, vl));
}
}
@ -435,6 +446,18 @@ void File::FileEvent(EventHandlerPtr h, val_list* vl)
{
mgr.QueueEvent(h, vl);
if ( h == file_new )
{
did_file_new_event = true;
while ( ! fonc_queue.empty() )
{
pair<EventHandlerPtr, val_list*> p = fonc_queue.front();
mgr.QueueEvent(p.first, p.second);
fonc_queue.pop();
}
}
if ( h == file_new || h == file_timeout )
{
// immediate feedback is required for these events.