diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 7f4d29d26b..07433512a2 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -316,7 +316,12 @@ type connection: record { tunnel: EncapsulatingConnVector &optional; }; +## Default amount of time a file can be inactive before the file analysis +## gives up and discards any internal state related to the file. const default_file_timeout_interval: interval = 2 mins &redef; + +## Default amount of bytes that file analysis will buffer before raising +## :bro:see:`file_new`. const default_file_bof_buffer_size: count = 1024 &redef; ## A file that Bro is analyzing. This is Bro's type for describing the basic diff --git a/scripts/base/protocols/http/file-ident.bro b/scripts/base/protocols/http/file-ident.bro index 10ff239aa0..9996a70faa 100644 --- a/scripts/base/protocols/http/file-ident.bro +++ b/scripts/base/protocols/http/file-ident.bro @@ -66,23 +66,18 @@ event file_new(f: fa_file) &priority=5 } } -event file_over_new_connection(f: fa_file) &priority=5 +event file_over_new_connection(f: fa_file, c: connection) &priority=5 { if ( ! f?$source ) return; if ( f$source != "HTTP" ) return; if ( ! f?$mime_type ) return; - if ( ! f?$conns ) return; + if ( ! c?$http ) return; # Spread the mime around (e.g. for partial content, file_type event only # happens once for the first connection, but if there's subsequent # connections to transfer the same file, they'll be lacking the mime_type # field if we don't do this). - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - if ( ! c?$http ) next; - c$http$mime_type = f$mime_type; - } + c$http$mime_type = f$mime_type; } # Tracks byte-range request / partial content response mime types, indexed diff --git a/src/event.bif b/src/event.bif index 08a2b64a84..763d3f0733 100644 --- a/src/event.bif +++ b/src/event.bif @@ -7000,17 +7000,48 @@ event event_queue_flush_point%(%); ## .. bro:see:: set_file_handle event get_file_handle%(tag: count, c: connection, is_orig: bool%); -# TODO: document +## Indicates that a analysis of a new file has begun. The analysis can be +## augmented at this time via :bro:see:`FileAnalysis::add_action`. +## +## f: The file. +## +## .. bro:see:: file_over_new_connection file_timeout file_gap file_state_remove event file_new%(f: fa_file%); -# TODO: give the new connection -event file_over_new_connection%(f: fa_file%); + +## Indicates that a file has been seen being transferred over a connection +## different from the original. +## +## f: The file. +## +## c: The new connection over which the file is seen being transferred. +## +## .. bro:see:: file_new file_timeout file_gap file_state_remove +event file_over_new_connection%(f: fa_file, c: connection%); + +## Indicates that file analysis has timed out because no activity was seen +## for the file in a while. +## +## f: The file. +## +## .. bro:see:: file_new file_over_new_connection file_gap file_state_remove +## default_file_timeout_interval event file_timeout%(f: fa_file%); -# TODO: give size of gap -event file_gap%(f: fa_file%); + +## Indicates that a chunk of the file is missing. +## +## f: The file. +## +## offset: The byte offset from the start of the file at which the gap begins. +## +## len: The number of missing bytes. +## +## .. bro:see:: file_new file_over_new_connection file_timeout file_state_remove +event file_gap%(f: fa_file, offset: count, len: count%); ## This event is generated each time file analysis is ending for a given file. ## ## f: The file. +## .. bro:see:: file_new file_over_new_connection file_timeout file_gap event file_state_remove%(f: fa_file%); ## This event is generated each time file analysis generates a digest of the diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index a4713c32fe..b45af0c281 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -145,9 +145,16 @@ void File::UpdateConnectionFields(Connection* conn) Val* idx = get_conn_id_val(conn); if ( ! conns->AsTableVal()->Lookup(idx) ) { - conns->AsTableVal()->Assign(idx, conn->BuildConnVal()); - if ( ! is_first ) - file_mgr->FileEvent(file_over_new_connection, this); + Val* conn_val = conn->BuildConnVal(); + conns->AsTableVal()->Assign(idx, conn_val); + + if ( ! is_first && FileEventAvailable(file_over_new_connection) ) + { + val_list* vl = new val_list(); + vl->append(val->Ref()); + vl->append(conn_val->Ref()); + FileEvent(file_over_new_connection, vl); + } } Unref(idx); @@ -266,8 +273,7 @@ void File::ReplayBOF() DetectTypes(bs->Bytes(), bs->Len()); - file_mgr->FileEvent(file_new, this); - mgr.Drain(); // need immediate feedback about actions to add + FileEvent(file_new); for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i ) DataIn(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len()); @@ -281,9 +287,7 @@ void File::DataIn(const u_char* data, uint64 len, uint64 offset) { // TODO: this should all really be delayed until we attempt reassembly DetectTypes(data, len); - file_mgr->FileEvent(file_new, this); - mgr.Drain(); // need immediate feedback about actions to add - actions.DrainModifications(); + FileEvent(file_new); first_chunk = false; } @@ -318,9 +322,7 @@ void File::DataIn(const u_char* data, uint64 len) if ( missed_bof ) { DetectTypes(data, len); - file_mgr->FileEvent(file_new, this); - mgr.Drain(); // need immediate feedback about actions to add - actions.DrainModifications(); + FileEvent(file_new); missed_bof = false; } @@ -366,7 +368,7 @@ void File::EndOfFile() actions.QueueRemoveAction(act->Args()); } - file_mgr->FileEvent(file_state_remove, this); + FileEvent(file_state_remove); actions.DrainModifications(); } @@ -388,8 +390,41 @@ void File::Gap(uint64 offset, uint64 len) actions.QueueRemoveAction(act->Args()); } - file_mgr->FileEvent(file_gap, this); + if ( FileEventAvailable(file_gap) ) + { + val_list* vl = new val_list(); + vl->append(val->Ref()); + vl->append(new Val(offset, TYPE_COUNT)); + vl->append(new Val(len, TYPE_COUNT)); + FileEvent(file_gap, vl); + } actions.DrainModifications(); IncrementByteCount(len, missing_bytes_idx); } + +bool File::FileEventAvailable(EventHandlerPtr h) + { + return h && ! file_mgr->IsIgnored(unique); + } + +void File::FileEvent(EventHandlerPtr h) + { + if ( ! FileEventAvailable(h) ) return; + + val_list* vl = new val_list(); + vl->append(val->Ref()); + FileEvent(h, vl); + } + +void File::FileEvent(EventHandlerPtr h, val_list* vl) + { + mgr.QueueEvent(h, vl); + + if ( h == file_new || h == file_timeout ) + { + // immediate feedback is required for these events. + mgr.Drain(); + actions.DrainModifications(); + } + } diff --git a/src/file_analysis/File.h b/src/file_analysis/File.h index aaa172b8b2..bfb24a72db 100644 --- a/src/file_analysis/File.h +++ b/src/file_analysis/File.h @@ -106,6 +106,22 @@ public: */ void Gap(uint64 offset, uint64 len); + /** + * @return true if event has a handler and the file isn't ignored. + */ + bool FileEventAvailable(EventHandlerPtr h); + + /** + * Raises an event related to the file's life-cycle, the only parameter + * to that event is the \c fa_file record.. + */ + void FileEvent(EventHandlerPtr h); + + /** + * Raises an event related to the file's life-cycle. + */ + void FileEvent(EventHandlerPtr h, val_list* vl); + protected: /** diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index 1d9849d6b8..0f9a75bb2f 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -147,17 +147,6 @@ void Manager::SetSize(uint64 size, File* file) RemoveFile(file->GetUnique()); } -void Manager::FileEvent(EventHandlerPtr h, File* file) - { - if ( ! h ) return; - if ( IsIgnored(file->GetUnique()) ) return; - - val_list * vl = new val_list(); - vl->append(file->GetVal()->Ref()); - - mgr.QueueEvent(h, vl); - } - bool Manager::PostponeTimeout(const FileID& file_id) const { File* file = Lookup(file_id); @@ -235,8 +224,7 @@ void Manager::Timeout(const FileID& file_id, bool is_terminating) file->postpone_timeout = false; - FileEvent(file_timeout, file); - mgr.Drain(); // need immediate feedback about whether to postpone + file->FileEvent(file_timeout); if ( file->postpone_timeout && ! is_terminating ) { diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h index 0fc6384e40..f01b6c8503 100644 --- a/src/file_analysis/Manager.h +++ b/src/file_analysis/Manager.h @@ -111,9 +111,9 @@ public: bool RemoveAction(const FileID& file_id, const RecordVal* args) const; /** - * Queues an event related to the file's life-cycle. + * @return whether the file mapped to \a unique is being ignored. */ - void FileEvent(EventHandlerPtr h, File* file); + bool IsIgnored(const string& unique); protected: @@ -149,11 +149,6 @@ protected: */ bool RemoveFile(const string& unique); - /** - * @return whether the file mapped to \a unique is being ignored. - */ - bool IsIgnored(const string& unique); - /** * Sets #current_handle to a unique file handle string based on what the * \c get_file_handle event derives from the connection params. The diff --git a/testing/scripts/file-analysis-test.bro b/testing/scripts/file-analysis-test.bro index 0af335e75c..3d7e86a34e 100644 --- a/testing/scripts/file-analysis-test.bro +++ b/testing/scripts/file-analysis-test.bro @@ -62,7 +62,7 @@ event file_new(f: fa_file) } } -event file_over_new_connection(f: fa_file) +event file_over_new_connection(f: fa_file, c: connection) { print "FILE_OVER_NEW_CONNECTION"; } @@ -72,7 +72,7 @@ event file_timeout(f: fa_file) print "FILE_TIMEOUT"; } -event file_gap(f: fa_file) +event file_gap(f: fa_file, offset: count, len: count) { print "FILE_GAP"; }